Repository: wannesm/LeuvenMapMatching Branch: master Commit: 9ca9f0b73665 Files: 69 Total size: 352.0 KB Directory structure: gitextract_6vcq1ios/ ├── .gitignore ├── .readthedocs.yaml ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── docs/ │ ├── Makefile │ ├── classes/ │ │ ├── map/ │ │ │ ├── BaseMap.rst │ │ │ ├── InMemMap.rst │ │ │ └── SqliteMap.rst │ │ ├── matcher/ │ │ │ ├── BaseMatcher.rst │ │ │ ├── BaseMatching.rst │ │ │ ├── DistanceMatcher.rst │ │ │ └── SimpleMatcher.rst │ │ ├── overview.rst │ │ └── util/ │ │ └── Segment.rst │ ├── conf.py │ ├── index.rst │ ├── make.bat │ ├── requirements.txt │ └── usage/ │ ├── customdistributions.rst │ ├── debug.rst │ ├── incremental.rst │ ├── installation.rst │ ├── introduction.rst │ ├── latitudelongitude.rst │ ├── openstreetmap.rst │ └── visualisation.rst ├── leuvenmapmatching/ │ ├── __init__.py │ ├── map/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── inmem.py │ │ └── sqlite.py │ ├── matcher/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── distance.py │ │ ├── newsonkrumm.py │ │ └── simple.py │ ├── util/ │ │ ├── __init__.py │ │ ├── debug.py │ │ ├── dist_euclidean.py │ │ ├── dist_latlon.py │ │ ├── dist_latlon_nvector.py │ │ ├── evaluation.py │ │ ├── gpx.py │ │ ├── kalman.py │ │ ├── openstreetmap.py │ │ ├── projections.py │ │ └── segment.py │ └── visualization.py ├── setup.cfg ├── setup.py └── tests/ ├── examples/ │ ├── example_1_simple.py │ └── example_using_osmnx_and_geopandas.py ├── rsrc/ │ ├── bug2/ │ │ └── readme.md │ ├── newson_krumm_2009/ │ │ └── readme.md │ └── path_latlon/ │ ├── readme.md │ ├── route.gpx │ └── route2.gpx ├── test_bugs.py ├── test_conversion.py ├── test_examples.py ├── test_newsonkrumm2009.py ├── test_nonemitting.py ├── test_nonemitting_circle.py ├── test_parallelroads.py ├── test_path.py ├── test_path_latlon.py └── test_path_onlyedges.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ .cache .eggs .idea dist *.egg-info venv* README .ipynb_checkpoints *.zip *.gv *.gv.pdf *.xml tests/route.gpx examples/Leuven\ Stadswandeling* .git-old .pytest_cache docs/_build build cache baselines ./examples *.pkl ================================================ FILE: .readthedocs.yaml ================================================ version: 2 build: os: ubuntu-22.04 tools: python: "3.11" sphinx: configuration: docs/conf.py python: install: - requirements: docs/requirements.txt ================================================ FILE: LICENSE ================================================ Leuven.MapMatching ------------------ Copyright 2015-2018 KU Leuven, DTAI Research Group Copyright 2017-2018 Sirris Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. This package/repository contains code from the dtaimapmatching project as well as some open-source works: Latitude/longitude spherical geodesy tools ------------------------------------------ Latitude/longitude spherical geodesy tools (c) Chris Veness 2002-2017, MIT Licence www.movable-type.co.uk/scripts/latlong.html www.movable-type.co.uk/scripts/geodesy/docs/module-latlon-spherical.html Nvector ------- Gade, K. (2010). A Nonsingular Horizontal Position Representation, The Journal of Navigation, Volume 63, Issue 03, pp 395-417, July 2010. (www.navlab.net/Publications/A_Nonsingular_Horizontal_Position_Representation.pdf) This paper should be cited in publications using this library. Copyright (c) 2015, Norwegian Defence Research Establishment (FFI) All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above publication information, copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above publication information, copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: MANIFEST.in ================================================ include README.md include LICENSE ================================================ FILE: Makefile ================================================ .PHONY: test3 test3: @#export PYTHONPATH=.;./venv/bin/py.test --ignore=venv -vv python3 setup.py test .PHONY: test2 test2: python2 setup.py test .PHONY: test test: test3 test2 .PHONY: version version: @python3 setup.py --version .PHONY: prepare_dist prepare_dist: rm -rf dist/* python3 setup.py sdist bdist_wheel .PHONY: prepare_tag prepare_tag: @echo "Check whether repo is clean" git diff-index --quiet HEAD @echo "Check correct branch" if [[ "$$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then echo 'Not master branch'; exit 1; fi @echo "Add tag" git tag "v$$(python3 setup.py --version)" git push --tags .PHONY: deploy deploy: prepare_dist prepare_tag @echo "Check whether repo is clean" git diff-index --quiet HEAD @echo "Start uploading" twine upload --repository leuvenmapmatching dist/* .PHONY: docs docs: export PYTHONPATH=..; cd docs; make html .PHONY: docsclean docsclean: cd docs; make clean .PHONY: clean clean: docsclean ================================================ FILE: README.md ================================================ # Leuven.MapMatching [![PyPi Version](https://img.shields.io/pypi/v/leuvenmapmatching.svg)](https://pypi.org/project/leuvenmapmatching/) [![Documentation Status](https://readthedocs.org/projects/leuvenmapmatching/badge/?version=latest)](https://leuvenmapmatching.readthedocs.io/en/latest/?badge=latest) Align a trace of GPS measurements to a map or road segments. The matching is based on a Hidden Markov Model (HMM) with non-emitting states. The model can deal with missing data and you can plug in custom transition and emission probability distributions. ![example](http://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/example1.png?v=2) Main reference: > Meert Wannes, Mathias Verbeke, "HMM with Non-Emitting States for Map Matching", > European Conference on Data Analysis (ECDA), Paderborn, Germany, 2018. Other references: > Devos Laurens, Vandebril Raf (supervisor), Meert Wannes (supervisor), > "Traffic patterns revealed through matrix functions and map matching", > Master thesis, Faculty of Engineering Science, KU Leuven, 2018 ## Installation and usage $ pip install leuvenmapmatching More information and examples: [leuvenmapmatching.readthedocs.io](https://leuvenmapmatching.readthedocs.io) ## Dependencies Required: - [numpy](http://www.numpy.org) - [scipy](https://www.scipy.org) Optional (only loaded when methods are called to rely on these packages): - [matplotlib](http://matplotlib.org): For visualisation - [smopy](https://github.com/rossant/smopy): For visualisation - [nvector](https://github.com/pbrod/Nvector): For latitude-longitude computations - [gpxpy](https://github.com/tkrajina/gpxpy): To import GPX files - [pykalman](https://pykalman.github.io): So smooth paths using a Kalman filter - [pyproj](https://jswhit.github.io/pyproj/): To project latitude-longitude coordinates to an XY-plane - [rtree](http://toblerity.org/rtree/): To quickly search locations ## Contact Wannes Meert, DTAI, KU Leuven wannes.meert@cs.kuleuven.be https://dtai.cs.kuleuven.be Mathias Verbeke, Sirris mathias.verbeke@sirris.be http://www.sirris.be/expertise/data-innovation Developed with the support of [Elucidata.be](http://www.elucidata.be). ## License Copyright 2015-2022, KU Leuven - DTAI Research Group, Sirris - Elucidata Group Apache License, Version 2.0. ================================================ FILE: docs/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build SPHINXPROJ = DTAIMap-Matching SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/classes/map/BaseMap.rst ================================================ BaseMap ======= .. autoclass:: leuvenmapmatching.map.base.BaseMap :members: ================================================ FILE: docs/classes/map/InMemMap.rst ================================================ InMemMap ======== .. autoclass:: leuvenmapmatching.map.inmem.InMemMap :members: ================================================ FILE: docs/classes/map/SqliteMap.rst ================================================ SqliteMap ========= .. autoclass:: leuvenmapmatching.map.sqlite.SqliteMap :members: ================================================ FILE: docs/classes/matcher/BaseMatcher.rst ================================================ BaseMatcher =========== This a generic base class to be used by matchers. This class itself does not implement a working matcher. Use a matcher such as ``SimpleMatcher``, ``DistanceMatcher``, ... .. autoclass:: leuvenmapmatching.matcher.base.BaseMatcher :members: ================================================ FILE: docs/classes/matcher/BaseMatching.rst ================================================ BaseMatching ============ .. autoclass:: leuvenmapmatching.matcher.base.BaseMatching :members: ================================================ FILE: docs/classes/matcher/DistanceMatcher.rst ================================================ DistanceMatcher =============== .. autoclass:: leuvenmapmatching.matcher.distance.DistanceMatcher :members: ================================================ FILE: docs/classes/matcher/SimpleMatcher.rst ================================================ SimpleMatcher ============= .. autoclass:: leuvenmapmatching.matcher.simple.SimpleMatcher :members: ================================================ FILE: docs/classes/overview.rst ================================================ matcher ~~~~~~~ .. toctree:: :caption: Matcher matcher/BaseMatcher matcher/SimpleMatcher matcher/DistanceMatcher .. toctree:: :caption: Matching matcher/BaseMatching map ~~~ .. toctree:: :caption: Map map/BaseMap map/InMemMap map/SqliteMap util ~~~~ .. toctree:: :caption: Util util/Segment ================================================ FILE: docs/classes/util/Segment.rst ================================================ Segment ======= .. autoclass:: leuvenmapmatching.util.segment.Segment :members: ================================================ FILE: docs/conf.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Leuven.MapMatching documentation build configuration file, created by # sphinx-quickstart on Sat Apr 14 23:24:31 2018. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys sys.path.insert(0, os.path.abspath('..')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode'] autoclass_content = 'both' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = 'Leuven.MapMatching' copyright = '2018-2022, Wannes Meert' author = 'Wannes Meert' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '1.1.1' # The full version, including alpha/beta/rc tags. release = '1.1.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # # html_theme = 'alabaster' html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = 'LeuvenMapMatchingDoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'LeuvenMapMatching.tex', 'Leuven.MapMatching Documentation', 'Wannes Meert', 'manual'), ] # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ (master_doc, 'leuvenmapmatching', 'Leuven.MapMatching Documentation', [author], 1) ] # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'LeuvenMapMatching', 'Leuven.MapMatching Documentation', author, 'LeuvenMapMatching', 'Map Matching', 'Miscellaneous'), ] ================================================ FILE: docs/index.rst ================================================ .. Leuven.MapMatching documentation master file, created by sphinx-quickstart on Sat Apr 14 23:24:31 2018. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Leuven.MapMatching's documentation ================================== Align a trace of coordinates (e.g. GPS measurements) to a map of road segments. The matching is based on a Hidden Markov Model (HMM) with non-emitting states. The model can deal with missing data and you can plug in custom transition and emission probability distributions. Reference: Meert Wannes, Mathias Verbeke, "HMM with Non-Emitting States for Map Matching", European Conference on Data Analysis (ECDA), Paderborn, Germany, 2018. .. figure:: https://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/example1.png?v=2 :alt: example .. toctree:: :maxdepth: 2 :caption: Contents: .. toctree:: :caption: Usage usage/installation usage/introduction usage/openstreetmap usage/visualisation usage/latitudelongitude usage/customdistributions usage/incremental usage/debug .. toctree:: :maxdepth: 2 :caption: Classes classes/overview Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: docs/make.bat ================================================ @ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=. set BUILDDIR=_build set SPHINXPROJ=DTAIMap-Matching if "%1" == "" goto help %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% :end popd ================================================ FILE: docs/requirements.txt ================================================ numpy scipy sphinx_rtd_theme ================================================ FILE: docs/usage/customdistributions.rst ================================================ Custom probability distributions ================================ You can use your own custom probability distributions for the transition and emission probabilities. This is achieved by inheriting from the :class:`BaseMatcher` class. Examples are available in the :class:`SimpleMatching` class and :class:`DistanceMatching` class. The latter implements a variation based on Newson and Krumm (2009). Transition probability distribution ----------------------------------- Overwrite the :meth:`logprob_trans` method. For example, if you want to use a uniform distribution over the possible road segments: .. code-block:: python def logprob_trans(self, prev_m, edge_m, edge_o, is_prev_ne, is_next_ne): return -math.log(len(self.matcher.map.nodes_nbrto(self.edge_m.last_point()))) Note that ``prev_m.edge_m`` and ``edge_m`` are not necessarily connected. For example if the ``Map`` object returns a neighbor state that is not connected in the roadmap. This functionality is used to allow switching lanes. Emission probability distribution --------------------------------- Overwrite the :meth:`logprob_obs` method for non-emitting nodes. These methods are given the closest distance as `dist`, the previous :class:`Matching` object in the lattice, the state as `edge_m`, and the observation as `edge_o`. The latter two are :class:`Segment` objects that can represent either a segment or a point. Each segment also has a project point which is the point on the segment that is the closest point. For example, a simple step function with more tolerance for non-emitting nodes: .. code-block:: python def logprob_obs(self, dist, prev_m, new_edge_m, new_edge_o, is_ne): if is_ne: if dist < 50: return -math.log(50) else: if dist < 10: return -math.log(10) return -np.inf Note that an emission probability can be given for a non-emitting node. This allows you to rank non-emitting nodes even when no observations are available. It will then insert pseudo-observations on the line between the previous and next observations. To have a pure non-emitting node, the `logprob_obs` method should always return 0 if the ``is_ne`` argument is true. Custom lattice objects ---------------------- If you need to store additional information in the lattice, inherit from the :class:`Matching` class and pass your custom object to the :class:`Matcher` object. .. code-block:: python from leuvenmapmatching.map.base import BaseMatching class MyMatching(BaseMatching): ... matcher = MyMatcher(mapdb, matching=MyMatching) ================================================ FILE: docs/usage/debug.rst ================================================ Debug ===== Increasing the verbosity level ------------------------------ To inspect the intermediate steps that the algorithm take, you can increase the verbosity level of the package. For example: .. code-block:: python import sys import logging import leuvenmapmatching logger = leuvenmapmatching.logger logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) Inspect the best matching ------------------------- The best match is available in ``matcher.lattice_best``. This is a list of ``Matching`` objects. For example after running the first example in the introduction: .. code-block:: python >>> matcher.lattice_best [Matching, Matching, Matching, ... A matching object summarizes its information as a tuple with three values if the best match is with a vertex: . And a tuple with four values if the best match is with an edge: . In the example above, the first observation (with index 0) is matched to a point on the edge A-B. If you want to inspect the exact locations, you can query the ``Segment`` objects that express the observation and map: ``matching.edge_o`` and ``matching.edge_m``. .. code-block:: python >>> match = matcher.lattice_best[0] >>> match.edge_m.l1, match.edge_m.l2 # Edge start/end labels ('A', 'B') >>> match.edge_m.pi # Best point on A-B edge (1.0, 1.0) >>> match.edge_m.p1, match.edge_m.p2 # Locations of A and B ((1, 1), (1, 3)) >>> match.edge_o.l1, match.edge_o.l2 # Observation ('O0', None) >>> match.edge_o.pi # Location of observation O0, because no second location (0.8, 0.7) >>> match.edge_o.p1 # Same as pi because no interpolation (0.8, 0.7) Inspect the matching lattice ---------------------------- All paths through the lattice are available in ``matcher.lattice``. The lattice is a dictionary with a ``LatticeColumn`` object for each observation (in case the full path of observations is matched). For each observation, you can inspect the ``Matching`` objects with: .. code-block:: python >>> matcher.lattice {0: , 1: , 2: , ... >>> matcher.lattice[0].values_all() {Matching, Matching, Matching, ... To start backtracking you can, for example, see which matching object for the last element has the highest probability (thus the best match): .. code-block:: python >>> m = max(matcher.lattice[len(path)-1].values_all(), key=lambda m: m.logprob) >>> m.logprob -0.6835815469734807 The previous matching objects can be queried with. These are only those matches that are connected to this matchin the lattice (in this case nodes in the street graph with an edge to the current node): .. code-block:: python >>> m.prev # Best previous match with a connection (multiple if equal probability) {Matching} >>> m.prev_other # All previous matches in the lattice with a connection {Matching, Matching, Matching, Matching} ================================================ FILE: docs/usage/incremental.rst ================================================ Incremental matching ==================== Example: Incremental matching ------------------------------- If the observations are collected in a streaming setting. The matching can also be invoked incrementally. The lattice will be built further every time a new subsequence of the path is given. .. code-block:: python from leuvenmapmatching.matcher.distance import DistanceMatcher from leuvenmapmatching.map.inmemmap import InMemMap map_con = InMemMap("mymap", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "D", "E", "K", "L"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.3, 3.5), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] matcher = DistanceMatcher(map_con, max_dist=2, obs_noise=1, min_prob_norm=0.5) states, _ = matcher.match(path[:5]) states, _ = matcher.match(path, expand=True) nodes = matcher.path_pred_onlynodes print("States\n------") print(states) print("Nodes\n------") print(nodes) print("") matcher.print_lattice_stats() ================================================ FILE: docs/usage/installation.rst ================================================ Installation ============ Dependencies ------------ Required: - `numpy `__ - `scipy `__ Optional (only loaded when methods are called that rely on these packages): - `rtree `__ - `nvector `__ - `gpxpy `__ - `pyproj `__ - `pykalman `__ - `matplotlib `__ - `smopy `__ Using pip --------- If you want to install the latest released version using pip: :: $ pip install leuvenmapmatching If you want to install the latest non-released version (add ``@develop``) for the latest development version: :: $ pip install git+https://github.com/wannesm/leuvenmapmatching From source ----------- The library can also be compiled and/or installed directly from source. * Download the source from https://github.com/wannesm/leuvenmapmatching * To compile and install in your site-package directory: ``python3 setup.py install`` ================================================ FILE: docs/usage/introduction.rst ================================================ Examples ======== Example 1: Simple ----------------- A first, simple example. Some parameters are given to tune the algorithm. The ``max_dist`` and ``obs_noise`` are distances that indicate the maximal distance between observation and road segment and the expected noise in the measurements, respectively. The ``min_prob_norm`` prunes the lattice in that it drops paths that drop below 0.5 normalized probability. The probability is normalized to allow for easier reasoning about the probability of a path. It is computed as the exponential smoothed log probability components instead of the sum as would be the case for log likelihood. Because the number of possible paths quickly grows, it is recommended to set the ``max_lattice_width`` argument to speed up the algorithm (available from version 1.0 onwards). It will only continue the search with this number of possible paths at every step. If no solution is found, this value can be incremented using the ``increase_max_lattice_width`` method. .. code-block:: python from leuvenmapmatching.matcher.distance import DistanceMatcher from leuvenmapmatching.map.inmem import InMemMap map_con = InMemMap("mymap", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "F", "E", "K", "L"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.3, 3.5), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] matcher = DistanceMatcher(map_con, max_dist=2, obs_noise=1, min_prob_norm=0.5, max_lattice_width=5) states, _ = matcher.match(path) nodes = matcher.path_pred_onlynodes print("States\n------") print(states) print("Nodes\n------") print(nodes) print("") matcher.print_lattice_stats() Example 2: Non-emitting states ------------------------------ In case there are less observations that states (an assumption of HMMs), non-emittings states allow you to deal with this. States will be inserted that are not associated with any of the given observations if this improves the probability of the path. It is possible to also associate a distribtion over the distance between observations and the non-emitting states (`obs_noise_ne`). This allows the algorithm to prefer nearby road segments. This value should be larger than `obs_noise` as it is mapped to the line between the previous and next observation, which does not necessarily run over the relevant segment. Setting this to infinity is the same as using pure non-emitting states that ignore observations completely. .. code-block:: python from leuvenmapmatching.matcher.distance import DistanceMatcher from leuvenmapmatching.map.inmem import InMemMap from leuvenmapmatching import visualization as mmviz path = [(1, 0), (7.5, 0.65), (10.1, 1.9)] mapdb = InMemMap("mymap", graph={ "A": ((1, 0.00), ["B"]), "B": ((3, 0.00), ["A", "C"]), "C": ((4, 0.70), ["B", "D"]), "D": ((5, 1.00), ["C", "E"]), "E": ((6, 1.00), ["D", "F"]), "F": ((7, 0.70), ["E", "G"]), "G": ((8, 0.00), ["F", "H"]), "H": ((10, 0.0), ["G", "I"]), "I": ((10, 2.0), ["H"]) }, use_latlon=False) matcher = DistanceMatcher(mapdb, max_dist_init=0.2, obs_noise=1, obs_noise_ne=10, non_emitting_states=True, only_edges=True , max_lattice_width=5) states, _ = matcher.match(path) nodes = matcher.path_pred_onlynodes print("States\n------") print(states) print("Nodes\n------") print(nodes) print("") matcher.print_lattice_stats() mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True filename="output.png")) ================================================ FILE: docs/usage/latitudelongitude.rst ================================================ Dealing with Latitude-Longitude =============================== The toolbox can deal with latitude-longitude coordinates directly. Map matching, however, requires a lot of repeated computations between points and latitude-longitude computations will be more expensive than Euclidean distances. There are three different options how you can handle latitude-longitude coordinates: Option 1: Use Latitude-Longitude directly ----------------------------------------- Set the ``use_latlon`` flag in the :class:`Map` to true. For example to read in an OpenStreetMap file directly to a :class:`InMemMap` object: .. code-block:: python from leuvenmapmatching.map.inmem import InMemMap map_con = InMemMap("myosm", use_latlon=True) for entity in osmread.parse_file(osm_fn): if isinstance(entity, osmread.Way) and 'highway' in entity.tags: for node_a, node_b in zip(entity.nodes, entity.nodes[1:]): map_con.add_edge(node_a, node_b) map_con.add_edge(node_b, node_a) if isinstance(entity, osmread.Node): map_con.add_node(entity.id, (entity.lat, entity.lon)) map_con.purge() Option 2: Project Latitude-Longitude to X-Y ------------------------------------------- Latitude-Longitude coordinates can be transformed two a frame with two orthogonal axis. .. code-block:: python from leuvenmapmatching.map.inmem import InMemMap map_con_latlon = InMemMap("myosm", use_latlon=True) # Add edges/nodes map_con_xy = map_con_latlon.to_xy() route_latlon = [] # Add GPS locations route_xy = [map_con_xy.latlon2yx(latlon) for latlon in route_latlon] This can also be done directly using the `pyproj `_ toolbox. For example, using the Lambert Conformal projection to project the route GPS coordinates: .. code-block:: python import pyproj route = [(4.67878,50.864),(4.68054,50.86381),(4.68098,50.86332),(4.68129,50.86303),(4.6817,50.86284), (4.68277,50.86371),(4.68894,50.86895),(4.69344,50.86987),(4.69354,50.86992),(4.69427,50.87157), (4.69643,50.87315),(4.69768,50.87552),(4.6997,50.87828)] lon_0, lat_0 = route[0] proj = pyproj.Proj(f"+proj=merc +ellps=GRS80 +units=m +lon_0={lon_0} +lat_0={lat_0} +lat_ts={lat_0} +no_defs") xs, ys = [], [] for lon, lat in route: x, y = proj(lon, lat) xs.append(x) ys.append(y) Notice that the pyproj package uses the convention to express coordinates as x-y which is longitude-latitude because it is defined this way in the CRS definitions while the Leuven.MapMatching toolbox follows the ISO 6709 standard and expresses coordinates as latitude-longitude. If you want ``pyproj`` to use latitude-longitude you can use set the `axisswap option `_. If you want to define both the from and to projections: .. code-block:: python import pyproj route = [(4.67878,50.864),(4.68054,50.86381),(4.68098,50.86332),(4.68129,50.86303),(4.6817,50.86284), (4.68277,50.86371),(4.68894,50.86895),(4.69344,50.86987),(4.69354,50.86992),(4.69427,50.87157), (4.69643,50.87315),(4.69768,50.87552),(4.6997,50.87828)] p1 = pyproj.Proj(proj='latlon', datum='WGS84') p2 = pyproj.Proj(proj='utm', datum='WGS84') xs, ys = [], [] for lon, lat in route: x, y = pyproj.transform(lon, lat) xs.append(x) ys.append(y) Option 3: Use Latitude-Longitude as if they are X-Y points ---------------------------------------------------------- A naive solution would be to use latitude-longitude coordinate pairs as if they are X-Y coordinates. For small distances, far away from the poles and not crossing the dateline, this option might work. But it is not adviced. For example, for long distances the error is quite large. In the image beneath, the blue line is the computation of the intersection using latitude-longitude while the red line is the intersection using Eucludean distances. .. figure:: https://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/latlon_mismatch_1.png?v=1 :alt: Latitude-Longitude mismatch .. figure:: https://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/latlon_mismatch_2.png?v=1 :alt: Latitude-Longitude mismatch detail ================================================ FILE: docs/usage/openstreetmap.rst ================================================ Map from OpenStreetMap ====================== You can download a graph for map-matching from the OpenStreetMap.org service. Multiple methods exists, we illustrate two. Using requests, osmread and gpx ------------------------------- You can perform map matching on a OpenStreetMap database by combing ``leuvenmapmatching`` with the packages ``requests``, ``osmread`` and ``gpx``. Download a map as XML ~~~~~~~~~~~~~~~~~~~~~ You can use the overpass-api.de service: .. code-block:: python from pathlib import Path import requests xml_file = Path(".") / "osm.xml" url = 'http://overpass-api.de/api/map?bbox=4.694933,50.870047,4.709256000000001,50.879628' r = requests.get(url, stream=True) with xml_file.open('wb') as ofile: for chunk in r.iter_content(chunk_size=1024): if chunk: ofile.write(chunk) Create graph using osmread ~~~~~~~~~~~~~~~~~~~~~~~~~~ Once we have a file containing the region we are interested in, we can select the roads we want to use to create a graph from. In this case we focus on 'ways' with a 'highway' tag. Those represent a variety of roads. For a more detailed filtering look at the `possible values of the highway tag `_. .. code-block:: python from leuvenmapmatching.map.inmem import InMemMap import osmread map_con = InMemMap("myosm", use_latlon=True, use_rtree=True, index_edges=True) for entity in osmread.parse_file(str(xml_file)): if isinstance(entity, osmread.Way) and 'highway' in entity.tags: for node_a, node_b in zip(entity.nodes, entity.nodes[1:]): map_con.add_edge(node_a, node_b) # Some roads are one-way. We'll add both directions. map_con.add_edge(node_b, node_a) if isinstance(entity, osmread.Node): map_con.add_node(entity.id, (entity.lat, entity.lon)) map_con.purge() Note that ``InMemMap`` is a simple container for a map. It is recommended to use your own optimized connecter to your map dataset. If you want to allow transitions that are not following the exact road segments you can inherit from the ``Map`` class and define a new class with your own transitions. The transitions are defined using the ``nodes_nbrto`` and ``edges_nbrt`` methods. Perform map matching on an OpenStreetMap database ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can create a list of latitude-longitude coordinates manually. Or read a gpx file. .. code-block:: python from leuvenmapmatching.util.gpx import gpx_to_path track = gpx_to_path("mytrack.gpx") matcher = DistanceMatcher(map_con, max_dist=100, max_dist_init=25, # meter min_prob_norm=0.001, non_emitting_length_factor=0.75, obs_noise=50, obs_noise_ne=75, # meter dist_noise=50, # meter non_emitting_states=True, max_lattice_width=5) states, lastidx = matcher.match(track) Using osmnx and geopandas ------------------------- Another great library to interact with OpenStreetMap data is the `osmnx `_ package. The osmnx package can retrieve relevant data automatically, for example when given a name of a region. This package is build on top of the `geopandas `_ package. .. code-block:: python import osmnx graph = ox.graph_from_place('Leuven, Belgium', network_type='drive', simplify=False) graph_proj = ox.project_graph(graph) # Create GeoDataFrames (gdfs) # Approach 1 nodes_proj, edges_proj = ox.graph_to_gdfs(graph_proj, nodes=True, edges=True) for nid, row in nodes_proj[['x', 'y']].iterrows(): map_con.add_node(nid, (row['x'], row['y'])) for eid, _ in edges_proj.iterrows(): map_con.add_edge(eid[0], eid[1]) # Approach 2 nodes, edges = ox.graph_to_gdfs(graph_proj, nodes=True, edges=True) nodes_proj = nodes.to_crs("EPSG:3395") edges_proj = edges.to_crs("EPSG:3395") for nid, row in nodes_proj.iterrows(): map_con.add_node(nid, (row['lat'], row['lon'])) # We can also extract edges also directly from networkx graph for nid1, nid2, _ in graph.edges: map_con.add_edge(nid1, nid2) ================================================ FILE: docs/usage/visualisation.rst ================================================ Visualisation ============= To inspect the results, a plotting function is included. Simple plotting --------------- To plot the graph in a matplotlib figure use: .. code-block:: python from leuvenmapmatching import visualization as mmviz mmviz.plot_map(map_con, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename="my_plot.png") This will result in the following figure: .. figure:: https://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/plot1.png?v=1 :alt: Plot1 You can also define your own figure by passing a matplotlib axis object: .. code-block:: python fig, ax = plt.subplots(1, 1) mmviz.plot_map(map_con, matcher=matcher, ax=ax, show_labels=True, show_matching=True, show_graph=True, filename="my_plot.png") Plotting with an OpenStreetMap background ----------------------------------------- The plotting function also supports a link with the ``smopy`` package. Set the ``use_osm`` argument to true and pass a map that is defined with latitude-longitude (thus ``use_latlon=True``). You can set ``zoom_path`` to true to only see the relevant part and not the entire map that is available in the map. Alternatively you can also set the bounding box manually using the ``bb`` argument. .. code-block:: python mm_viz.plot_map(map_con, matcher=matcher, use_osm=True, zoom_path=True, show_labels=False, show_matching=True, show_graph=False, filename="my_osm_plot.png") This will result in the following figure: .. figure:: https://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/plot2.png?v=1 :alt: Plot2 Or when some GPS points are missing in the track, the matching is more visible as the matched route deviates from the straight line between two GPS points: .. figure:: https://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/plot3.png?v=1 :alt: Plot3 ================================================ FILE: leuvenmapmatching/__init__.py ================================================ # encoding: utf-8 """ leuvenmapmatching ~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2022 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging from . import map, matcher, util # visualization is not loaded by default (avoid loading unnecessary dependencies such as matplotlib). logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") __version__ = '1.1.4' ================================================ FILE: leuvenmapmatching/map/__init__.py ================================================ # encoding: utf-8 """ leuvenmapmatching.map ~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ ================================================ FILE: leuvenmapmatching/map/base.py ================================================ # encoding: utf-8 """ leuvenmapmatching.map.base ~~~~~~~~~~~~~~~~~~~~~~~~~~ Base Map class. To be used in a Matcher object, the following functions need to be defined: - ``edges_closeto`` - ``nodes_closeto`` - ``nodes_nbrto`` - ``edges_nbrto`` For visualiation purposes the following methods need to be implemented: - ``bb`` - ``labels`` - ``size`` - ``coordinates`` - ``node_coordinates`` - ``all_edges`` - ``all_nodes`` :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ from abc import abstractmethod import logging MYPY = False if MYPY: from typing import Tuple, Union, List LabelType = Union[int, str] LocType = Tuple[float, float] logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") class BaseMap(object): """Abstract class for a Map.""" def __init__(self, name, use_latlon=True): """Simple database wrapper/stub.""" self.name = name self._use_latlon = None self.distance = None self.distance_point_to_segment = None self.distance_segment_to_segment = None self.box_around_point = None self.use_latlon = use_latlon @property def use_latlon(self): return self._use_latlon @use_latlon.setter def use_latlon(self, value): self._use_latlon = value if self._use_latlon: from ..util import dist_latlon as dist_lib else: from ..util import dist_euclidean as dist_lib self.distance = dist_lib.distance self.distance_point_to_segment = dist_lib.distance_point_to_segment self.distance_segment_to_segment = dist_lib.distance_segment_to_segment self.box_around_point = dist_lib.box_around_point self.lines_parallel = dist_lib.lines_parallel @abstractmethod def bb(self): """Bounding box. :return: (lat_min, lon_min, lat_max, lon_max) """ @abstractmethod def labels(self): """Labels of all nodes.""" @abstractmethod def size(self): """Number of nodes.""" @abstractmethod def node_coordinates(self, node_key): """Coordinates for given node key.""" @abstractmethod def edges_closeto(self, loc, max_dist=None, max_elmt=None): """Find edges close to a certain location. :param loc: Latitude, Longitude :param max_dist: Maximal distance that returned nodes can be from lat-lon :param max_elmt: Maximal number of elements returned after sorting according to distance. :return: list[tuple[dist, label, loc]] """ return None @abstractmethod def nodes_closeto(self, loc, max_dist=None, max_elmt=None): """Find nodes close to a certain location. :param loc: Latitude, Longitude :param max_dist: Maximal distance that returned nodes can be from lat-lon :param max_elmt: Maximal number of elements returned after sorting according to distance. :return: list[tuple[dist, label, loc]] """ return None @abstractmethod def nodes_nbrto(self, node): # type: (BaseMap, LabelType) -> List[Tuple[LabelType, LocType]] """Return all nodes that are linked to ``node``. :param node: Node identifier :return: list[tuple[label, loc]] """ return [] def edges_nbrto(self, edge): # type: (BaseMap, Tuple[LabelType, LabelType]) -> List[Tuple[LabelType, LocType, LabelType, LocType]] """Return all edges that are linked to ``edge``. Defaults to ``nodes_nbrto``. :param edge: Edge identifier :return: list[tuple[label1, label2, loc1, loc2]] """ results = [] l1, l2 = edge p2 = self.node_coordinates(l2) for l3, p3 in self.nodes_nbrto(l2): results.append((l2, p2, l3, p3)) return results @abstractmethod def all_nodes(self, bb=None): """All node keys and coordinates. :return: [(key, (lat, lon))] """ @abstractmethod def all_edges(self, bb=None): """All edges. :return: [(key_a, loc_a, key_b, loc_b)] """ ================================================ FILE: leuvenmapmatching/map/inmem.py ================================================ # encoding: utf-8 """ leuvenmapmatching.map.inmem ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Simple in-memory map representation. Not suited for production purposes. Write your own map class that connects to your map (e.g. a database instance). :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging import time from pathlib import Path import pickle from functools import partial try: import rtree except ImportError: rtree = None try: import pyproj except ImportError: pyproj = None try: import tqdm except ImportError: tqdm = None from .base import BaseMap MYPY = False if MYPY: from typing import Optional, Set, Tuple, Dict, Union LabelType = Union[int, str] LocType = Tuple[float, float] EdgeType = Tuple[LabelType, LabelType] logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") class InMemMap(BaseMap): def __init__(self, name, use_latlon=True, use_rtree=False, index_edges=False, crs_lonlat=None, crs_xy=None, graph=None, linked_edges=None, dir=None, deserializing=False): """In-memory representation of a map. This is a simple database-like object to perform experiments with map matching. For production purposes it is recommended to use your own derived class (e.g. to connect to your database instance). This class supports: - Indexing using rtrees to allow for fast searching of points on the map. When using the rtree index, only integer numbers are allowed as node labels. - Serializing to write and read from files. - Projecting points to a different frame (e.g. GPS to Lambert) :param name: Map name (mandatory) :param use_latlon: The locations represent latitude-longitude pairs, otherwise y-x coordinates are assumed. :param use_rtree: Build an rtree index to quickly search for locations. :param index_edges: Build an index for the edges in the map instead of the vertices. :param crs_lonlat: Coordinate reference system for the latitude-longitude coordinates. :param crs_xy: Coordiante reference system for the y-x coordinates. :param graph: Initial graph of form Dict[label, Tuple[Tuple[y,x], List[neighbor]]]] :param dir: Directory where to serialize to. If given, the rtree index structure will be written to a file immediately. :param deserializing: Internal variable to indicate that the object is being build from a file. """ super(InMemMap, self).__init__(name, use_latlon=use_latlon) self.dir = None if dir is None else Path(dir) self.index_edges = index_edges self.graph = dict() if graph is None else graph self.rtree = None self.use_rtree = use_rtree if self.use_rtree: self.setup_index(deserializing=deserializing) self.crs_lonlat = 'EPSG:4326' if crs_lonlat is None else crs_lonlat # GPS self.crs_xy = 'EPSG:3395' if crs_xy is None else crs_xy # Mercator projection if pyproj: # proj_lonlat = pyproj.Proj(self.crs_lonlat, preserve_units=True) # proj_xy = pyproj.Proj(self.crs_xy, preserve_units=True) # self.lonlat2xy = partial(pyproj.transform, proj_lonlat, proj_xy) # self.xy2lonlat = partial(pyproj.transform, proj_xy, proj_lonlat) tr_lonlat2xy = pyproj.Transformer.from_crs(self.crs_lonlat, self.crs_xy) self.lonlat2xy = tr_lonlat2xy.transform tr_xy2lonlat = pyproj.Transformer.from_crs(self.crs_xy, self.crs_lonlat) self.xy2lonlat = tr_xy2lonlat.transform else: def pyproj_notfound(*_args, **_kwargs): raise Exception("pyproj package not found") self.lonlat2xy = pyproj_notfound self.xy2lonlat = pyproj_notfound self.linked_edges = linked_edges # type: Optional[Dict[EdgeType, Set[Tuple[EdgeType]]]] self.vertex_label_map = None def vertex_label_to_int(self, label, create=False): if type(label) is int: return label if self.vertex_label_map is None: if not create: return label self.vertex_label_map = dict() if label in self.vertex_label_map: new_label = self.vertex_label_map[label] else: new_label = len(self.vertex_label_map) self.vertex_label_map[label] = new_label return new_label def vertices_labels_to_int(self): graph = dict() for label, (loc, nbrs) in self.graph.items(): new_label = self.vertex_label_to_int(label, create=True) new_nbrs = [self.vertex_label_to_int(nbr, create=True) for nbr in nbrs] graph[new_label] = (loc, new_nbrs) self.graph = graph def serialize(self): """Create a serializable data structure.""" data = { "name": self.name, "graph": self.graph, "use_latlon": self.use_latlon, "use_rtree": self.use_rtree, "index_edges": self.index_edges, "crs_lonlat": self.crs_lonlat, "crs_xy": self.crs_xy, "linked_edges": self.linked_edges } if self.dir is not None: data["dir"] = self.dir return data @classmethod def deserialize(cls, data): """Create a new instance from a dictionary.""" nmap = cls(data["name"], dir=data.get("dir", None), use_latlon=data["use_latlon"], use_rtree=data["use_rtree"], index_edges=data["index_edges"], crs_lonlat=data.get("crs_lonlat", None), crs_xy=data.get("crs_xy", None), graph=data.get("graph", None), linked_edges=data.get("linked_edges", None), deserializing=True) return nmap def dump(self): """Serialize map using pickle. All files will be saved to the `dir` directory using the `name` as filename. """ if self.dir is None: logger.error(f"No directory set where to save (see InMemMap.__init__)") return filename = self.dir / (self.name + ".pkl") with filename.open("wb") as ofile: pickle.dump(self.serialize(), ofile) logger.debug(f"Saved map to {filename}") if self.rtree: rtree_fn = self.rtree_fn() if rtree_fn is not None: self.rtree.close() self.rtree = rtree.index.Index(str(rtree_fn)) @classmethod def from_pickle(cls, filename): """Deserialize map using pickle to the given filename.""" filename = Path(filename) with filename.open("rb") as ifile: data = pickle.load(ifile) nmap = cls.deserialize(data) return nmap def bb(self): """Bounding box. :return: (lat_min, lon_min, lat_max, lon_max) or (y_min, x_min, y_max, x_max) """ if self.use_rtree: lat_min, lon_min, lat_max, lon_max = self.rtree.bounds else: glat, glon = zip(*[t[0] for t in self.graph.values()]) lat_min, lat_max = min(glat), max(glat) lon_min, lon_max = min(glon), max(glon) return lat_min, lon_min, lat_max, lon_max def labels(self): """All labels.""" return self.graph.keys() def size(self): return len(self.graph) def node_coordinates(self, node_key): """Get the coordinates of the given node. :param node_key: Node label/key :return: (lat, lon) """ return self.graph[node_key][0] def add_node(self, node, loc): """Add new node to the map. :param node: label :param loc: (lat, lon) or (y, x) """ if node in self.graph: if self.graph[node][0] is None: self.graph[node] = (loc, self.graph[node][1]) else: self.graph[node] = (loc, []) if self.use_rtree and self.rtree is not None and not self.index_edges: if type(node) is not int: raise Exception(f"Rtree index only supports integer keys for vertices") self.rtree.insert(node, (loc[0], loc[1], loc[0], loc[1])) def del_node(self, node): if node not in self.graph: return if self.rtree: data = self.graph[node] loc = data[0] self.rtree.delete(node, (loc[0], loc[1], loc[0], loc[1])) del self.graph[node] def add_edge(self, node_a, node_b): """Add new edge to the map. :param node_a: Label for the node that is the start of the edge :param node_b: Label for the node that is the end of the edge """ if node_a not in self.graph: raise ValueError(f"Add {node_a} first as node") if node_b not in self.graph: raise ValueError(f"Add {node_b} first as node") if node_b not in self.graph[node_a][1]: self.graph[node_a][1].append(node_b) if self.use_rtree and self.rtree is not None and self.index_edges: if type(node_a) is not int or type(node_b) is not int: raise Exception(f"Rtree index only supports integer keys for vertices") loc_a = self.graph[node_a][0] loc_b = self.graph[node_b][0] bb = (min(loc_a[0], loc_b[0]), min(loc_a[1], loc_b[1]), # y_min, x_min max(loc_a[0], loc_b[0]), max(loc_a[1], loc_b[1])) # y_max, x_max self.rtree.insert(node_a, bb) # self.rtree.insert(node_b, bb) def _items_in_bb(self, bb): if self.rtree is not None: node_idxs = self.rtree.intersection(bb) for key in node_idxs: yield (key, self.graph[key]) else: lat_min, lon_min, lat_max, lon_max = bb for key, value in self.graph.items(): ((lat, lon), nbrs) = value if lat_min <= lat <= lat_max and lon_min <= lon <= lon_max: yield (key, value) def all_edges(self, bb=None): """Return all edges. :param bb: Bounding box :return: (key_a, loc_a, nbr, loc_b) """ if bb is None: keyvals = self.graph.items() else: keyvals = self._items_in_bb(bb) for key_a, (loc_a, nbrs) in keyvals: if loc_a is not None: for nbr in nbrs: try: loc_b, _ = self.graph[nbr] if loc_b is not None: yield (key_a, loc_a, nbr, loc_b) except KeyError: # print("Node not found: {}".format(nbr)) pass def all_nodes(self, bb=None): """Return all nodes. :param bb: Bounding box :return: """ if bb is None: keyvals = self.graph.items() else: keyvals = self._items_in_bb(bb) for key_a, (loc_a, nbrs) in keyvals: if loc_a is not None: yield key_a, loc_a def purge(self): cnt_noloc = 0 cnt_noedges = 0 remove = [] for node in self.graph.keys(): if self.graph[node][0] is None: cnt_noloc += 1 remove.append(node) # print("No location for node {}".format(node)) elif len(self.graph[node][1]) == 0: cnt_noedges += 1 remove.append(node) for node in remove: self.del_node(node) logger.debug("Removed {} nodes without location".format(cnt_noloc)) logger.debug("Removed {} nodes without edges".format(cnt_noedges)) def rtree_size(self): bb = self.rtree.bounds if bb[0] < bb[2] and bb[1] < bb[3]: rtree_size = self.rtree.count(bb) else: rtree_size = 0 return rtree_size def rtree_fn(self): rtree_fn = None if self.dir is not None: rtree_fn = self.dir / self.name return rtree_fn def setup_index(self, force=False, deserializing=False): if not self.use_rtree: return if self.rtree is not None and not force: return if rtree is None: raise Exception("rtree package not found") rtree_fn = self.rtree_fn() args = [] if deserializing and (rtree_fn is None or not rtree_fn.exists()): deserializing = False if self.graph and len(self.graph) > 0 and not deserializing: if self.index_edges: logger.debug("Generator to index edges") def generator_function(): for label, data in self.graph.items(): lat_min, lon_min = data[0] lat_max, lon_max = lat_min, lon_min for idx in data[1]: olat, olon = self.graph[idx][0] lat_min = min(lat_min, olat) lat_max = max(lat_max, olat) lon_min = min(lon_min, olon) lon_max = max(lon_max, olon) if type(label) is not int: raise Exception(f"Rtree index only supports integer keys for vertices") yield (label, (lat_min, lon_min, lat_max, lon_max), None) else: logger.debug("Generator to index nodes") def generator_function(): for label, data in self.graph.items(): lat, lon = data[0] if type(label) is not int: raise Exception(f"Rtree index only supports integer keys for vertices") yield (label, (lat, lon, lat, lon), None) args.append(generator_function()) t_start = time.time() if self.dir is not None: # props = rtree.index.Property() # if force: # props.overwrite = True logger.debug(f"Creating new file-based rtree index ({rtree_fn}) ...") args.insert(0, str(rtree_fn)) elif deserializing: raise Exception("Cannot deserialize, no directory given") else: logger.debug(f"Creating new in-memory rtree index (args={args}) ...") self.rtree = rtree.index.Index(*args) t_delta = time.time() - t_start logger.debug(f"... done: rtree size = {self.rtree_size()}, time = {t_delta} sec") def fill_index(self): if not self.use_rtree or self.rtree is None: return for label, data in self.graph.items(): loc = data[0] self.rtree.insert(label, (loc[1], loc[0], loc[1], loc[0])) logger.debug(f"After filling rtree, size = {self.rtree_size()}") def to_xy(self, name=None, use_rtree=None): """Create a map that uses a projected XY representation on which Euclidean distances can be used. """ if not self.use_latlon: return self if name is None: name = self.name + "_xy" if use_rtree is None: use_rtree = self.use_rtree ngraph = dict() for label, row in self.graph.items(): lat, lon = row[0] x, y = self.lonlat2xy(lon, lat) ngraph[label] = ((y, x), row[1]) nmap = self.__class__(name, dir=self.dir, graph=ngraph, use_latlon=False, use_rtree=use_rtree, index_edges=self.index_edges, crs_xy=self.crs_xy, crs_lonlat=self.crs_lonlat) return nmap def latlon2xy(self, lat, lon): x, y = self.lonlat2xy(lon, lat) return x, y def latlon2yx(self, lat, lon): x, y = self.lonlat2xy(lon, lat) return y, x def xy2latlon(self, x, y): lon, lat = self.xy2lonlat(x, y) return lat, lon def yx2latlon(self, y, x): lon, lat = self.xy2lonlat(x, y) return lat, lon def nodes_closeto(self, loc, max_dist=None, max_elmt=None): """Return all nodes close to the given location. :param loc: Location :param max_dist: Maximal distance from the location :param max_elmt: Return only the most nearby nodes """ t_start = time.time() lat, lon = loc[:2] lat_b, lon_l, lat_t, lon_r = self.box_around_point((lat, lon), max_dist) bb = (lat_b, lon_l, # y_min, x_min lat_t, lon_r) # y_max, x_max if self.rtree is not None and max_dist is not None: logger.debug(f"Search closeby nodes to {loc}, bb={bb}") nodes = self.rtree.intersection(bb) else: logger.warning("Searching closeby nodes with linear search, use an index and set max_dist") if max_dist is not None: nodes = (key for key, _ in self._items_in_bb(self.box_around_point((lat, lon), max_dist))) else: nodes = self.graph.keys() t_delta_search = time.time() - t_start t_start = time.time() results = [] for label in nodes: oloc, nbrs = self.graph[label] dist = self.distance(loc, oloc) if dist < max_dist: results.append((dist, label, oloc)) results.sort() t_delta_dist = time.time() - t_start logger.debug(f"Found {len(results)} closeby nodes " f"in {t_delta_search} sec and computed distances in {t_delta_dist} sec") if max_elmt is not None: results = results[:max_elmt] return results def edges_closeto(self, loc, max_dist=None, max_elmt=None): """Return all nodes that are on an edge that is close to the given location. :param loc: Location :param max_dist: Maximal distance from the location :param max_elmt: Return only the most nearby nodes """ t_start = time.time() lat, lon = loc[:2] if self.rtree is not None and max_dist is not None and self.index_edges: lat_b, lon_l, lat_t, lon_r = self.box_around_point((lat, lon), max_dist) bb = (lat_b, lon_l, # y_min, x_min lat_t, lon_r) # y_max, x_max logger.debug(f"Search closeby edges to {loc}, bb={bb}") nodes = self.rtree.intersection(bb) else: if self.rtree is not None and max_dist is not None and not self.index_edges: logger.warning("Index is built for nodes, not for edges, set the index_edges argument to true") logger.warning("Searching closeby nodes with linear search, use an index and set max_dist") if max_dist is not None: bb = self.box_around_point((lat, lon), max_dist) nodes = (key for key, _ in self._items_in_bb(bb)) else: nodes = self.graph.keys() t_delta_search = time.time() - t_start t_start = time.time() results = [] for label in nodes: oloc, nbrs = self.graph[label] for nbr in nbrs: if label == nbr: continue nbr_data = self.graph[nbr] dist, pi, ti = self.distance_point_to_segment(loc, oloc, nbr_data[0]) # print(f"label={label}/{oloc}, nbr={nbr}/{nbr_data[0]} -- loc={loc} -> {dist}, {pi}, {ti}") if dist < max_dist: results.append((dist, label, oloc, nbr, nbr_data[0], pi, ti)) results.sort() t_delta_dist = time.time() - t_start logger.debug(f"Found {len(results)} closeby edges " f"in {t_delta_search} sec and computed distances in {t_delta_dist} sec") if max_elmt is not None: results = results[:max_elmt] return results def nodes_nbrto(self, node): results = [] if node not in self.graph: return results loc_node, nbrs = self.graph[node] for nbr_label in nbrs + [node]: try: loc_nbr = self.graph[nbr_label][0] if loc_nbr is not None: results.append((nbr_label, loc_nbr)) except KeyError: pass return results def edges_nbrto(self, edge): results = [] l1, l2 = edge p1 = self.node_coordinates(l1) p2 = self.node_coordinates(l2) # Edges that connect at end of this edge for l3, p3 in self.nodes_nbrto(l2): results.append((l2, p2, l3, p3)) # Edges that are in parallel and close if self.linked_edges: for (l3, l4) in self.linked_edges.get(edge, []): p3 = self.node_coordinates(l3) p4 = self.node_coordinates(l4) results.append((l3, p3, l4, p4)) return results def find_duplicates(self, func=None): """Find entries with identical locations.""" cnt = 0 for label, data in self.graph.items(): lat, lon = data[0] idxs = list(self.rtree.nearest((lat, lon, lat, lon), num_results=1)) idxs.remove(label) if len(idxs) > 0: # logger.info(f"Found doubles for {label}: {idxs}") if func: func(label, idxs) logger.info(f"Found {cnt} doubles") def connect_parallelroads(self, dist=0.5, bb=None): if self.rtree is None or not self.index_edges: logger.error("Finding parallel roads requires and edge-based index") return self.linked_edges = {} it = self.all_edges(bb=bb) if tqdm: it = tqdm.tqdm(list(it)) for key_a, loc_a, key_b, loc_b in it: bb2 = [min(loc_a[0], loc_b[0]), min(loc_a[1], loc_b[1]), max(loc_a[0], loc_b[0]), max(loc_a[1], loc_b[1])] for key_c, loc_c, key_d, loc_d in self.all_edges(bb=bb2): if key_a == key_c or key_a == key_d or key_b == key_c or key_b == key_d: continue # print(f"Test: ({key_a},{key_b}) - ({key_c},{key_d})") if self.lines_parallel(loc_a, loc_b, loc_c, loc_d, d=dist): # print(f"Parallel: ({key_a},{key_b}) - ({key_c},{key_d})") key = (key_a, key_b) if key in self.linked_edges: self.linked_edges[key].add((key_c, key_d)) else: self.linked_edges[key] = {(key_c, key_d)} logger.debug(f"Linked {len(self.linked_edges)} edges") def print_stats(self): print("Graph\n-----") print("Nodes: {}".format(len(self.graph))) def __str__(self): # s = "" # for label, (loc, nbrs, _) in self.graph.items(): # s += f"{label:<10} - ({loc[0]:10.4f}, {loc[1]:10.4f})\n" # return s return f"InMemMap({self.name}, size={self.size()})" ================================================ FILE: leuvenmapmatching/map/sqlite.py ================================================ # encoding: utf-8 """ leuvenmapmatching.map.sqlite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Map representation based on a sqlite database. Not optimized for production purposes. :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import sqlite3 import tempfile import logging import time from pathlib import Path import pickle from functools import partial try: import pyproj except ImportError: pyproj = None try: import tqdm except ImportError: tqdm = None from .base import BaseMap MYPY = False if MYPY: from typing import Optional, Set, Tuple, Dict, Union LabelType = Union[int, str] LocType = Tuple[float, float] EdgeType = Tuple[LabelType, LabelType] logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") class SqliteMap(BaseMap): def __init__(self, name, use_latlon=True, crs_lonlat=None, crs_xy=None, dir=None, deserializing=False): """Store a map as a SQLite instance. This class supports: - Indexing using rtrees to allow for fast searching of points on the map. When using the rtree index, only integer numbers are allowed as node labels. - Serializing to write and read from files. - Projecting points to a different frame (e.g. GPS to Lambert) :param name: Name of database file :param use_latlon: The locations represent latitude-longitude pairs, otherwise y-x coordinates are assumed. :param crs_lonlat: Coordinate reference system for the latitude-longitude coordinates. :param crs_xy: Coordiante reference system for the y-x coordinates. :param dir: Directory where to serialize to. If not given, a temporary location will be used. :param deserializing: Internal variable to indicate that the object is being build from a file. """ super(SqliteMap, self).__init__(name, use_latlon=use_latlon) self.dir = Path(tempfile.gettempdir()) if dir is None else Path(dir) name = Path(name) suffix = name.suffix if suffix == '': name = name.with_suffix('.sqlite') self.db_fn = self.dir / name if deserializing and not self.db_fn.exists(): raise Exception(f"File not found: {self.db_fn}") logger.debug(f"Opening database: {self.db_fn}") try: self.db = sqlite3.connect(str(self.db_fn)) except Exception as exc: raise Exception(f'Problem with database: {self.db_fn}') from exc self.crs_lonlat = crs_lonlat self.crs_xy = crs_xy self.use_latlon = use_latlon if deserializing: self.read_properties() else: self.create_db() if self.crs_lonlat is None: self.crs_lonlat = 'EPSG:4326' # GPS if self.crs_xy is None: self.crs_xy = 'EPSG:3395' # Mercator projection self.save_properties() if pyproj: proj_lonlat = pyproj.Proj(self.crs_lonlat, preserve_units=True) proj_xy = pyproj.Proj(self.crs_xy, preserve_units=True) self.lonlat2xy = partial(pyproj.transform, proj_lonlat, proj_xy) self.xy2lonlat = partial(pyproj.transform, proj_xy, proj_lonlat) else: def pyproj_notfound(*_args, **_kwargs): raise Exception("pyproj package not found") self.lonlat2xy = pyproj_notfound self.xy2lonlat = pyproj_notfound def read_properties(self): c = self.db.cursor() for row in c.execute("SELECT key, value FROM properties;"): key, value = row[0], pickle.loads(row[1]) self.__dict__[key] = value def save_properties(self): c = self.db.cursor() q = "INSERT INTO properties (key, value) VALUES (?, ?)" v = [('name', pickle.dumps(self.name)), ('use_latlon', pickle.dumps(self.use_latlon)), ('crs_lonlat', pickle.dumps(self.crs_lonlat)), ('crs_xy', pickle.dumps(self.crs_xy))] c.executemany(q, v) self.db.commit() def create_db(self): logger.debug("Cleaning database file and creating new tables") c = self.db.cursor() c.execute("DROP INDEX IF EXISTS edges_from_index") c.execute("DROP INDEX IF EXISTS close_edges_index") c.execute("DROP TABLE IF EXISTS nodes_index") c.execute("DROP TABLE IF EXISTS nodes") c.execute("DROP TABLE IF EXISTS edges_index") c.execute("DROP TABLE IF EXISTS edges") c.execute("DROP TABLE IF EXISTS close_edges") c.execute("DROP TABLE IF EXISTS properties") self.db.commit() # Create tables q = ("CREATE VIRTUAL TABLE nodes_index USING rtree(\n" "id, -- Integer primary key\n" "minX, maxX, -- Minimum and maximum X coordinate\n" "minY, maxY -- Minimum and maximum Y coordinate\n" ")") c.execute(q) q = ("CREATE TABLE nodes(\n" "id INTEGER PRIMARY KEY,\n" "x REAL,\n" "y REAL\n" ")") c.execute(q) q = ("CREATE VIRTUAL TABLE edges_index USING rtree(\n" "id, -- Integer primary key\n" "minX, maxX, -- Minimum and maximum X coordinate\n" "minY, maxY -- Minimum and maximum Y coordinate\n" ")") c.execute(q) q = ("CREATE TABLE edges(\n" "id INTEGER PRIMARY KEY,\n" "path INTEGER,\n" # Not necessarily unique, a pathway id can consist of multiple edges "pathnum INTEGER,\n" "id1 INTEGER,\n" # node 1 "id2 INTEGER,\n" # node 2 "speed REAL,\n" # speed m/s "type INTEGER\n" # extra field ")") c.execute(q) q = ("CREATE TABLE close_edges(\n" "id1 INTEGER,\n" # edge 1 "id2 INTEGER\n" # edge 2 ")") c.execute(q) q = ("CREATE TABLE properties(\n" "key TEXT,\n" "value BLOB\n" ")") c.execute(q) q = "CREATE INDEX edges_from_index ON edges(id1)" c.execute(q) q = "CREATE INDEX close_edges_index ON close_edges(id1)" c.execute(q) self.db.commit() @classmethod def from_file(cls, filename): """Read from an existing file.""" filename = Path(filename).with_suffix('') nmap = cls(filename.name, dir=filename.parent, deserializing=True) return nmap def bb(self): """Bounding box. :return: (lat_min, lon_min, lat_max, lon_max) or (y_min, x_min, y_max, x_max) """ c = self.db.cursor() c.execute('SELECT min(minX), max(minX), min(maxX), max(maxX) FROM nodes_index;') lon_min, lon_max, lat_min, lat_max = c.fetchone() return lat_min, lon_min, lat_max, lon_max def labels(self): """All labels.""" c = self.db.cursor() c.execute('SELECT id FROM nodes;') result = [row[0] for row in c.fetchall()] return result def size(self): c = self.db.cursor() c.execute('SELECT count(*) FROM nodes') result = c.fetchone()[0] return result def node_coordinates(self, node_key): """Get the coordinates of the given node. :param node_key: Node label/key :return: (lat, lon) """ c = self.db.cursor() c.execute('SELECT y, x FROM nodes WHERE id = ?', (node_key, )) result = c.fetchone() if result is None: raise Exception(f"No coordinates found for node {node_key}") return result def add_node(self, node, loc, ignore_doubles=False, no_index=False, no_commit=False): """Add new node to the map. :param node: label :param loc: (lat, lon) or (y, x) :param ignore_doubles: When trying to add the same node, ignore it :param no_commit: Do not commit to database (remember to commit later) """ c = self.db.cursor() lat, lon = loc # Nodes q = "INSERT INTO nodes VALUES(?, ?, ?)" try: c.execute(q, (node, lon, lat)) except sqlite3.IntegrityError as exc: if ignore_doubles and "UNIQUE constraint failed: nodes.id" in str(exc): return logger.error(f"Problem with adding node {node} {loc}") raise exc # Nodes index if not no_index: q = "INSERT INTO nodes_index VALUES(?, ?, ?, ?, ?)" try: c.execute(q, (node, lon, lon, lat, lat)) except sqlite3.IntegrityError as exc: logger.error(f"Problem with adding node to index {node} {loc}") raise exc if not no_commit: self.db.commit() def reindex_nodes(self): logger.debug("Reindexing nodes ...") t_start = time.time() c = self.db.cursor() c.execute('DELETE FROM nodes_index') q = ("INSERT INTO nodes_index " "SELECT id, x, x, y, y FROM nodes") c.execute(q) self.db.commit() c.execute('SELECT count(*) FROM nodes_index') cnt = c.fetchone()[0] t_delta = time.time() - t_start logger.debug(f"... done, #rows = {cnt}, time = {t_delta} sec") def add_nodes(self, nodes): """Add list of nodes to database. :param nodes: List[Tuple[node_key, Tuple[lat, lon]]] """ c = self.db.cursor() def get_node_index(): for key, (lat, lon) in nodes: yield key, lon, lon, lat, lat q = "INSERT INTO nodes_index VALUES(?, ?, ?, ?, ?)" c.executemany(q, get_node_index()) def get_node_vals(): for key, (lat, lon) in nodes: yield key, lon, lat q = "INSERT INTO nodes VALUES(?, ?, ?)" c.executemany(q, get_node_vals()) self.db.commit() def del_node(self, node): raise Exception("TODO") def add_edge(self, node_a, node_b, loc_a=None, loc_b=None, speed=None, edge_type=None, path=None, pathnum=None, no_index=False, no_commit=False): """Add new edge to the map. :param node_a: Label for the node that is the start of the edge :param node_b: Label for the node that is the end of the edge :param no_commit: Do not commit to database (remember to commit later) """ c = self.db.cursor() eid = (node_a, node_b).__hash__() c.execute('INSERT OR IGNORE INTO edges(id, path, pathnum, id1, id2, type, speed) VALUES (?, ?, ?, ?, ?, ?, ?)', (eid, path, pathnum, node_a, node_b, edge_type, speed)) # c.execute('SELECT last_insert_rowid();') # eid = c.fetchone()[0] if not no_index: if loc_a is None: c.execute('SELECT y, x FROM nodes WHERE id = ?;', (node_a, )) loc_a = c.fetchone() if loc_b is None: c.execute('SELECT y, x FROM nodes WHERE id = ?;', (node_b, )) loc_b = c.fetchone() lat1, lon1 = loc_a lat2, lon2 = loc_b if lat1 > lat2: lat1, lat2 = lat2, lat1 if lon1 > lon2: lon1, lon2 = lon2, lon1 c.execute('INSERT OR IGNORE INTO edges_index(id, minX, maxX, minY, maxY) VALUES (?, ?, ?, ?, ?)', (eid, lon1, lon2, lat1, lat2)) if not no_commit: self.db.commit() def add_edges(self, edges, no_index=False): """Add list of nodes to database. :param edges: List[Tuple[node_key, node_key]] or List[Tuple[node_key, node_key, path_key, int]] """ c = self.db.cursor() def get_edge(): for row in edges: row = list(row) + ([None] * (6 - len(row))) key_a, key_b, path, pathnum, edge_type, speed = row eid = (key_a, key_b).__hash__() yield eid, path, pathnum, key_a, key_b, edge_type, speed q = "INSERT INTO edges(id, path, pathnum, id1, id2, type, speed) VALUES(?, ?, ?, ?, ?, ?, ?);" c.executemany(q, get_edge()) self.db.commit() if not no_index: self.reindex_edges() def reindex_edges(self): logger.debug("Reindexing edges ...") t_start = time.time() c = self.db.cursor() # c2 = self.db.cursor() c.execute('DELETE FROM edges_index') q = ('INSERT INTO edges_index ' 'SELECT e.id, MIN(n1.x,n2.x), MAX(n1.x,n2.x), ' ' MIN(n1.y,n2.y), MAX(n1.y,n2.y) ' 'FROM edges e ' 'INNER JOIN nodes n1 ON n1.id = e.id1 ' 'INNER JOIN nodes n2 ON n2.id = e.id2') c.execute(q) # cnt = 0 # for row in c.execute(q): # # Contained in query # c2.execute('INSERT INTO edges_index(id, minX, maxX, minY, maxY) VALUES (?, ?, ?, ?, ?)', row) # cnt += 1 self.db.commit() c.execute('SELECT count(*) FROM edges_index') cnt = c.fetchone()[0] t_delta = time.time() - t_start logger.debug(f"... done, #rows = {cnt}, time = {t_delta} sec") def all_edges(self, bb=None): """Return all edges. :param bb: Bounding box :return: (key_a, loc_a, nbr, loc_b) """ c = self.db.cursor() q = 'SELECT e.id1, e.id2, n1.x AS n1x, n2.x AS n2x, n1.y AS n1y, n2.y AS n2y ' + \ 'FROM edges e, edges_index ei ' + \ 'LEFT JOIN nodes n1 ON n1.id = e.id1 ' + \ 'LEFT JOIN nodes n2 ON n2.id = e.id2 ' + \ 'WHERE ei.id == e.id' if bb: min_y, min_x, max_y, max_x = bb # Intersecting with query q += ' AND ei.maxX >= ? AND ei.minX <= ? AND ei.maxY >= ? AND ei.minY <= ?' c.execute(q, (min_x, max_x, min_y, max_y)) else: c.execute(q) for row in c.fetchall(): key_a, key_b, lon_a, lon_b, lat_a, lat_b = row yield key_a, (lat_a, lon_a), key_b, (lat_b, lon_b) def all_nodes(self, bb=None): """Return all nodes. :param bb: Bounding box (minY, minX, maxY, maxX) :return: """ c = self.db.cursor() q = ('SELECT n.id, n.x, n.y ' 'FROM nodes n, nodes_index ni ' 'WHERE n.id = ni.id ') if bb: minY, minX, maxY, maxX = bb q += 'AND ni.minX >= ? AND ni.maxX <= ? AND ni.minY >= ? AND ni.maxY <= ?' c.execute(q, (minX, maxX, minY, maxY)) else: c.execute(q) for row in c.fetchall(): key_a, lon_a, lat_a = row yield key_a, (lat_a, lon_a) def purge(self): pass def to_xy(self, name=None): """Create a map that uses a projected XY representation on which Euclidean distances can be used. """ if not self.use_latlon: return self if name is None: name = self.name + "_xy" logger.debug("Start transformation ...") t_start = time.time() nmap = self.__class__(name, dir=self.dir, use_latlon=self.use_latlon, crs_xy=self.crs_xy, crs_lonlat=self.crs_lonlat) raise Exception("to implement") t_delta = time.time() - t_start logger.debug(f"... done: rtree size = {self.rtree_size()}, time = {t_delta} sec") return nmap def latlon2xy(self, lat, lon): x, y = self.lonlat2xy(lon, lat) return x, y def latlon2yx(self, lat, lon): x, y = self.lonlat2xy(lon, lat) return y, x def xy2latlon(self, x, y): lon, lat = self.xy2lonlat(x, y) return lat, lon def yx2latlon(self, y, x): lon, lat = self.xy2lonlat(x, y) return lat, lon def nodes_closeto(self, loc, max_dist=None, max_elmt=None): """Return all nodes close to the given location. :param loc: Location :param max_dist: Maximal distance from the location :param max_elmt: Return only the most nearby nodes """ t_start = time.time() lat, lon = loc[:2] lat_b, lon_l, lat_t, lon_r = self.box_around_point((lat, lon), max_dist) bb = (lat_b, lon_l, # y_min, x_min lat_t, lon_r) # y_max, x_max nodes = self.all_nodes(bb=bb) t_delta_search = time.time() - t_start t_start = time.time() results = [] for key_o, loc_o in nodes: dist = self.distance(loc, loc_o) if dist < max_dist: results.append((dist, key_o, loc_o)) results.sort() t_delta_dist = time.time() - t_start logger.debug(f"Found {len(results)} closeby nodes " f"in {t_delta_search} sec and computed distances in {t_delta_dist} sec") if max_elmt is not None: results = results[:max_elmt] return results def edges_closeto(self, loc, max_dist=None, max_elmt=None): """Return all nodes that are on an edge that is close to the given location. :param loc: Location :param max_dist: Maximal distance from the location :param max_elmt: Return only the most nearby nodes """ print(f"edges_closeto({loc})") t_start = time.time() lat, lon = loc[:2] lat_b, lon_l, lat_t, lon_r = self.box_around_point((lat, lon), max_dist) bb = (lat_b, lon_l, # y_min, x_min lat_t, lon_r) # y_max, x_max logger.debug(f"Search in bounding box {bb}") nodes = self.all_edges(bb=bb) t_delta_search = time.time() - t_start t_start = time.time() results = [] for key_a, loc_a, key_b, loc_b in nodes: dist, pi, ti = self.distance_point_to_segment(loc, loc_a, loc_b) if dist < max_dist: results.append((dist, key_a, loc_a, key_b, loc_b, pi, ti)) results.sort() t_delta_dist = time.time() - t_start logger.debug(f"Found {len(results)} closeby edges " f"in {t_delta_search} sec and computed distances in {t_delta_dist} sec") if max_elmt is not None: results = results[:max_elmt] return results def nodes_nbrto(self, node): c = self.db.cursor() q = ('SELECT e.id2, n2.y, n2.x FROM edges e ' 'INNER JOIN nodes n2 ON n2.id = e.id2 ' 'WHERE e.id1 = ?') results = [] for nbr_label, nbr_lat, nbr_lon in c.execute(q, (node, )): results.append((nbr_label, (nbr_lat, nbr_lon))) return results def edges_nbrto(self, edge): l1, l2 = edge c = self.db.cursor() c.execute('SELECT n.y, n.x FROM nodes n WHERE id = ?', (l2, )) p2 = c.fetchone() results = [] # Edges that connect at end of this edge for l3, p3 in self.nodes_nbrto(l2): results.append((l2, p2, l3, p3)) # Edges that are in parallel and close edge_id = edge.__hash__() q = ('SELECT e.id1, e.id2, n1.y, n1.x, n2.y, n2.x FROM close_edges ce ' 'INNER JOIN edges e ON e.id = ce.id2 ' 'INNER JOIN nodes n1 ON n1.id = e.id1 ' 'INNER JOIN nodes n2 ON n2.id = e.id2 ' 'WHERE ce.id1 = ?') for l3, l4, p3lat, p3lon, p4lat, p4lon in c.execute(q, (edge_id,)): results.append((l3, (p3lat, p3lon), l4, (p4lat, p4lon))) return results def find_duplicates(self, func=None): """Find entries with identical locations.""" c = self.db.cursor() logger.debug('Find duplicates ...') t_start = time.time() cnt = 0 q = ('select count(*)as qty, group_concat(id) ' 'from nodes ' 'group by y, x ' 'having qty > 1 ') for ncnt, idxs in c.execute(q): func(int(idx) for idx in idxs.split(",")) t_delta = time.time() - t_start logger.info(f"Found {cnt} doubles, time: {t_delta} seconds") def connect_parallelroads(self, dist=0.5, bb=None): c = self.db.cursor() it = self.all_edges(bb=bb) if tqdm: it = tqdm.tqdm(list(it)) cnt = 0 for key_a, loc_a, key_b, loc_b in it: e_id1 = (key_a, key_b).__hash__() bb2 = [min(loc_a[0], loc_b[0]), min(loc_a[1], loc_b[1]), max(loc_a[0], loc_b[0]), max(loc_a[1], loc_b[1])] for key_c, loc_c, key_d, loc_d in self.all_edges(bb=bb2): e_id2 = (key_c, key_d).__hash__() if key_a == key_c or key_a == key_d or key_b == key_c or key_b == key_d: continue # print(f"Test: ({key_a},{key_b}) - ({key_c},{key_d})") if self.lines_parallel(loc_a, loc_b, loc_c, loc_d, d=dist): # print(f"Parallel: ({key_a},{key_b}) - ({key_c},{key_d})") c.execute('INSERT INTO close_edges(id1, id2) VALUES (?, ?)', (e_id1, e_id2)) c.execute('INSERT INTO close_edges(id1, id2) VALUES (?, ?)', (e_id2, e_id1)) cnt += 1 logger.debug(f"Linked {cnt} edges") self.db.commit() def nodes_to_paths(self, nodes, ignore_nopath=True): c = self.db.cursor() prev_path = None paths = [] for begin, end in zip(nodes[:-1], nodes[1:]): c.execute("SELECT path FROM edges WHERE id1=? AND id2=?", (begin, end)) path = c.fetchone()[0] if path is None and ignore_nopath: continue if path != prev_path: paths.append(path) prev_path = path return paths def path_dist(self, path): c = self.db.cursor() dist = 0 q = ('SELECT n1.y, n1.x, n2.y, n2.x FROM edges e ' 'INNER JOIN nodes n1 ON n1.id = e.id1 ' 'INNER JOIN nodes n2 ON n2.id = e.id2 ' 'WHERE e.pathnum>0 AND e.path=?') for lat1, lon1, lat2, lon2 in c.execute(q, (path,)): dist += self.distance((lat1, lon1), (lat2, lon2)) return dist def print_stats(self): print("Graph\n-----") print("Nodes: {}".format(len(self.graph))) def __str__(self): # s = "" # for label, (loc, nbrs, _) in self.graph.items(): # s += f"{label:<10} - ({loc[0]:10.4f}, {loc[1]:10.4f})\n" # return s c = self.db.cursor() c.execute("select sqlite_version()") row = c.fetchone() version = row[0] return f"SqliteMap({self.name}, size={self.size()}, version={version})" ================================================ FILE: leuvenmapmatching/matcher/__init__.py ================================================ # encoding: utf-8 """ leuvenmapmatching.matcher ~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ ================================================ FILE: leuvenmapmatching/matcher/base.py ================================================ # encoding: utf-8 """ leuvenmapmatching.matcher.base ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Base Matcher and Matching classes. This a generic base class to be used by matchers. This class itself does not implement a working matcher. :author: Wannes Meert :copyright: Copyright 2015-2021 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ from __future__ import print_function import math import sys import logging import time from collections import OrderedDict, defaultdict, namedtuple from itertools import islice from typing import List, Tuple, Dict, Any, Optional, Set import numpy as np from ..util.segment import Segment from ..util import approx_equal, approx_leq logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") approx_value = 0.0000000001 ema_const = namedtuple('EMAConst', ['prev', 'cur'])(0.7, 0.3) default_label_width = 25 class BaseMatching(object): """Matching object that represents a node in the Viterbi lattice.""" __slots__ = ['matcher', 'edge_m', 'edge_o', 'logprob', 'logprobema', 'logprobe', 'logprobne', 'obs', 'obs_ne', 'dist_obs', 'prev', 'prev_other', 'stop', 'length', 'delayed'] def __init__(self, matcher: 'BaseMatcher', edge_m: Segment, edge_o: Segment, logprob=-np.inf, logprobema=-np.inf, logprobe=-np.inf, logprobne=-np.inf, dist_obs: float = 0.0, obs: int = 0, obs_ne: int = 0, prev: Optional[Set['BaseMatching']] = None, stop: bool = False, length: int = 1, delayed: int = 0, **_kwargs): """ :param matcher: Reference to the Matcher used to generate this matching object. :param edge_m: Segment in the given graph (thus line between two nodes in the graph). :param edge_o: Segment in the given observations (thus line in between two observations). :param logprob: Log probability of this matching. :param logprobema: Exponential Mean Average of Log probability. :param logprobe: Emitting :param logprobne: Non-emitting :param dist_obs: Distance between map point and observation :param obs: Reference to path entry index (observation) :param obs_ne: Number of non-emitting states for this observation :param prev: Previous best matching objects :param stop: Stop after this matching (e.g. because probability is too low) :param length: Lenght of current matching sequence through lattice. :param delayed: This matching is temporarily stopped if >0 (e.g. to first explore better options). :param dist_m: Distance over graph :param dist_o: Distance over observations :param _kwargs: """ self.edge_m: Segment = edge_m self.edge_o: Segment = edge_o self.logprob: float = logprob # max probability self.logprobe: float = logprobe # Emitting self.logprobne: float = logprobne # Non-emitting self.logprobema: float = logprobema # exponential moving average log probability # TODO: Not used anymore? self.obs: int = obs # reference to path entry index (observation) self.obs_ne: int = obs_ne # number of non-emitting states for this observation self.dist_obs: float = dist_obs # Distance between map point and observation self.prev: Set[BaseMatching] = set() if prev is None else prev # Previous best matching objects self.prev_other: Set[BaseMatching] = set() # Previous matching objects with lower logprob self.stop: bool = stop self.length: int = length self.delayed: int = delayed self.matcher: BaseMatcher = matcher @property def prune_value(self): """Pruning the lattice (e.g. to delay) is based on this key.""" return self.logprob # return self.logprobema def next(self, edge_m: Segment, edge_o: Segment, obs: int = 0, obs_ne: int = 0): """Create a next lattice Matching object with this Matching object as the previous one in the lattice.""" new_stop = False if edge_m.is_point() and edge_o.is_point(): # node to node dist = self.matcher.map.distance(edge_m.p1, edge_o.p1) # proj_m = edge_m.p1 # proj_o = edge_o.pi elif edge_m.is_point() and not edge_o.is_point(): # node to edge dist, proj_o, t_o = self.matcher.map.distance_point_to_segment(edge_m.p1, edge_o.p1, edge_o.p2) # proj_m = edge_m.p1 edge_o.pi = proj_o edge_o.ti = t_o elif not edge_m.is_point() and edge_o.is_point(): # edge to node dist, proj_m, t_m = self.matcher.map.distance_point_to_segment(edge_o.p1, edge_m.p1, edge_m.p2) if not self.matcher.only_edges and (approx_equal(t_m, 0.0) or approx_equal(t_m, 1.0)): if __debug__ and logger.isEnabledFor(logging.DEBUG): logger.debug(f" | Stopped trace: Too close to end, {t_m}") new_stop = True else: return None edge_m.pi = proj_m edge_m.ti = t_m # proj_o = edge_o.pi elif not edge_m.is_point() and not edge_o.is_point(): # edge to edge dist, proj_m, proj_o, t_m, t_o = self.matcher.map.distance_segment_to_segment(edge_m.p1, edge_m.p2, edge_o.p1, edge_o.p2) edge_m.pi = proj_m edge_m.ti = t_m edge_o.pi = proj_o edge_o.ti = t_o else: raise Exception(f"Should not happen") logprob_trans, props_trans = self.matcher.logprob_trans(self, edge_m, edge_o, is_prev_ne=(self.obs_ne != 0), is_next_ne=(obs_ne != 0)) logprob_obs, props_obs = self.matcher.logprob_obs(dist, self, edge_m, edge_o, is_ne=(obs_ne != 0)) if __debug__ and logprob_trans > 0: raise Exception(f"logprob_trans = {logprob_trans} > 0") if __debug__ and logprob_obs > 0: raise Exception(f"logprob_obs = {logprob_obs} > 0") new_logprob_delta = logprob_trans + logprob_obs if obs_ne == 0: new_logprobe = self.logprob + new_logprob_delta new_logprobne = 0 new_logprob = new_logprobe new_length = self.length + 1 else: # Non-emitting states require normalisation # "* e^(ne_length_factor_log)" or "- ne_length_factor_log" for every step to a non-emitting # state to prefer shorter paths new_logprobe = self.logprobe + self.matcher.ne_length_factor_log # The obvious choice would be average to compensate for that non-emitting states # create different path lengths between emitting nodes. # We use min() as it is a monotonic function, in contrast with an average new_logprobne = min(self.logprobne, new_logprob_delta) new_logprob = new_logprobe + new_logprobne # Alternative approach with an average # new_logprobne = self.logprobne + new_logprob_delta # "+ 1" to punish non-emitting states a bit less. Otherwise it would be # similar to (Pr_tr*Pr_obs)**2, which punishes just one non-emitting state too much. # new_logprob = new_logprobe + new_logprobne / (obs_ne + 1) new_length = self.length new_logprobema = ema_const.cur * new_logprob_delta + ema_const.prev * self.logprobema new_stop |= self.matcher.do_stop(new_logprob / new_length, dist, logprob_trans, logprob_obs) if __debug__ and new_logprob > self.logprob: raise Exception(f"Expecting a monotonic probability, " f"new_logprob = {new_logprob} > logprob = {self.logprob}") if not new_stop or (__debug__ and logger.isEnabledFor(logging.DEBUG)): m_next = self.__class__(self.matcher, edge_m, edge_o, logprob=new_logprob, logprobne=new_logprobne, logprobe=new_logprobe, logprobema=new_logprobema, obs=obs, obs_ne=obs_ne, prev={self}, dist_obs=dist, stop=new_stop, length=new_length, delayed=self.delayed, **props_trans, **props_obs) return m_next else: return None @classmethod def first(cls, logprob_init, edge_m, edge_o, matcher, dist_obs): """Create an initial lattice Matching object.""" logprob_obs, props_obs = matcher.logprob_obs(dist_obs, None, edge_m, edge_o) logprob = logprob_init + logprob_obs new_stop = matcher.do_stop(logprob, dist_obs, logprob_init, logprob_obs) if not new_stop or logger.isEnabledFor(logging.DEBUG): m_next = cls(matcher, edge_m=edge_m, edge_o=edge_o, logprob=logprob, logprobema=logprob, logprobe=logprob, logprobne=0, dist_obs=dist_obs, obs=0, stop=new_stop, **props_obs) return m_next else: return None def update(self, m_next): """Update the current entry if the new matching object for this state is better. :param m_next: The new matching object representing the same node in the lattice. :return: True if the current object is replaced, False otherwise """ # if self.length != m_next.length: # slogprob_norm = self.logprob / self.length # nlogprob_norm = m_next.logprob / m_next.length # else: # slogprob_norm = self.logprob # nlogprob_norm = m_next.logprob # if (self.stop == m_next.stop and slogprob_norm < nlogprob_norm) or (self.stop and not m_next.stop): # self._update_inner(m_next) # return True # elif abs(slogprob_norm - nlogprob_norm) < approx_value and self.stop == m_next.stop: # self.prev.update(m_next.prev) # self.stop = m_next.stop # return False assert self.length == m_next.length if (self.stop and not m_next.stop) \ or (self.stop == m_next.stop and self.logprob < m_next.logprob): self._update_inner(m_next) return True else: self.prev_other.update(m_next.prev) return False def _update_inner(self, m_other: 'BaseMatching'): self.edge_m = m_other.edge_m self.edge_o = m_other.edge_o self.logprob = m_other.logprob self.logprobe = m_other.logprobe self.logprobne = m_other.logprobne self.logprobema = m_other.logprobema self.dist_obs = m_other.dist_obs self.obs = m_other.obs self.obs_ne = m_other.obs_ne self.prev_other.update(self.prev) # Do we use this? self.prev = m_other.prev self.stop = m_other.stop self.delayed = m_other.delayed self.length = m_other.length def is_nonemitting(self): return self.obs_ne != 0 def is_emitting(self): return self.obs_ne == 0 def last_emitting_logprob(self): if self.is_emitting(): return self.logprob elif self.prev is None or len(self.prev) == 0: return 0 else: return next(iter(self.prev)).last_emitting_logprob() def __str__(self, label_width=None): stop = '' if self.stop: stop = 'x' else: stop = f'{self.delayed}' if label_width is None: label_width = default_label_width repr_tmpl = "{:<2} | {:<"+str(label_width)+"} | {:10.5f} | {:10.5f} | {:10.5f} | {:10.5f} | " +\ "{:<3} | {:10.5f} | {:<" + str(label_width) + "} |" return repr_tmpl.format(stop, self.label, self.logprob, self.logprob / self.length, self.logprobema, self.logprobne, self.obs, self.dist_obs, ",".join([str(prev.label) for prev in self.prev])) def __repr__(self): return "Matching<"+str(self.label)+">" @staticmethod def repr_header(label_width=None, stop=""): if label_width is None: label_width = default_label_width repr_tmpl = "{:<2} | {:<"+str(label_width)+"} | {:<10} | {:<10} | {:<10} | {:<10} | " + \ "{:<3} | {:<10} | {:<"+str(label_width)+"} |" return repr_tmpl.format(stop, "", "lg(Pr)", "nlg(Pr)", "slg(Pr)", "lg(Pr-ne)", "obs", "d(obs)", "prev") @staticmethod def repr_static(fields, label_width=None): if label_width is None: label_width = default_label_width default_fields = ["", "", float('nan'), float('nan'), float('nan'), float('nan'), "", float('nan'), "", ""] repr_tmpl = "{:<2} | {:<" + str(label_width) + "} | {:10.5f} | {:10.5f} | {:10.5f} | {:10.5f} | " + \ "{:<3} | {:10.5f} | {:<" + str(label_width) + "} |" if len(fields) < 8: fields = list(fields) + default_fields[len(fields):] return repr_tmpl.format(*fields) @property def label(self): if self.edge_m.p2 is None: return "{}---{}-{}".format(self.edge_m.l1, self.obs, self.obs_ne) else: return "{}-{}-{}-{}".format(self.edge_m.l1, self.edge_m.l2, self.obs, self.obs_ne) @property def cname(self): if self.edge_m.l2 is None: return "{}_{}_{}".format(self.edge_m.l1, self.obs, self.obs_ne) else: return "{}_{}_{}_{}".format(self.edge_m.l1, self.edge_m.l2, self.obs, self.obs_ne) @property def key(self): """Key that indicates the node or edge, observation and non-emitting step. This is the unique key that is used in the lattice. """ if self.edge_m.l2 is None: return tuple([self.edge_m.l1, self.obs, self.obs_ne]) else: return tuple([self.edge_m.l1, self.edge_m.l2, self.obs, self.obs_ne]) @property def shortkey(self): """Key that indicates the node or edge. Irrespective of the current observation.""" if self.edge_m.l2 is None: return self.edge_m.l1 else: return tuple([self.edge_m.l1, self.edge_m.l2]) @property def nodes(self): if self.edge_m.l2 is None: return [self.edge_m.l1] else: return [self.edge_m.l1, self.edge_m.l2] def __hash__(self): return self.cname.__hash__() def __lt__(self, o): return self.logprob < o.logprob def __le__(self, o): return self.logprob <= o.logprob def __eq__(self, o): return self.logprob == o.logprob def __ne__(self, o): return self.logprob != o.logprob def __ge__(self, o): return self.logprob >= o.logprob def __gt__(self, o): return self.logprob > o.logprob class LatticeColumn: def __init__(self, obs_idx): # 0 = obs, >0 = non-emitting between this obs and next self.obs_idx = obs_idx self.o = [] # type list[dict[label,Matching]] def __contains__(self, item): for c in self.o: if item in c: return True return False def __len__(self): return len(self.o) def set_delayed(self, delayed): """Update all delayed values.""" for c in self.o: for m in c.values(): m.delayed = delayed def dict(self, obs_ne=None): if obs_ne is None: raise AttributeError('obs_ne should be value') while obs_ne >= len(self.o): self.o.append({}) return self.o[obs_ne] def values_all(self): """All matches for the emitting layer and all non-emitting layers.""" values = set() for o in self.o: values.update(o.values()) return values def values(self, obs_ne=None): if obs_ne is None: raise AttributeError('obs_ne should be value') if len(self.o) <= obs_ne: return [] return self.o[obs_ne].values() def upsert(self, matching): # type: (BaseMatching) -> None if matching is None: return None while matching.obs_ne >= len(self.o): self.o.append({}) c = self.o[matching.obs_ne] if matching.key in c: other_matching = c[matching.key] # type: BaseMatching other_matching.update(matching) else: c[matching.key] = matching return c[matching.key] def prune(self, obs_ne, max_lattice_width, expand_upto, prune_thr=None): """Prune given column in the lattice to fit in max_lattice_width. Also ignore all matchings with a probability lower than prune_thr. These are matchings that are worse than the matchings at the next observation that are retained after pruning. :param obs_ne: :param max_lattice_width: :param expand_upto: The current expand level :return: """ cur_lattice = [m for m in self.values(obs_ne) if not m.stop] if __debug__: logger.debug('Prune lattice[{},{}] from {} to {}, with prune thr {}' .format(self.obs_idx, obs_ne, len([m for m in cur_lattice if not m.stop and m.delayed == expand_upto]), max_lattice_width, prune_thr)) cnt_pruned = 0 if max_lattice_width is not None and len(cur_lattice) > max_lattice_width: ms = sorted(cur_lattice, key=lambda t: t.prune_value, reverse=True) cur_width = max_lattice_width m_last = ms[cur_width - 1] # Extend current width if next pruned matching has same logprob as last kept matching # This increases the lattice width but otherwise the algorithm depends on the # order of edges/nodes and is not deterministic. while cur_width < len(ms) and ms[cur_width].prune_value == m_last.prune_value: m_last = ms[cur_width] cur_width += 1 if prune_thr is not None: while cur_width > 0 and ms[cur_width - 1].prune_value < prune_thr: cur_width -= 1 for m in ms[:cur_width]: # type: BaseMatching if m.delayed > expand_upto: m.delayed = expand_upto # expand now for m in ms[cur_width:]: if m.delayed <= expand_upto: if __debug__: cnt_pruned += 1 m.delayed = expand_upto + 1 # expand later if cur_width > 0: prune_thr = ms[cur_width - 1].prune_value if __debug__: logger.debug(f'Pruned {cnt_pruned} matchings, return {prune_thr}') return prune_thr class BaseMatcher: def __init__(self, map_con, obs_noise=1, max_dist_init=None, max_dist=None, min_prob_norm=None, non_emitting_states=True, max_lattice_width=None, only_edges=True, obs_noise_ne=None, matching=BaseMatching, non_emitting_length_factor=0.75, **kwargs): """Initialize a matcher for map matching. This a generic base class to be used by matchers. This class itself does not implement a working matcher. Distances are in meters when using latitude-longitude. :param map_con: Map object to connect to map database :param obs_noise: Standard deviation of noise :param obs_noise_ne: Standard deviation of noise for non-emitting states (is set to obs_noise if not give) :param max_dist_init: Maximum distance from start location (if not given, uses max_dist) :param max_dist: Maximum distance from path (this is a hard cut, min_prob_norm should be better) :param min_prob_norm: Minimum normalized probability of observations (ema) :param non_emitting_states: Allow non-emitting states. A non-emitting state is a state that is not associated with an observation. Here we assume it can be associated with a location in between two observations to allow for pruning. It is advised to set min_prob_norm and/or max_dist to avoid visiting all possible nodes in the graph. :param max_lattice_width: Only continue from a limited number of states (thus locations) for a given observation. This possibly speeds up the matching by a lot. If there are more possible next states, the states with the best likelihood so far are selected. The other states are 'delayed'. If the matching is continued later with a larger value using `increase_max_lattice_width`, the algorithms continuous from these delayed states. :param only_edges: Do not include nodes as states, only edges. This is the typical setting for HMM methods. :param matching: Matching type :param non_emitting_length_factor: Reduce the probability of a sequence of non-emitting states the longer it is. This can be used to prefer shorter paths. This is separate from the transition probabilities because transition probabilities are averaged for non-emitting states and thus the length is also averaged out. To define a custom transition and/or emission probability distribtion, overwrite the following functions: - :meth:`logprob_trans` - :meth:`logprob_obs` """ self.map = map_con # type: BaseMap if max_dist: self.max_dist = max_dist else: self.max_dist = np.inf if max_dist_init: self.max_dist_init = max_dist_init else: self.max_dist_init = self.max_dist if min_prob_norm: self.min_logprob_norm = math.log(min_prob_norm) else: self.min_logprob_norm = -np.inf logger.debug(f"Matcher.min_logprob_norm = {self.min_logprob_norm}, Matcher.max_dist = {self.max_dist}") self.obs_noise = obs_noise if obs_noise_ne is None: self.obs_noise_ne = obs_noise else: self.obs_noise_ne = obs_noise_ne self.path = None self.lattice = None # type: Optional[dict[int,LatticeColumn]] # Best path through lattice: self.lattice_best = None # type: Optional[list[BaseMatching]] self.node_path = None # type: Optional[list[str]] self.matching = matching self.non_emitting_states = non_emitting_states # type: bool self.non_emitting_states_maxnb = 100 self.max_lattice_width = max_lattice_width # type: Optional[int] self.only_edges = only_edges # type: bool self.expand_now = 0 # all m.delayed <= expand_upto will be expanded self.early_stop_idx = None # Penalties self.ne_length_factor_log = math.log(non_emitting_length_factor) def logprob_trans(self, prev_m, edge_m, edge_o, is_prev_ne=False, is_next_ne=False): # type: (BaseMatcher, BaseMatching, Segment, Segment, bool, bool) -> Tuple[float, Dict[str, Any]] """Transition probability. Note: In contrast with a regular HMM, this cannot be a probability density function, it needs to be a proper probability (thus values between 0.0 and 1.0). :return: probability, properties that are passed to the matching object """ return 0, {} # All probabilities are 1 (thus technically not a distribution) def logprob_obs(self, dist, prev_m, new_edge_m, new_edge_o, is_ne=False): """Emission probability. Note: In contrast with a regular HMM, this cannot be a probability density function, it needs to be a proper probability (thus values between 0.0 and 1.0). :return: probability, properties that are passed to the matching object """ return 0, {} def match_gpx(self, gpx_file, unique=True): """Map matching from a gpx file""" from ..util.gpx import gpx_to_path path = gpx_to_path(gpx_file) return self.match(path, unique=unique) def do_stop(self, logprob_norm, dist, logprob_trans, logprob_obs): if logprob_norm < self.min_logprob_norm: logger.debug(f" | Stopped trace: norm(log(Pr)) too small: {logprob_norm} < {self.min_logprob_norm}" f" -- lPr_t = {logprob_trans:.3f}, lPr_o = {logprob_obs:.3f}") return True if dist > self.max_dist: logger.debug(f" | Stopped trace: distance too large: {dist} > {self.max_dist}") return True return False def _insert(self, m_next): return self.lattice[m_next.obs].upsert(m_next) def match(self, path, unique=False, tqdm=None, expand=False): """Dynamic Programming based (HMM-like) map matcher. If the matcher fails to match the entire path, the last matched index is returned. This index can be used to run the matcher again from that observation onwards. :param path: list[Union[tuple[lat, lon], tuple[lat, lon, time]] :param unique: Only retain unique nodes in the sequence (avoid repetitions) :param tqdm: Use a tqdm progress reporter (default is None) :param expand: Expand the current lattice (delayed matches) :return: Tuple of (List of BaseMatching, index of last observation that was matched) """ if __debug__: logger.debug("Start matching path of length {}".format(len(path))) # Initialisation if expand: self.expand_now += 1 if self.path != path: is_path_extended = True if len(path) > len(self.path): for pi, spi in zip(path, self.path): if pi != spi: is_path_extended = False break else: is_path_extended = False if is_path_extended: self.lattice[len(self.path) - 1].set_delayed(self.expand_now) for obs_idx in range(len(self.path), len(path)): if obs_idx not in self.lattice: self.lattice[obs_idx] = LatticeColumn(obs_idx) self.path = path else: raise Exception(f'Cannot expand for a new path, should be the same path (or an extension).') else: self.path = path self.expand_now = 0 nb_start_nodes = self._create_start_nodes(use_edges=self.only_edges) if nb_start_nodes == 0: self.lattice_best = [] return [], 0 if __debug__ and logger.isEnabledFor(logging.DEBUG): self.print_lattice(obs_idx=0, label_width=default_label_width, debug=True) # Start iterating over observations 1..end t_start = time.time() iterator = range(1, len(path)) if tqdm: iterator = tqdm(iterator) self.early_stop_idx = None for obs_idx in iterator: if __debug__: logger.debug("--- obs {} --- {} ---".format(obs_idx, self.path[obs_idx])) # check if early stopping has occured cnt_lat_size_not_zero = False for m_tmp in self.lattice[obs_idx - 1].values(0): if not m_tmp.stop: cnt_lat_size_not_zero = True break # if len(self.lattice[obs_idx - 1]) == 0: if not cnt_lat_size_not_zero: if __debug__: logger.debug("No solutions found anymore") self.early_stop_idx = obs_idx - 1 logger.info(f'Stopped early at observation {self.early_stop_idx}') break # Expand matches self._match_states(obs_idx) if self.non_emitting_states: # Fill in non-emitting states between previous and current observation self._match_non_emitting_states(obs_idx - 1, expand=expand) if self.max_lattice_width: # Prune again if non_emitting_states reactives matches from match_states self.lattice[obs_idx].prune(0, self.max_lattice_width, self.expand_now) if __debug__ and logger.isEnabledFor(logging.DEBUG): self.print_lattice(obs_idx=obs_idx, label_width=default_label_width, debug=True) logger.debug(f"--- end obs {obs_idx} ---") t_delta = time.time() - t_start logger.info("--- end ---") logger.info("Build lattice in {} seconds".format(t_delta)) # Backtrack to find best path if not self.early_stop_idx: one_no_stop = False for m in self.lattice[len(path) - 1].values_all(): # todo: could be values(0) ? if not m.stop: one_no_stop = True break if not one_no_stop: self.early_stop_idx = len(path) - 1 if self.early_stop_idx is not None: if self.early_stop_idx == 0: self.lattice_best = [] return [], 0 start_idx = self.early_stop_idx - 1 else: start_idx = len(self.path) - 1 node_path = self._build_node_path(start_idx, unique) return node_path, start_idx def _skip_ne_states(self, prev_m): # type: (BaseMatcher, BaseMatching) -> bool return False def _create_start_nodes(self, use_edges=True): """Find those nodes that are close to the first point in the path. :return: Number of created start points. """ # Initialisation on first observation if self.expand_now > 0: # No need to search for new points, only activate delayed matches self.lattice[0].prune(0, self.max_lattice_width, self.expand_now) return len(self.lattice[0]) t_start = time.time() self.lattice = dict() for obs_idx in range(len(self.path)): self.lattice[obs_idx] = LatticeColumn(obs_idx) if use_edges: nodes = self.map.edges_closeto(self.path[0], max_dist=self.max_dist_init) else: nodes = self.map.nodes_closeto(self.path[0], max_dist=self.max_dist_init) if __debug__: logger.debug("--- obs {} --- {} ---".format(0, self.path[0])) t_delta = time.time() - t_start logger.info("Initialized lattice with {} starting points in {} seconds".format(len(nodes), t_delta)) if len(nodes) == 0: logger.info(f'Stopped early at observation 0' f', no starting points/edges x found for which ' f'|x - ({self.path[0][0]:.2f},{self.path[0][1]:.2f})| < {self.max_dist_init}') return 0 if __debug__: logger.debug(self.matching.repr_header()) logprob_init = 0 # math.log(1.0/len(nodes)) if use_edges: # Search for nearby edges for dist_obs, label1, loc1, label2, loc2, pi, ti in nodes: if label2 == label1: continue edge_m = Segment(label1, loc1, label2, loc2, pi, ti) edge_o = Segment(f"O{0}", self.path[0]) m_next = self.matching.first(logprob_init, edge_m, edge_o, self, dist_obs) if m_next is not None: self.lattice[0].upsert(m_next) if __debug__: logger.debug(str(m_next)) else: # Search for nearby nodes for dist_obs, label, loc in nodes: edge_m = Segment(label, loc) edge_o = Segment(f"O{0}", self.path[0]) m_next = self.matching.first(logprob_init, edge_m, edge_o, self, dist_obs) if m_next is not None: self.lattice[0].upsert(m_next) if __debug__: logger.debug(str(m_next)) if self.max_lattice_width: self.lattice[0].prune(0, max_lattice_width=self.max_lattice_width, expand_upto=self.expand_now) # if self.non_emitting_states: # self._match_non_emitting_states(0, path) return len(self.lattice[0]) def increase_delayed(self, expand_from=None): if expand_from is None: expand_from = self.expand_now + 1 for col in self.lattice.values(): for colo in col.o: for m in colo.values(): if m.delayed >= expand_from: m.delayed += 1 def _match_states(self, obs_idx, prev_lattice=None, max_dist=None, inc_delayed=False): """Match states :param obs_idx: :param prev_lattice: Start from this list instead of the previous column in the lattice :param max_dist: Use map.*_closeto instead of map.*_nbrto :param inc_delayed: Increase delayed property when new state is created :return: True is new states have been found, False otherwise. """ if prev_lattice is None: prev_lattice = [m for m in self.lattice[obs_idx - 1].values(0) if not m.stop and m.delayed == self.expand_now] count = 0 for m in prev_lattice: # type: BaseMatching if m.stop: assert False # should not happen continue count += 1 if m.edge_m.is_point(): # == Move to neighbour from node == if max_dist is None: nbrs = self.map.nodes_nbrto(m.edge_m.l1) else: nbrs = self.map.nodes_closeto(m.edge_m.p1, max_dist=max_dist) # print("Neighbours for {}: {}".format(m, nbrs)) if nbrs is None: if __debug__: logger.debug("No neighbours found for node {}".format(m.edge_m.l1)) continue if __debug__: logger.debug(" + Move to {} neighbours from node {}".format(len(nbrs), m.edge_m.l1)) logger.debug(m.repr_header()) for nbr_label, nbr_loc in nbrs: # === Move from node to node (or stay on node) === if not self.only_edges: edge_m = Segment(nbr_label, nbr_loc) edge_o = Segment(f"O{obs_idx}", self.path[obs_idx]) m_next = m.next(edge_m, edge_o, obs=obs_idx) if m_next is not None: if inc_delayed: m_next.delayed += 1 self._insert(m_next) if __debug__: logger.debug(str(m_next)) # === Move from node to edge === if m.edge_m.l1 != nbr_label: edge_m = Segment(m.edge_m.l1, m.edge_m.p1, nbr_label, nbr_loc) edge_o = Segment(f"O{obs_idx}", self.path[obs_idx]) m_next = m.next(edge_m, edge_o, obs=obs_idx) if m_next is not None: if inc_delayed: m_next.delayed += 1 self._insert(m_next) if __debug__: logger.debug(str(m_next)) else: if __debug__: logger.debug(self.matching.repr_static(('x', f'{nbr_label}-{nbr_label} < self-loop'))) else: # == Move to neighbour from edge == if __debug__: logger.debug(" + Move to neighbour from edge {}".format(m.label)) logger.debug(m.repr_header()) # === Stay on edge === edge_m = Segment(m.edge_m.l1, m.edge_m.p1, m.edge_m.l2, m.edge_m.p2) edge_o = Segment(f"O{obs_idx}", self.path[obs_idx]) m_next = m.next(edge_m, edge_o, obs=obs_idx) if m_next is not None: if inc_delayed: m_next.delayed += 1 self._insert(m_next) if __debug__: logger.debug(str(m_next)) # === Move from edge to node === if not self.only_edges: edge_m = Segment(m.edge_m.l2, m.edge_m.p2) edge_o = Segment(f"O{obs_idx}", self.path[obs_idx]) m_next = m.next(edge_m, edge_o, obs=obs_idx) if m_next is not None: if inc_delayed: m_next.delayed += 1 self._insert(m_next) if __debug__: logger.debug(str(m_next)) else: # === Move from edge to next edge === if max_dist is None: nbrs = self.map.edges_nbrto((m.edge_m.l1, m.edge_m.l2)) # type: list else: nbrs = [(l1, p1, l2, p2) for _, l1, p1, l2, p2, _, _ in self.map.edges_closeto(m.edge_m.pi, max_dist=max_dist)] if nbrs is None or len(nbrs) == 0: if __debug__: logger.debug(f"No neighbours found for edge {m.edge_m.label}") continue for nbr_label1, nbr_loc1, nbr_label2, nbr_loc2 in nbrs: # same edge is different action, opposite edge should be allowed to return in a one-way street if m.edge_m.l2 != nbr_label2 and m.edge_m.l1 != nbr_label1: edge_m = Segment(nbr_label1, nbr_loc1, nbr_label2, nbr_loc2) edge_o = Segment(f"O{obs_idx}", self.path[obs_idx]) m_next = m.next(edge_m, edge_o, obs=obs_idx) if m_next is not None: if inc_delayed: m_next.delayed += 1 self._insert(m_next) if __debug__: mstr = str(m_next) logger.debug(mstr) if self.max_lattice_width: self.lattice[obs_idx].prune(0, self.max_lattice_width, self.expand_now) if count == 0: if __debug__: logger.debug("No active solution found anymore") return False return True def _match_non_emitting_states(self, obs_idx, expand=False): """Match sequences of nodes that all refer to the same observation at obs_idx. Assumptions: This method assumes that the lattice is filled up for both obs_idx and obs_idx + 1. :param obs_idx: Index of the first observation used (the second will be obs_idx + 1) :return: None """ obs = self.path[obs_idx] if obs_idx < len(self.path) - 1: obs_next = self.path[obs_idx + 1] else: obs_next = None # The current states are the current observation's states if expand: cur_lattice = dict((m.key, m) for m in self.lattice[obs_idx].values(0) if not m.stop and m.delayed == self.expand_now) else: cur_lattice = dict((m.key, m) for m in self.lattice[obs_idx].values(0) if not (m.stop or m.delayed > 0)) lattice_toinsert = list() # The current best states are the next observation's states if you would ignore non-emitting states lattice_best = dict((m.shortkey, m) for m in self.lattice[obs_idx + 1].values(0) if not m.stop) lattice_ne = set(m.shortkey for m in self.lattice[obs_idx + 1].values(0) if not m.stop and self._skip_ne_states(m)) # cur_lattice = set(self.lattice[obs_idx].values()) nb_ne = 0 prune_thr = None while len(cur_lattice) > 0 and nb_ne < self.non_emitting_states_maxnb: nb_ne += 1 if __debug__: logger.debug("--- obs {}:{} --- {} - {} ---".format(obs_idx, nb_ne, obs, obs_next)) cur_lattice = self._match_non_emitting_states_inner(cur_lattice, obs_idx, obs, obs_next, nb_ne, lattice_best, lattice_ne) if self.max_lattice_width is not None: self.lattice[obs_idx].prune(nb_ne, self.max_lattice_width, self.expand_now, prune_thr) # Link to next observation self._match_non_emitting_states_end(cur_lattice, obs_idx + 1, obs_next, lattice_best, expand=expand) if self.max_lattice_width is not None: prune_thr = self.lattice[obs_idx + 1].prune(0, self.max_lattice_width, self.expand_now, None) if self.max_lattice_width is not None: self.lattice[obs_idx + 1].prune(0, self.max_lattice_width, self.expand_now, None) # logger.info('Used {} levels of non-emitting states'.format(nb_ne)) # for m in lattice_toinsert: # self._insert(m) def _node_in_prev_ne(self, m_next, label): """Is the given node already visited in the chain of non-emitting states. :param m_next: :param label: Node label :return: True or False """ # for m in itertools.chain(m_next.prev, m_next.prev_other): for m in m_next.prev: # type: BaseMatching if m.obs != m_next.obs: return False assert(m_next.obs_ne != m.obs_ne) # print('prev', m.shortkey, 'checking for ', label) # if label == m.shortkey: if label in m.nodes: return True if m.obs_ne == 0: return False if self._node_in_prev_ne(m, label): return True return False @staticmethod def _insert_tmp(m_next, lattice): if m_next.key in lattice: return lattice[m_next.key].update(m_next) else: lattice[m_next.key] = m_next return True def _match_non_emitting_states_inner(self, cur_lattice, obs_idx, obs, obs_next, nb_ne, lattice_best, lattice_ne): # cur_lattice_new = dict() cur_lattice_new = self.lattice[obs_idx].dict(nb_ne) for m in cur_lattice.values(): # type: BaseMatching if m.stop or m.delayed != self.expand_now: continue if m.shortkey in lattice_ne: logger.debug(f"Skip non-emitting states from {m.label}, already visited") continue # == Move to neighbour edge from edge == if m.edge_m.l2 is not None and self.only_edges: nbrs = self.map.edges_nbrto((m.edge_m.l1, m.edge_m.l2)) # print("Neighbours for {}: {}".format(m, nbrs)) if nbrs is None or len(nbrs) == 0: if __debug__: logger.debug(f"No neighbours found for edge {m.edge_m.label} ({m.label}, non-emitting)") continue for nbr_label1, nbr_loc1, nbr_label2, nbr_loc2 in nbrs: if self._node_in_prev_ne(m, nbr_label2): if __debug__: logger.debug(self.matching.repr_static(('x', '{} < node in prev ne'.format(nbr_label2)))) continue # === Move to next edge === if m.edge_m.l2 != nbr_label2 and m.edge_m.l1 != nbr_label2: edge_m = Segment(nbr_label1, nbr_loc1, nbr_label2, nbr_loc2) edge_o = Segment(f"O{obs_idx}", obs, f"O{obs_idx+1}", obs_next) m_next = m.next(edge_m, edge_o, obs=obs_idx, obs_ne=nb_ne) if m_next is not None: if m_next.key in cur_lattice_new: if m_next.shortkey in lattice_best: if approx_leq(m_next.dist_obs, lattice_best[m_next.shortkey].dist_obs): cur_lattice_new[m_next.key].update(m_next) else: m_next.stop = True if __debug__ and logger.isEnabledFor(logging.DEBUG): logger.debug(f" | Stopped trace: distance larger than best for key {m_next.shortkey}: " f"{m_next.dist_obs} > {lattice_best[m_next.shortkey].dist_obs}") else: cur_lattice_new[m_next.key].update(m_next) else: if m_next.shortkey in lattice_best: # if m_next.logprob > lattice_best[m_next.shortkey].logprob: if approx_leq(m_next.dist_obs, lattice_best[m_next.shortkey].dist_obs): cur_lattice_new[m_next.key] = m_next # lattice_best[m_next.shortkey] = m_next # lattice_toinsert.append(m_next) else: if __debug__ and logger.isEnabledFor(logging.DEBUG): logger.debug(f" | Stopped trace: distance larger than best for key {m_next.shortkey}: " f"{m_next.dist_obs} > {lattice_best[m_next.shortkey].dist_obs}") m_next.stop = True else: cur_lattice_new[m_next.key] = m_next # lattice_best[m_next.shortkey] = m_next # lattice_toinsert.append(m_next) # cur_lattice_new.add(m_next) if __debug__: logger.debug(str(m_next)) else: if __debug__: logger.debug(self.matching.repr_static(('x', f'{nbr_label1}-{nbr_label2} < goes back (ne)'))) # == Move to neighbour node from node== if m.edge_m.l2 is None and not self.only_edges: cur_node = m.edge_m.l1 nbrs = self.map.nodes_nbrto(cur_node) if nbrs is None: if __debug__: logger.debug( f"No neighbours found for node {cur_node} ({m.label}, non-emitting)") continue if __debug__: logger.debug( f" + Move to {len(nbrs)} neighbours from node {cur_node} ({m.label}, non-emitting)") logger.debug(m.repr_header()) for nbr_label, nbr_loc in nbrs: # print(f"self._node_in_prev_ne({m.label}, {nbr_label}) = {self._node_in_prev_ne(m, nbr_label)}") if self._node_in_prev_ne(m, nbr_label): if __debug__: logger.debug(self.matching.repr_static(('x', '{} < node in prev ne'.format(nbr_label)))) continue # === Move to next node === if m.edge_m.l1 != nbr_label: edge_m = Segment(nbr_label, nbr_loc) edge_o = Segment(f"O{obs_idx}", obs, f"O{obs_idx+1}", obs_next) m_next = m.next(edge_m, edge_o, obs=obs_idx, obs_ne=nb_ne) if m_next is not None: if m_next.key in cur_lattice_new: cur_lattice_new[m_next.key].update(m_next) else: if m_next.shortkey in lattice_best: # if m_next.logprob > lattice_best[m_next.shortkey].logprob: if m_next.dist_obs < lattice_best[m_next.shortkey].dist_obs: cur_lattice_new[m_next.key] = m_next lattice_best[m_next.shortkey] = m_next # lattice_toinsert.append(m_next) elif __debug__ and logger.isEnabledFor(logging.DEBUG): m_next.stop = True cur_lattice_new[m_next.key] = m_next # lattice_toinsert.append(m_next) else: cur_lattice_new[m_next.key] = m_next lattice_best[m_next.shortkey] = m_next # lattice_toinsert.append(m_next) # cur_lattice_new.add(m_next) if __debug__: logger.debug(str(m_next)) else: if __debug__: logger.debug(f"x | {m.edge_m.l1}-{nbr_label} < self-loop") return cur_lattice_new def _match_non_emitting_states_end(self, cur_lattice, obs_idx, obs_next, lattice_best, expand=False): for m in cur_lattice.values(): # type: BaseMatching if m.stop or m.delayed > self.expand_now: continue if m.edge_m.l2 is not None: # Move to neighbour edge from edge nbrs = self.map.edges_nbrto((m.edge_m.l1, m.edge_m.l2)) # print("Neighbours for {}: {}".format(m, nbrs)) if nbrs is None or len(nbrs) == 0: if __debug__: logger.debug("No neighbours found for edge {} ({})".format(m.edge_m.label, m.label)) continue if __debug__: logger.debug(f" + Move to {len(nbrs)} neighbours from edge {m.edge_m.label} " f"({m.label}, non-emitting->emitting)") logger.debug(m.repr_header()) for nbr_label1, nbr_loc1, nbr_label2, nbr_loc2 in nbrs: if self._node_in_prev_ne(m, nbr_label2): if __debug__: logger.debug(self.matching.repr_static(('x', '{} < node in prev ne'.format(nbr_label2)))) continue # Move to next edge if m.edge_m.l1 != nbr_label2 and m.edge_m.l2 != nbr_label2: edge_m = Segment(nbr_label1, nbr_loc1, nbr_label2, nbr_loc2) edge_o = Segment(f"O{obs_idx+1}", obs_next) m_next = m.next(edge_m, edge_o, obs=obs_idx) if m_next is not None: if m_next.shortkey in lattice_best: # if m_next.dist_obs < lattice_best[m_next.shortkey].dist_obs: if m_next.logprob > lattice_best[m_next.shortkey].logprob: lattice_best[m_next.shortkey] = m_next # lattice_toinsert.append(m_next) self.lattice[obs_idx].upsert(m_next) elif __debug__ and logger.isEnabledFor(logging.DEBUG): m_next.stop = True # lattice_toinsert.append(m_next) self.lattice[obs_idx].upsert(m_next) else: lattice_best[m_next.shortkey] = m_next # lattice_toinsert.append(m_next) self.lattice[obs_idx].upsert(m_next) if __debug__: logger.debug(str(m_next)) else: if __debug__: logger.debug(self.matching.repr_static(('x', '{} < going back'.format(nbr_label2)))) else: # m.edge_m.l2 is None: # Move to neighbour node from node cur_node = m.edge_m.l1 nbrs = self.map.nodes_nbrto(cur_node) # print("Neighbours for {}: {}".format(m, nbrs)) if nbrs is None: if __debug__: logger.debug("No neighbours found for node {}".format(cur_node, m.label)) continue if __debug__: logger.debug(f" + Move to {len(nbrs)} neighbours from node {cur_node} " f"({m.label}, non-emitting->emitting)") logger.debug(m.repr_header()) for nbr_label, nbr_loc in nbrs: if self._node_in_prev_ne(m, nbr_label): if __debug__: logger.debug(self.matching.repr_static(('x', '{} < node in prev ne'.format(nbr_label)))) continue # Move to next node if m.edge_m.l1 != nbr_label: # edge_m = Segment(m.edge_m.l1, m.edge_m.p1, nbr_label, nbr_loc) edge_m = Segment(nbr_label, nbr_loc) edge_o = Segment(f"O{obs_idx+1}", obs_next) m_next = m.next(edge_m, edge_o, obs=obs_idx) if m_next is not None: if m_next.shortkey in lattice_best: # if m_next.dist_obs < lattice_best[m_next.shortkey].dist_obs: if m_next.logprob > lattice_best[m_next.shortkey].logprob: lattice_best[m_next.shortkey] = m_next # lattice_toinsert.append(m_next) self.lattice[obs_idx].upsert(m_next) elif __debug__ and logger.isEnabledFor(logging.DEBUG): m_next.stop = True # lattice_toinsert.append(m_next) self.lattice[obs_idx].upsert(m_next) else: lattice_best[m_next.shortkey] = m_next # lattice_toinsert.append(m_next) self.lattice[obs_idx].upsert(m_next) if __debug__: logger.debug(str(m_next)) else: if __debug__: logger.debug(self.matching.repr_static(('x', '{} < self-loop'.format(nbr_label)))) def get_matching(self, identifier=None): m = None # type: Optional[BaseMatching] if isinstance(identifier, BaseMatching): m = identifier elif identifier is None: col = self.lattice[len(self.lattice) - 1] for curm in col.values_all(): if m is None or curm.logprob > m.logprob: m = curm elif type(identifier) is int: # If integer, search for the best matching at this index in the lattice for cur_m in self.lattice[identifier].values_all(): # type:BaseMatching if not cur_m.stop and (m is None or cur_m.logprob > m.logprob): m = cur_m elif type(identifier) is str: # If string, try to parse identifier parts = identifier.split('-') idx, ne, key = None, None, None if len(parts) == 4: nodea, nodeb, idx, ne = [int(part) for part in parts] key = (nodea, nodeb, idx, ne) col = self.lattice[idx] # type: LatticeColumn col_ne = col.o[ne] m = col_ne[key] elif len(parts) == 3: node, idx, ne = [int(part) for part in parts] key = (node, idx, ne) col = self.lattice[idx] # type: LatticeColumn col_ne = col.o[ne] m = col_ne[key] elif len(parts) == 1: m = None l1 = int(parts[0]) for l in self.lattice.values(): # type: LatticeColumn for curm in l.values_all(): if (curm.edge_m.l1 == l1 or curm.edge_m.l2 == l1) and \ (m is None or curm.logprob > m.logprob): m = curm else: raise AttributeError(f'Unknown string format for matching. ' 'Expects -- or ---.') return m def get_matching_path(self, start_m): """List of Matching objects that end in the given Matching object.""" start_m = self.get_matching(start_m) return self._build_matching_path(start_m) def get_node_path(self, start_m, only_nodes=False): """List of node/edge names that end in the given Matching object.""" path = self.get_matching_path(start_m) node_path = [m.shortkey for m in path] if only_nodes: node_path = self.node_path_to_only_nodes(node_path) return node_path def get_path(self, only_nodes=True, allow_jumps=False, only_closest=True): """A list with all the nodes (no edges) the matched path passes through.""" if only_nodes is False: return self.node_path if self.node_path is None or len(self.node_path) == 0: return [] path = self.node_path_to_only_nodes(self.node_path, allow_jumps=allow_jumps) if only_closest: m = self.lattice_best[0] if m.edge_m.ti > 0.5: path.pop(0) return path def node_path_to_only_nodes(self, path, allow_jumps=False): """Path of nodes and edges to only nodes. :param path: List of node names or edges as (node name, node name) :param allow_jumps: Allow a path over edges that are not connected. This occurs when matches are added without an edge, for example, when searching for edges in the distance neighborhood instead in the graph. :return: List of node names """ nodes = [] prev_state = path[0] if type(prev_state) is tuple: nodes.append(prev_state[0]) nodes.append(prev_state[1]) prev_node = prev_state[1] else: nodes.append(prev_state) prev_node = prev_state for state in path[1:]: if state == prev_state: continue if type(state) is not tuple: if state != prev_node: nodes.append(state) prev_node = state elif type(state) is tuple: if state[0] == prev_node: if state[1] != prev_node: nodes.append(state[1]) prev_node = state[1] elif state[1] == prev_node: if state[0] != prev_node: nodes.append(state[0]) prev_node = state[0] elif not allow_jumps: raise Exception(f"State {state} does not have as previous node {prev_node}") else: nodes.append(state[0]) nodes.append(state[1]) prev_node = state[1] else: raise Exception(f"Unknown type of state: {state} ({type(state)})") prev_state = state return nodes def _build_matching_path(self, start_m, max_depth=None): lattice_best = [] node_max = start_m cur_depth = 0 if __debug__ and logger.isEnabledFor(logging.DEBUG): logger.debug(self.matching.repr_header(stop=" ")) logger.debug("Start ({}): {}".format(node_max.obs, node_max)) lattice_best.append(node_max) if node_max.is_emitting(): cur_depth += 1 # for obs_idx in reversed(range(start_idx)): if max_depth is None: max_depth = len(self.lattice) + 1 while cur_depth < max_depth and len(node_max.prev) > 0: node_max_last = node_max node_max: Optional[BaseMatching] = None for prev_m in node_max_last.prev: if prev_m is not None and (node_max is None or prev_m.logprob > node_max.logprob): node_max = prev_m if node_max is None: logger.error("Did not find a matching node for path point at index {}. ".format(node_max_last.obs) + "Stopped building path.") break logger.debug("Max ({}): {}".format(node_max.obs, node_max)) lattice_best.append(node_max) if node_max.is_emitting(): cur_depth += 1 lattice_best = list(reversed(lattice_best)) return lattice_best def _build_node_path(self, start_idx, unique=True, max_depth=None, last_is_e=False): """Build the path from the lattice. :param start_idx: :param unique: :param max_depth: :param last_is_e: Last matched lattice node should be an emitting state. In case the matching stops early, the longest path can be in between two observations and thus be a nonemitting state (which by definition has a lower probability than the last emitting state). If this argument is set to true, the longer match is preferred. :return: """ node_max = None node_max_ne = 0 if last_is_e: for m in self.lattice[start_idx].values_all(): # type:BaseMatching if not m.stop and (node_max is None or m.logprob > node_max.logprob): node_max = m else: for m in self.lattice[start_idx].values_all(): # type:BaseMatching if not m.stop and (node_max is None or m.obs_ne > node_max_ne or m.logprob > node_max.logprob): node_max_ne = m.obs_ne node_max = m if node_max is None: logger.error("Did not find a matching node for path point at index {}".format(start_idx)) return None self.lattice_best = self._build_matching_path(node_max, max_depth) node_path = [m.shortkey for m in self.lattice_best] if unique: self.node_path = [] prev_node = None for node in node_path: if node != prev_node: self.node_path.append(node) prev_node = node else: self.node_path = node_path return self.node_path def increase_max_lattice_width(self, max_lattice_width, unique=False, tqdm=None): """Increase the value for max_lattice_width and continue the matching with all paths that were ignored so far (up to the new max_lattice_width). This is useful when the matcher is first run with a small max_lattice_width to be fast, but when the true path is not obvious and excluded from the first guesses. When the matcher stops early, this method allows to easily expand the search space. :param max_lattice_width: New maximal number of paths to consider :param unique: See match method :param tqdm: See match method """ self.max_lattice_width = max_lattice_width return self.match(self.path, unique=unique, tqdm=tqdm, expand=True) def continue_with_distance(self, from_matches=None, k=2, nb_obs=2, max_dist=None): """Continue the matcher but ignore edges and allow jumps to nearby edged. :param from_matches: Search in the neigborhood of these matches :param k: If from_matches is not given, the k best matches are used in the last nb_obs observations since last early_stop_idx :praram nb_obs: If from_matches is not given, the k best matches are used in the last nb_obs observations since last early_stop_idx :param max_dist: Add edges that are maximally max_dist away from the previous match. If none, self.max_dist * 3 is used. """ if from_matches is None: from_matches = self.best_last_matches(k=k, nb_obs=nb_obs) self.increase_delayed() if max_dist is None: max_dist = self.max_dist * 3 for obs_idx, cur_matches in from_matches.items(): self._match_states(obs_idx, prev_lattice=cur_matches, max_dist=max_dist, inc_delayed=True) def path_bb(self): """Get boundig box of matched path (if it exists, otherwise return None).""" path = self.path plat, plon = islice(zip(*path), 2) lat_min, lat_max = min(plat), max(plat) lon_min, lon_max = min(plon), max(plon) bb = lat_min, lon_min, lat_max, lon_max return bb def print_lattice(self, file=None, obs_idx=None, obs_ne=0, label_width=None, debug=False): if debug: xprint = logger.debug else: if file is None: file = sys.stdout xprint = lambda arg: print(arg, file=file) # print("Lattice:", file=file) if obs_idx is not None: idxs = [obs_idx] else: idxs = range(len(self.lattice)) for idx in idxs: if len(self.lattice[idx]) > 0: if label_width is None: label_width = 0 for m in self.lattice[idx].values(obs_ne): label_width = max(label_width, len(str(m.label))) xprint("--- obs {} ---".format(idx)) xprint(self.matching.repr_header(label_width=label_width)) for m in sorted(self.lattice[idx].values(obs_ne), key=lambda t: str(t.label)): xprint(m.__str__(label_width=label_width)) def lattice_dot(self, file=None, precision=None, render=False): """Write the lattice as a Graphviz DOT file. :param file: File object to print to. Prints to stdout if None. :param precision: Precision of (log) probabilities. :param render: Try to render the generated Graphviz file. """ if file is None: file = sys.stdout if precision is None: prfmt = '' else: prfmt = f'.{precision}f' print('digraph lattice {', file=file) print('\trankdir=LR;', file=file) # Vertices for idx_ob in range(len(self.lattice)): col = self.lattice[idx_ob] for idx_ne in range(len(col)): ms = col.values(idx_ne) if len(ms) == 0: continue cnames = [(m.obs_ne, m.cname, m.stop, m.delayed) for m in ms] cnames.sort() cur_obs_ne = -1 print('\t{\n\t\trank=same; ', file=file) for obs_ne, cname, stop, delayed in cnames: if obs_ne != cur_obs_ne: if cur_obs_ne != -1: print('\t};\n\t{\n\t\trank=same; ', file=file) cur_obs_ne = obs_ne if stop: options = 'label="{} x",color=gray,fontcolor=gray'.format(cname) elif delayed > self.expand_now: options = 'label="{} d{}",color=gray,fontcolor=gray'.format(cname, delayed) elif self.expand_now != 0: options = 'label="{} d{}"'.format(cname, delayed) else: options = 'label="{} "'.format(cname) print('\t\t{} [{}];'.format(cname, options), file=file) print('\t};', file=file) # Edges for idx_ob in range(len(self.lattice)): col = self.lattice[idx_ob] for idx_ne in range(len(col)): ms = col.values(idx_ne) if len(ms) == 0: continue for m in ms: for mp in m.prev: if m.stop or m.delayed > self.expand_now: options = ',color=gray,fontcolor=gray' else: options = '' print(f'\t {mp.cname} -> {m.cname} [label="{m.logprob:{prfmt}}"{options}];', file=file) for mp in m.prev_other: if m.stop or m.delayed > self.expand_now: options = ',color=gray,fontcolor=gray' else: options = '' print(f'\t {mp.cname} -> {m.cname} [color=gray,label="{m.logprob:{prfmt}}"{options}];', file=file) print('}', file=file) if render and file is not None: import subprocess as sp from pathlib import Path from io import TextIOWrapper if isinstance(file, Path): fn = str(file.canonical()) elif isinstance(file, TextIOWrapper): file.flush() fn = file.name else: fn = str(file) cmd = ['dot', '-Tpdf', '-O', fn] logger.debug(' '.join(cmd)) sp.call(cmd) def print_lattice_stats(self, file=None, verbose=False): if file is None: file = sys.stdout print("Stats lattice", file=file) print("-------------", file=file) stats = OrderedDict() stats["nbr levels"] = len(self.lattice) if self.lattice else "?" total_nodes = 0 max_nodes = 0 min_nodes = 9999999 if self.lattice: sizes = [] for idx in range(len(self.lattice)): level = self.lattice[idx].values(0) # stats["#nodes[{}]".format(idx)] = len(level) sizes.append(len(level)) total_nodes += len(level) if len(level) < min_nodes: min_nodes = len(level) if len(level) > max_nodes: max_nodes = len(level) stats["nbr lattice"] = total_nodes if verbose: stats["nbr lattice[level]"] = ", ".join([str(s) for s in sizes]) stats["avg lattice[level]"] = total_nodes/len(self.lattice) stats["min lattice[level]"] = min_nodes stats["max lattice[level]"] = max_nodes if self.lattice_best and len(self.lattice_best) > 0: stats["avg obs distance"] = np.mean([m.dist_obs for m in self.lattice_best]) stats["last logprob"] = self.lattice_best[-1].logprob stats["last length"] = self.lattice_best[-1].length stats["last norm logprob"] = self.lattice_best[-1].logprob / self.lattice_best[-1].length if verbose: stats["best logprob"] = ", ".join(["{:.3f}".format(m.logprob) for m in self.lattice_best]) stats["best norm logprob"] = \ ", ".join(["{:.3f}".format(m.logprob/m.length) for i, m in enumerate(self.lattice_best)]) stats["best norm prob"] = \ ", ".join(["{:.3f}".format(math.exp(m.logprob/m.length)) for i, m in enumerate(self.lattice_best)]) for key, val in stats.items(): print("{:<24} : {}".format(key, val), file=file) def node_counts(self): if self.lattice is None: return None counts = defaultdict(lambda: 0) for level in self.lattice.values(): for m in level.values_all(): counts[m.label] += 1 return counts def inspect_early_stopping(self): """Analyze the lattice and try to find most plausible reason why the matching stopped early and print to stdout.""" if self.early_stop_idx is None: print("No early stopping.") return col = self.lattice[self.early_stop_idx - 1] print("The last matched nodes or edges were:") first_row = True ignore = set() for ne_i in range(len(col.o) - 1, -1, -1): for v in col.o[ne_i].values(): if v.key not in ignore: if first_row: print(v.repr_header()) first_row = False print(v) ignore.update(r.key for r in v.prev) def best_last_matches(self, k=1, nb_obs=3): """Return the k best last matches. :param k: Number of best matches to keep for an observation :param nb_obs: How many last matched observations to consider """ import heapq if self.early_stop_idx is None: col_idx = len(self.lattice) - 1 else: col_idx = self.early_stop_idx - 1 hh = [] obs_cnt = 0 while col_idx >= 0 and obs_cnt < nb_obs: h = [] col = self.lattice[col_idx] col_oneselected = False for ne_i in range(len(col.o) - 1, -1, -1): for v in col.o[ne_i].values(): if v.stop: continue if len(h) < k: heapq.heappush(h, (v.logprob, v)) col_oneselected = True elif v.logprob > h[0][0]: heapq.heappop(h) heapq.heappush(h, (v.logprob, v)) col_oneselected = True hh.extend(h) if col_oneselected is False: print(f'break in {col_idx=}') break col_idx -= 1 obs_cnt += 1 result = defaultdict(list) for m in hh: m = m[1] result[m.obs + 1].append(m) # return [m[1] for m in hh] return result def copy_lastinterface(self, nb_interfaces=1): """Copy the current matcher and keep the last interface as the start point. This method allows you to perform incremental matching without keeping the entire lattice in memory. You need to run :meth:`match_incremental` on this object to continue from the existing (partial) lattice. Otherwise, if you use :meth:`match`, it will be overwritten. Open question, if there is no need to keep track of older lattices, it will probably be more efficient to clear the older parts of the interface instead of copying the newer parts. :param nb_interfaces: Nb of interfaces (columns in lattice) to keep. Default is 1, the last one. :return: new Matcher object """ matcher = self.__class__(self.map, obs_noise=self.obs_noise, max_dist_init=self.max_dist_init, max_dist=self.max_dist, min_prob_norm=self.min_logprob_norm, non_emitting_states=self.non_emitting_states, max_lattice_width=self.max_lattice_width, only_edges=self.only_edges, obs_noise_ne=self.obs_noise_ne, matching=self.matching, avoid_goingback=self.avoid_goingback, non_emitting_length_factor=math.exp(self.ne_length_factor_log)) matcher.lattice = [] matcher.path = [] for int_i in range(len(self.lattice) - nb_interfaces, len(self.lattice)): matcher.lattice.append(self.lattice[int_i]) matcher.path.append(self.path[int_i]) return matcher @property def path_pred(self): """The matched path, both nodes and/or edges (depending on your settings).""" return self.node_path @property def path_pred_onlynodes(self): """A list with all the nodes (no edges) the matched path passes through.""" return self.get_path(only_nodes=True, allow_jumps=False) @property def path_pred_onlynodes_withjumps(self): """A list with all the nodes (no edges) the matched path passes through.""" return self.get_path(only_nodes=True, allow_jumps=True) def path_pred_distance(self): """Total distance of the matched path.""" if self.lattice_best is None: return None if len(self.lattice_best) == 1: return 0 dist = 0 m_prev = self.lattice_best[0] for idx, m in enumerate(self.lattice_best[1:]): if m_prev.edge_m.label != m.edge_m.label and m_prev.edge_m.l2 == m.edge_m.l1: # Go over the connection between two edges to compute the distance cdist = self.map.distance(m_prev.edge_m.pi, m_prev.edge_m.p2) cdist += self.map.distance(m_prev.edge_m.p2, m.edge_m.pi) else: cdist = self.map.distance(m_prev.edge_m.pi, m.edge_m.pi) dist += cdist m_prev = m return dist def path_distance(self): """Total distance of the observations.""" if self.lattice_best is None: return None if len(self.lattice_best) == 1: return 0 dist = 0 m_prev = self.lattice_best[0] for m in self.lattice_best[1:]: dist += self.map.distance(m_prev.edge_o.pi, m.edge_o.pi) m_prev = m return dist def path_all_distances(self): """Return a list of all distances between observed trace and map. One entry for each point in the map and point in the trace that are mapped to each other. In case non-emitting nodes are used, extra entries can be present where a point in the trace or a point in the map is mapped to a segment. """ path = self.lattice_best dists = [m.dist_obs for m in path] return dists ================================================ FILE: leuvenmapmatching/matcher/distance.py ================================================ # encoding: utf-8 """ leuvenmapmatching.matcher.distance ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging import math from .base import BaseMatching, BaseMatcher from ..util.segment import Segment from ..util.debug import printd MYPY = False if MYPY: from typing import Tuple, Any, Dict logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") class DistanceMatching(BaseMatching): __slots__ = ['d_s', 'd_o', 'lpe', 'lpt'] # Additional fields def __init__(self, *args, d_s=0.0, d_o=0.0, lpe=0.0, lpt=0.0, **kwargs): """ :param args: Arguments for BaseMatching :param d_s: Distance between two (interpolated) states :param d_o: Distance between two (interpolated) observations :param lpe: Log probability of emission :param lpt: Log probablity of transition :param kwargs: Arguments for BaseMatching """ super().__init__(*args, **kwargs) self.d_o: float = d_o self.d_s: float = d_s self.lpe: float = lpe self.lpt: float = lpt def _update_inner(self, m_other): # type: (DistanceMatching, DistanceMatching) -> None super()._update_inner(m_other) self.d_s = m_other.d_s self.d_o = m_other.d_o self.lpe = m_other.lpe self.lpt = m_other.lpt @staticmethod def repr_header(label_width=None, stop=""): res = BaseMatching.repr_header(label_width) res += f" {'dt(o)':<6} | {'dt(s)':<6} |" if logger.isEnabledFor(logging.DEBUG): res += f" {'lg(Pr-t)':<9} | {'lg(Pr-e)':<9} |" return res def __str__(self, label_width=None): res = super().__str__(label_width) res += f" {self.d_o:>6.2f} | {self.d_s:>6.2f} |" if logger.isEnabledFor(logging.DEBUG): res += f" {self.lpt:>9.2f} | {self.lpe:>9.2f} |" return res def __repr__(self): return self.label class DistanceMatcher(BaseMatcher): """ Map Matching that takes into account the distance between matched locations on the map compared to the distance between the observations (that are matched to these locations). It thus prefers matched paths that have a similar distance than the observations. Inspired on the method presented in: P. Newson and J. Krumm. Hidden markov map matching through noise and sparseness. In Proceedings of the 17th ACM SIGSPATIAL international conference on advances in geographic information systems, pages 336–343. ACM, 2009. The options available in :class:``BaseMatcher`` are inherited. Additionally, this class offers: - Transition probability is lower if the distance between observations and states is different - Transition probability is lower if the the next match is going back on an edge or to a previous edge - Transition probability is lower if two neighboring states represent not connected edges - Skip non-emitting states if distance between states and observations is close to each other """ def __init__(self, *args, **kwargs): """Create a new object. :param map_con: Map object to connect to map database :param obs_noise: Standard deviation of noise :param obs_noise_ne: Standard deviation of noise for non-emitting states (is set to obs_noise if not given) :param max_dist_init: Maximum distance from start location (if not given, uses max_dist) :param max_dist: Maximum distance from path (this is a hard cut, min_prob_norm should be better) :param min_prob_norm: Minimum normalized probability of observations (ema) :param non_emitting_states: Allow non-emitting states. A non-emitting state is a state that is not associated with an observation. Here we assume it can be associated with a location in between two observations to allow for pruning. It is advised to set min_prob_norm and/or max_dist to avoid visiting all possible nodes in the graph. :param non_emitting_length_factor: Reduce the probability of a sequence of non-emitting states the longer it is. This can be used to prefer shorter paths. This is separate from the transition probabilities because transition probabilities are averaged for non-emitting states and thus the length is also averaged out. :param max_lattice_width: Restrict the lattice (or possible candidate states per observation) to this value. If there are more possible next states, the states with the best likelihood so far are selected. :param dist_noise: Standard deviation of difference between distance between states and distance between observatoins. If not given, set to obs_noise :param dist_noise_ne: If not given, set to dist_noise :param restrained_ne: Avoid non-emitting states if the distance between states and between observations is close to each other. :param avoid_goingback: If true, the probability is lowered for a transition that returns back to a previous edges or returns to a position on an edge. :param args: Arguments for BaseMatcher :param kwargs: Arguments for BaseMatcher """ if not kwargs.get("only_edges", True): logger.warning("The MatcherDistance method only works on edges as states. Nodes have been disabled.") kwargs["only_edges"] = True if "matching" not in kwargs: kwargs["matching"] = DistanceMatching super().__init__(*args, **kwargs) self.use_original = kwargs.get('use_original', False) # if not use_original, the following value for beta gives a prob of 0.5 at dist=x_half: # beta = np.sqrt(np.power(x_half, 2) / (np.log(2)*2)) self.dist_noise = kwargs.get('dist_noise', self.obs_noise) self.dist_noise_ne = kwargs.get('dist_noise_ne', self.dist_noise) self.beta = 2 * self.dist_noise ** 2 self.beta_ne = 2 * self.dist_noise_ne ** 2 self.sigma = 2 * self.obs_noise ** 2 self.sigma_ne = 2 * self.obs_noise_ne ** 2 self.restrained_ne = kwargs.get('restrained_ne', True) self.restrained_ne_thr = 1.25 # Threshold self.exact_dt_s = True # Newson and Krumm is 'True' self.avoid_goingback = kwargs.get('avoid_goingback', True) self.gobackonedge_factor_log = math.log(0.5) self.gobacktoedge_factor_log = math.log(0.5) self.first_farend_penalty = math.log(0.75) # should be > gobacktoedge_factor_log self.notconnectededges_factor_log = math.log(0.5) def logprob_trans(self, prev_m, edge_m, edge_o, is_prev_ne=False, is_next_ne=False): # type: (DistanceMatcher, DistanceMatching, Segment, Segment, bool, bool) -> Tuple[float, Dict[str, Any]] """Transition probability. The probability is defined with a formula from the exponential family. :math:`P(dt) = exp(-d_t^2 / (2 * dist_{noise}^2))` with :math:`d_t = |d_s - d_o|, d_s = |loc_{prev\_state} - loc_{cur\_state}|, d_o = |loc_{prev\_obs} - loc_{cur\_obs}|` This function is more tolerant for low values. The intuition is that values under a certain distance should all be close to probability 1.0. Note: We should also smooth the distance between observations to handle outliers better. :param prev_m: Previous matching / state :param edge_m: Edge between matchings / states :param edge_o: Edge between observations :param is_prev_ne: Is previous state non-emitting :param is_next_ne: Is the next state non-emitting :param dist_o: First output of distance_progress :param dist_m: Second output of distance_progress :return: """ d_z = self.map.distance(prev_m.edge_o.pi, edge_o.pi) is_same_edge = False if (prev_m.edge_m.l1 == edge_m.l1 and prev_m.edge_m.l2 == edge_m.l2) or \ (prev_m.edge_m.l1 == edge_m.l2 and prev_m.edge_m.l2 == edge_m.l1): is_same_edge = True if ((not self.exact_dt_s) or is_same_edge or # On same edge prev_m.edge_m.l2 != edge_m.l1): # Edges are not connected d_x = self.map.distance(prev_m.edge_m.pi, edge_m.pi) else: # Take into account the curvature d_x = self.map.distance(prev_m.edge_m.pi, prev_m.edge_m.p2) + self.map.distance(prev_m.edge_m.p2, edge_m.pi) if is_next_ne: # For non-emitting states, the distances are added # Otherwise it can map to a sequence of short segments and stay at the same # observation because the difference is then always small. d_z += prev_m.d_o d_x += prev_m.d_s d_t = abs(d_z - d_x) # p_dt = 1 / beta * math.exp(-d_t / beta) if is_prev_ne or is_next_ne: beta = self.beta_ne else: beta = self.beta logprob = -d_t ** 2 / beta # Penalties if prev_m.edge_m.label == edge_m.label: # Staying in same state if self.avoid_goingback and edge_m.key == prev_m.edge_m.key and edge_m.ti < prev_m.edge_m.ti: # Going back on edge (direction is from p1 to p2 of the segment) logprob += self.gobackonedge_factor_log # Prefer not going back elif (prev_m.edge_m.l1, prev_m.edge_m.l2) == (edge_m.l2, edge_m.l1): if self.avoid_goingback: logprob += self.gobackonedge_factor_log else: # Moving states if prev_m.edge_m.l2 != edge_m.l1: # We are moving between states that represent edges that are not connected through a node logprob += self.notconnectededges_factor_log elif self.avoid_goingback: # Goin back on state going_back = False for m in prev_m.prev: if edge_m.label == m.edge_m.label: going_back = True break if going_back: logprob += self.gobacktoedge_factor_log # prefer not going back props = { 'd_o': d_z, 'd_s': d_x, 'lpt': logprob } return logprob, props def logprob_obs(self, dist, prev_m=None, new_edge_m=None, new_edge_o=None, is_ne=False): # type: (DistanceMatcher, float, DistanceMatching, Segment, Segment, bool) -> Tuple[float, Dict[str, Any]] """Emission probability for emitting states. Exponential family: :math:`P(dt) = exp(-d_o^2 / (2 * obs_{noise}^2))` with :math:`d_o = |loc_{state} - loc_{obs}|` """ if is_ne: sigma = self.sigma_ne else: sigma = self.sigma result = -dist ** 2 / sigma props = { 'lpe': result } return result, props def _skip_ne_states(self, next_ne_m): # type: (DistanceMatcher, DistanceMatching) -> bool # Skip searching for non-emitting states when the distances between nodes # on the map are similar to the distances between the observation if not self.restrained_ne: return False if next_ne_m.d_s > 0: factor = (next_ne_m.d_o + next_ne_m.dist_obs) / next_ne_m.d_s else: factor = 0 if factor < self.restrained_ne_thr: logger.debug(f"Skip non-emitting states to {next_ne_m.label}: {factor} < {self.restrained_ne_thr} " "(observations close enough to each other)") return True return False ================================================ FILE: leuvenmapmatching/matcher/newsonkrumm.py ================================================ # encoding: utf-8 """ leuvenmapmatching.matcher.newsonkrumm ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Methods similar to Newson Krumm 2009 for comparison purposes. P. Newson and J. Krumm. Hidden markov map matching through noise and sparseness. In Proceedings of the 17th ACM SIGSPATIAL international conference on advances in geographic information systems, pages 336–343. ACM, 2009. :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ from scipy.stats import norm import math import logging from .base import BaseMatching, BaseMatcher logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") class NewsonKrummMatching(BaseMatching): __slots__ = ['d_s', 'd_o', 'lpe', 'lpt'] # Additional fields def __init__(self, *args, d_s=1.0, d_o=1.0, lpe=0.0, lpt=0.0, **kwargs): """ :param args: Arguments for BaseMatching :param d_s: Distance between two (interpolated) states :param d_o: Distance between two (interpolated) observations :param lpe: Log probability of emission :param lpt: Log probablity of transition :param kwargs: Arguments for BaseMatching """ super().__init__(*args, **kwargs) self.d_o: float = d_o self.d_s: float = d_s self.lpe: float = lpe self.lpt: float = lpt def _update_inner(self, m_other): # type: (NewsonKrummMatching, NewsonKrummMatching) -> None super()._update_inner(m_other) self.d_s = m_other.d_s self.d_o = m_other.d_o self.lpe = m_other.lpe self.lpt = m_other.lpt @staticmethod def repr_header(label_width=None, stop=""): res = BaseMatching.repr_header(label_width) res += f" {'dt(o)':<6} | {'dt(s)':<6} |" if logger.isEnabledFor(logging.DEBUG): res += f" {'lg(Pr-t)':<9} | {'lg(Pr-e)':<9} |" return res def __str__(self, label_width=None): res = super().__str__(label_width) res += f" {self.d_o:>6.2f} | {self.d_s:>6.2f} |" if logger.isEnabledFor(logging.DEBUG): res += f" {self.lpt:>9.2f} | {self.lpe:>9.2f} |" return res class NewsonKrummMatcher(BaseMatcher): """ Take distance between observations vs states into account. Based on the method presented in: P. Newson and J. Krumm. Hidden markov map matching through noise and sparseness. In Proceedings of the 17th ACM SIGSPATIAL international conference on advances in geographic information systems, pages 336–343. ACM, 2009. Two important differences: * Newson and Krumm use shortest path to handle situations where the distances between observations are larger than distances between nodes in the graph. The LeuvenMapMatching toolbox uses non-emitting states to handle this. We thus do not implement the shortest path algorithm in this class. * Transition and emission probability are transformed from densities to probababilities by taking the 1 - CDF instead of the PDF. Newson and Krumm defaults: - max_dist = 200 m - obs_noise = 4.07 m - beta = 1/6 - only_edges = True """ def __init__(self, *args, **kwargs): """ :param beta: Default is 1/6 :param beta_ne: Default is beta :param args: Arguments for BaseMatcher :param kwargs: Arguments for BaseMatcher """ if not kwargs.get("only_edges", True): logger.warning("The MatcherDistance method only works on edges as states. Nodes have been disabled.") kwargs["only_edges"] = True if "matching" not in kwargs: kwargs["matching"] = NewsonKrummMatching super().__init__(*args, **kwargs) # if not use_original, the following value for beta gives a prob of 0.5 at dist=x_half: # beta = np.sqrt(np.power(x_half, 2) / (np.log(2)*2)) self.beta = kwargs.get('beta', 1/6) self.beta_ne = kwargs.get('beta_ne', self.beta) self.obs_noise_dist = norm(scale=self.obs_noise) self.obs_noise_dist_ne = norm(scale=self.obs_noise_ne) self.ne_thr = 1.25 def logprob_trans(self, prev_m: NewsonKrummMatching, edge_m, edge_o, is_prev_ne=False, is_next_ne=False): """Transition probability. Main difference with Newson and Krumm: we know all points are connected thus do not compute the shortest path but the distance between two points. Original PDF: p(dt) = 1 / beta * e^(-dt / beta) with beta = 1/6 Transformed to probability: P(dt) = p(d > dt) = e^(-dt / beta) :param prev_m: :param edge_m: :param edge_o: :param is_prev_ne: :param is_next_ne: :return: """ d_z = self.map.distance(prev_m.edge_o.pi, edge_o.pi) if prev_m.edge_m.label == edge_m.label: d_x = self.map.distance(prev_m.edge_m.pi, edge_m.pi) else: d_x = self.map.distance(prev_m.edge_m.pi, prev_m.edge_m.p2) + self.map.distance(prev_m.edge_m.p2, edge_m.pi) d_t = abs(d_z - d_x) # p_dt = 1 / beta * math.exp(-d_t / beta) if is_prev_ne or is_next_ne: beta = self.beta_ne else: beta = self.beta # icp_dt = math.exp(-d_t / beta) # try: # licp_dt = math.log(icp_dt) # except ValueError: # licp_dt = float('-inf') licp_dt = -d_t / beta props = { 'd_o': d_z, 'd_s': d_x, 'lpt': licp_dt } return licp_dt, props def logprob_obs(self, dist, prev_m, new_edge_m, new_edge_o, is_ne=False): """Emission probability for emitting states. Original pdf: p(d) = N(0, sigma) with sigma = 4.07m Transformed to probability: P(d) = 2 * (1 - p(d > D)) = 2 * (1 - cdf) """ if is_ne: result = 2 * (1 - self.obs_noise_dist_ne.cdf(dist)) else: result = 2 * (1 - self.obs_noise_dist.cdf(dist)) try: result = math.log(result) except ValueError: result = -float("inf") props = { 'lpe': result } return result, props ================================================ FILE: leuvenmapmatching/matcher/simple.py ================================================ # encoding: utf-8 """ leuvenmapmatching.matcher.simple ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import math from scipy.stats import halfnorm, norm from .base import BaseMatcher, BaseMatching from ..util.segment import Segment class SimpleMatching(BaseMatching): pass class SimpleMatcher(BaseMatcher): def __init__(self, *args, **kwargs): """A simple matcher that prefers paths where each matched location is as close as possible to the observed position. :param avoid_goingback: Change the transition probability to be lower for the direction the path is coming from. :param kwargs: Arguments passed to :class:`BaseMatcher`. """ if "matching" not in kwargs: kwargs['matching'] = SimpleMatching super().__init__(*args, **kwargs) self.obs_noise_dist = halfnorm(scale=self.obs_noise) self.obs_noise_dist_ne = halfnorm(scale=self.obs_noise_ne) # normalize to max 1 to simulate a prob instead of density self.obs_noise_logint = math.log(self.obs_noise * math.sqrt(2 * math.pi) / 2) self.obs_noise_logint_ne = math.log(self.obs_noise_ne * math.sqrt(2 * math.pi) / 2) # Transition probability is divided (in logprob_trans) by this factor if we move back on the # current edge. self.avoid_goingback = kwargs.get('avoid_goingback', True) self.gobackonedge_factor_log = math.log(0.99) # Transition probability is divided (in logprob_trans) by this factor if the next state is # also the previous state, thus if we go back self.gobacktoedge_factor_log = math.log(0.5) # Transition probability is divided (in logprob_trans) by this factor if a transition is made # This is to try to stay on the same node unless there is a good reason self.transition_factor = math.log(0.9) def logprob_trans(self, prev_m: BaseMatching, edge_m: Segment, edge_o: Segment, is_prev_ne=False, is_next_ne=False): """Transition probability. Note: In contrast with a regular HMM, this is not a probability density function, it needs to be a proper probability (thus values between 0.0 and 1.0). """ logprob = 0 if prev_m.edge_m.label == edge_m.label: # Staying in same state if self.avoid_goingback and edge_m.key == prev_m.edge_m.key and edge_m.ti < prev_m.edge_m.ti: # Going back on edge logprob += self.gobackonedge_factor_log # prefer not going back else: # Moving states logprob += self.transition_factor if self.avoid_goingback: # Goin back on state going_back = False for m in prev_m.prev: if edge_m.label == m.edge_m.label: going_back = True break if going_back: logprob += self.gobacktoedge_factor_log # prefer not going back return logprob, {} # All probabilities are 1 (thus technically not a distribution) def logprob_obs(self, dist, prev_m, new_edge_m, new_edge_o, is_ne=False): """Emission probability. Note: In contrast with a regular HMM, this is not a probability density function, it needs to be a proper probability (thus values between 0.0 and 1.0). """ if is_ne: result = self.obs_noise_dist_ne.logpdf(dist) + self.obs_noise_logint_ne else: result = self.obs_noise_dist.logpdf(dist) + self.obs_noise_logint # print("logprob_obs: {} -> {:.5f} = {:.5f}".format(dist, result, math.exp(result))) return result, {} ================================================ FILE: leuvenmapmatching/util/__init__.py ================================================ # encoding: utf-8 """ leuvenmapmatching.util ~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ # No automatic loading to avoid dependency on packages such as nvector and gpxpy if not used. def approx_equal(a, b, rtol=0.0, atol=1e-08): return abs(a - b) <= (atol + rtol * abs(b)) def approx_leq(a, b, rtol=0.0, atol=1e-08): return (a - b) <= (atol + rtol * abs(b)) ================================================ FILE: leuvenmapmatching/util/debug.py ================================================ import logging logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") def printd(*args, **kwargs): """Print to debug output.""" logger.debug(*args, **kwargs) ================================================ FILE: leuvenmapmatching/util/dist_euclidean.py ================================================ # encoding: utf-8 """ leuvenmapmatching.util.dist_euclidean ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging import math import numpy as np logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") def distance(p1, p2): result = math.sqrt((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2) # print("distance({}, {}) -> {}".format(p1, p2, result)) return result def distance_point_to_segment(p, s1, s2, delta=0.0): p_int, ti = project(s1, s2, p, delta=delta) return distance(p_int, p), p_int, ti # l1a = np.array(s1) # l2a = np.array(s2) # pa = np.array(p) # return np.linalg.norm(np.cross(l2a - l1a, l1a - pa)) / np.linalg.norm(l2a - l1a) def distance_segment_to_segment(f1, f2, t1, t2): """Distance between segments.. :param f1: From :param f2: :param t1: To :param t2: :return: (distance, proj on f, proj on t, rel pos on f, rel pos on t) """ x1, y1 = f1 x2, y2 = f2 x3, y3 = t1 x4, y4 = t2 n = ((y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1)) if np.allclose([n], [0], rtol=0): # parallel is_parallel = True n = 0.0001 # TODO: simulates a point far away else: is_parallel = False u_f = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / n u_t = ((x2 - x1) * (y1 - y3) - (y2 - y1) * (x1 - x3)) / n xi = x1 + u_f * (x2 - x1) yi = y1 + u_f * (y2 - y1) changed_f = False changed_t = False if u_t > 1: u_t = 1 changed_t = True elif u_t < 0: u_t = 0 changed_t = True if u_f > 1: u_f = 1 changed_f = True elif u_f < 0: u_f = 0 changed_f = True if not changed_t and not changed_f: return 0, (xi, yi), (xi, yi), u_f, u_t xf = x1 + u_f * (x2 - x1) yf = y1 + u_f * (y2 - y1) xt = x3 + u_t * (x4 - x3) yt = y3 + u_t * (y4 - y3) if changed_t and changed_f: # Compare furthest point from intersection with segment df = (xf - xi) ** 2 + (yf - yi) ** 2 dt = (xt - xi) ** 2 + (yt - yi) ** 2 if df > dt: changed_t = False else: changed_f = False if changed_t: pt = (xt, yt) pf, u_f = project(f1, f2, pt) elif changed_f: pf = (xf, yf) pt, u_t = project(t1, t2, pf) else: raise Exception(f"Should not happen") d = distance(pf, pt) return d, pf, pt, u_f, u_t def project(s1, s2, p, delta=0.0): """ :param s1: Segment start :param s2: Segment end :param p: Point :param delta: Keep delta fraction away from ends :return: Point of projection, Relative position on segment """ if np.isclose(s1[0], s2[0], rtol=0) and np.isclose(s1[1], s2[1], rtol=0): return s1, 0.0 l2 = (s1[0]-s2[0])**2 + (s1[1]-s2[1])**2 t = max(delta, min(1-delta, ((p[0]-s1[0])*(s2[0]-s1[0]) + (p[1]-s1[1])*(s2[1]-s1[1])) / l2)) return (s1[0] + t * (s2[0]-s1[0]), s1[1] + t * (s2[1]-s1[1])), t def interpolate_path(path, dd): """ TODO: interplate time as third term :param path: (y, x) :param dd: Distance difference (meter) :return: """ path_new = [path[0]] for p1, p2 in zip(path, path[1:]): dist = distance(p1, p2) if dist > dd: dt = int(math.ceil(dist / dd)) dx = (p2[0] - p1[0]) / dt dy = (p2[1] - p1[1]) / dt px, py = p1[0], p1[1] for _ in range(dt): px += dx py += dy path_new.append((px, py)) path_new.append(p2) return path_new def box_around_point(p, dist): lat, lon = p lat_t, lon_r = lat + dist, lon + dist lat_b, lon_l = lat - dist, lon - dist return lat_b, lon_l, lat_t, lon_r def lines_parallel(la, lb, lc, ld, d=None): x1 = la[0] - lb[0] y1 = la[1] - lb[1] if x1 == 0: if y1 == 0: return False s1 = 0 else: s1 = math.atan(abs(y1 / x1)) x2 = lc[0] - ld[0] y2 = lc[1] - ld[1] if x2 == 0: s2 = 0 if y2 == 0: return False else: s2 = math.atan(abs(y2 / x2)) thr = math.pi / 180 if abs(s1 - s2) > thr: return False if d is not None: dist, _, _, _, _ = distance_segment_to_segment(la, lb, lc, ld) if dist > d: return False return True ================================================ FILE: leuvenmapmatching/util/dist_latlon.py ================================================ # encoding: utf-8 """ leuvenmapmatching.util.dist_latlon ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Based on: https://www.movable-type.co.uk/scripts/latlong.html https://www.movable-type.co.uk/scripts/latlong-vectors.html :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging import math from math import radians, cos, sin, asin, acos, sqrt, atan2, fabs, degrees, ceil, copysign from . import dist_euclidean as diste logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") earth_radius = 6371000 def distance(p1, p2): """Distance between two points. :param p1: (Lat, Lon) :param p2: (Lat, Lon) :return: Distance in meters """ lat1, lon1 = p1[0], p1[1] lat2, lon2 = p2[0], p2[1] lat1, lon1 = radians(lat1), radians(lon1) lat2, lon2 = radians(lat2), radians(lon2) dist = distance_haversine_radians(lat1, lon1, lat2, lon2) return dist def distance_point_to_segment(p, s1, s2, delta=0.0, constrain=True): """Distance between point and segment. Cross-track distance. https://www.movable-type.co.uk/scripts/latlong.html#cross-track :param s1: Segment start point :param s2: Segment end point :param p: Point to measure distance from path to :param delta: Stay away from the endpoints with this factor :return: (Distance in meters, projected location on segment, relative location on segment) """ lat1, lon1 = s1 # Start point lat2, lon2 = s2 # End point lat3, lon3 = p[0], p[1] lat1, lon1 = radians(lat1), radians(lon1) lat2, lon2 = radians(lat2), radians(lon2) lat3, lon3 = radians(lat3), radians(lon3) dist_hs = distance_haversine_radians(lat1, lon1, lat2, lon2) if dist_hs == 0: dist_ct, pi, ti = distance(p, s1), s1, 0 return dist_ct, pi, ti d13 = distance_haversine_radians(lat1, lon1, lat3, lon3) delta13 = d13 / earth_radius b13 = bearing_radians(lat1, lon1, lat3, lon3) b12 = bearing_radians(lat1, lon1, lat2, lon2) dxt = asin(sin(delta13) * sin(b13 - b12)) # b13d12 = (b13 - b12) % (2 * math.pi) # if b13d12 > math.pi: # b13d12 = 2 * math.pi - b13d12 dist_ct = fabs(dxt) * earth_radius # Correct to negative value if point is before segment # sgn = -1 if b13d12 > (math.pi / 2) else 1 sgn = copysign(1, cos(b12 - b13)) dat = sgn * acos(cos(delta13) / abs(cos(dxt))) * earth_radius ti = dat / dist_hs if not constrain: lati, loni = destination_radians(lat1, lon1, b12, dat) elif ti > 1.0: ti = 1.0 lati, loni = lat2, lon2 dist_ct = distance_haversine_radians(lat3, lon3, lati, loni) elif ti < 0.0: ti = 0.0 lati, loni = lat1, lon1 dist_ct = distance_haversine_radians(lat3, lon3, lati, loni) else: lati, loni = destination_radians(lat1, lon1, b12, dat) pi = (degrees(lati), degrees(loni)) return dist_ct, pi, ti def distance_segment_to_segment(f1, f2, t1, t2): """Distance between segments. If no intersection within range, simplified to distance from f2 to [t1,t2]. :param f1: From :param f2: :param t1: To :param t2: :return: (distance, proj on f, proj on t, rel pos on t) """ # Translate lat-lon to x-y and apply the Euclidean function latf1, lonf1 = f1 latf1, lonf1 = radians(latf1), radians(lonf1) f1 = 0, 0 # Origin latf2, lonf2 = f2 latf2, lonf2 = radians(latf2), radians(lonf2) df1f2 = distance_haversine_radians(latf1, lonf1, latf2, lonf2) bf1f2 = bearing_radians(latf1, lonf1, latf2, lonf2) # print(f"bf1f2 = {bf1f2} = {degrees(bf1f2)} degrees") f2 = (df1f2 * cos(bf1f2), df1f2 * sin(bf1f2)) latt1, lont1 = t1[0], t1[1] latt1, lont1 = radians(latt1), radians(lont1) df1t1 = distance_haversine_radians(latf1, lonf1, latt1, lont1) bf1t1 = bearing_radians(latf1, lonf1, latt1, lont1) # print(f"bf1t1 = {bf1t1} = {degrees(bf1t1)} degrees") t1 = (df1t1 * cos(bf1t1), df1t1 * sin(bf1t1)) latt2, lont2 = t2[0], t2[1] latt2, lont2 = radians(latt2), radians(lont2) dt1t2 = distance_haversine_radians(latt1, lont1, latt2, lont2) # print(f"dt1t2 = {dt1t2}") bt1t2 = bearing_radians(latt1, lont1, latt2, lont2) # print(f"bt1t2 = {bt1t2} = {degrees(bt1t2)} degrees") t2 = (t1[0] + dt1t2 * cos(bt1t2), t1[1] + dt1t2 * sin(bt1t2)) d, pf, pt, u_f, u_t = diste.distance_segment_to_segment(f1, f2, t1, t2) pf = destination_radians(latf1, lonf1, bf1f2, u_f * df1f2) pf = (degrees(pf[0]), degrees(pf[1])) pt = destination_radians(latt1, lont1, bt1t2, u_t * dt1t2) pt = (degrees(pt[0]), degrees(pt[1])) return d, pf, pt, u_f, u_t def project(s1, s2, p, delta=0.0): _, pi, ti = distance_point_to_segment(p, s1, s2, delta) return pi, ti def box_around_point(p, dist): lat, lon = p latr, lonr = radians(lat), radians(lon) # diag_dist = sqrt(2 * dist ** 2) diag_dist = dist lat_t, lon_r = destination_radians(latr, lonr, radians(45), diag_dist) lat_b, lon_l = destination_radians(latr, lonr, radians(225), diag_dist) lat_t, lon_r = degrees(lat_t), degrees(lon_r) lat_b, lon_l = degrees(lat_b), degrees(lon_l) return lat_b, lon_l, lat_t, lon_r def interpolate_path(path, dd): """ :param path: (lat, lon) :param dd: Distance difference (meter) :return: """ path_new = [path[0]] for p1, p2 in zip(path, path[1:]): lat1, lon1 = p1[0], p1[1] lat2, lon2 = p2[0], p2[1] lat1, lon1 = radians(lat1), radians(lon1) lat2, lon2 = radians(lat2), radians(lon2) dist = distance_haversine_radians(lat1, lon1, lat2, lon2) if dist > dd: dt = int(ceil(dist / dd)) distd = dist/dt disti = 0 brng = bearing_radians(lat1, lon1, lat2, lon2) for _ in range(dt): disti += distd lati, loni = destination_radians(lat1, lon1, brng, disti) path_new.append((degrees(lati), degrees(loni))) path_new.append(p2) return path_new def bearing_radians(lat1, lon1, lat2, lon2): """Initial bearing""" dlon = lon2 - lon1 y = sin(dlon) * cos(lat2) x = cos(lat1) * sin(lat2) - sin(lat1) * cos(lat2) * cos(dlon) return atan2(y, x) def distance_haversine_radians(lat1, lon1, lat2, lon2, radius=earth_radius): # type: (float, float, float, float, float) -> float lat = lat2 - lat1 lon = lon2 - lon1 a = sin(lat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(lon / 2) ** 2 # dist = 2 * radius * asin(sqrt(a)) dist = 2 * radius * atan2(sqrt(a), sqrt(1 - a)) return dist def destination_radians(lat1, lon1, bearing, dist): d = dist / earth_radius lat2 = asin(sin(lat1) * cos(d) + cos(lat1) * sin(d) * cos(bearing)) lon2 = lon1 + atan2(sin(bearing) * sin(d) * cos(lat1), cos(d) - sin(lat1) * sin(lat2)) return lat2, lon2 def lines_parallel(f1, f2, t1, t2, d=None): latf1, lonf1 = f1 latf1, lonf1 = radians(latf1), radians(lonf1) f1 = 0, 0 # Origin latf2, lonf2 = f2 latf2, lonf2 = radians(latf2), radians(lonf2) df1f2 = distance_haversine_radians(latf1, lonf1, latf2, lonf2) bf1f2 = bearing_radians(latf1, lonf1, latf2, lonf2) # print(f"bf1f2 = {bf1f2} = {degrees(bf1f2)} degrees") f2 = (df1f2 * cos(bf1f2), df1f2 * sin(bf1f2)) latt1, lont1 = t1 latt1, lont1 = radians(latt1), radians(lont1) df1t1 = distance_haversine_radians(latf1, lonf1, latt1, lont1) bf1t1 = bearing_radians(latf1, lonf1, latt1, lont1) # print(f"bf1t1 = {bf1t1} = {degrees(bf1t1)} degrees") t1 = (df1t1 * cos(bf1t1), df1t1 * sin(bf1t1)) latt2, lont2 = t2 latt2, lont2 = radians(latt2), radians(lont2) dt1t2 = distance_haversine_radians(latt1, lont1, latt2, lont2) # print(f"dt1t2 = {dt1t2}") bt1t2 = bearing_radians(latt1, lont1, latt2, lont2) # print(f"bt1t2 = {bt1t2} = {degrees(bt1t2)} degrees") t2 = (t1[0] + dt1t2 * cos(bt1t2), t1[1] + dt1t2 * sin(bt1t2)) return diste.lines_parallel(f1, f2, t1, t2, d=d) ================================================ FILE: leuvenmapmatching/util/dist_latlon_nvector.py ================================================ # encoding: utf-8 """ leuvenmapmatching.util.dist_latlon_nvector ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging import math import numpy as np from nvector._core import unit, n_E2lat_lon, great_circle_normal import nvector as nv frame = nv.FrameE(a=6371e3, f=0) logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") def distance(p1, p2): """ :param p1: :param p2: :return: Distance in meters """ p1 = frame.GeoPoint(p1[0], p1[1], degrees=True) p2 = frame.GeoPoint(p2[0], p2[1], degrees=True) d, _, _ = p1.distance_and_azimuth(p2) # print("distance_latlon({}, {}) -> {}".format(p1, p2, d)) return d def distance_gp(p1, p2): d, _, _ = p1.distance_and_azimuth(p2) return d def distance_point_to_segment(p, s1, s2, delta=0.0): """ TODO: A point exactly on the line gives an error. :param s1: Segment start point :param s2: Segment end point :param p: Point to measure distance from path to :param delta: Stay away from the endpoints with this factor :return: (Distance in meters, projected location on segmegnt) """ # TODO: Initialize all points as GeoPoint when loading data s1 = frame.GeoPoint(s1[0], s1[1], degrees=True) s2 = frame.GeoPoint(s2[0], s2[1], degrees=True) p = frame.GeoPoint(p[0], p[1], degrees=True) p_int, ti = _project_nvector(s1, s2, p) d, _, _ = p.distance_and_azimuth(p_int) return d, (p_int.latitude_deg[0], p_int.longitude_deg[0]), ti def distance_segment_to_segment(f1, f2, t1, t2): """Distance between segments. If no intersection within range, simplified to distance from f2 to [t1,t2]. :param f1: From :param f2: :param t1: To :param t2: :return: (distance, proj on f, proj on t, rel pos on t) """ # TODO: Should be improved f1_gp = frame.GeoPoint(f1[0], f1[1], degrees=True) f2_gp = frame.GeoPoint(f2[0], f2[1], degrees=True) path_f = nv.GeoPath(f1_gp, f2_gp) t1_gp = frame.GeoPoint(t1[0], t1[1], degrees=True) t2_gp = frame.GeoPoint(t2[0], t2[1], degrees=True) path_t = nv.GeoPath(t1_gp, t2_gp) p_int = path_f.intersect(path_t) p_int_gp = p_int.to_geo_point() if path_f.on_path(p_int)[0] and path_t.on_path(p_int)[0]: # Intersection point is on segments, between both begins and ends loc = (p_int_gp.latitude_deg[0], p_int_gp.longitude_deg[0]) u_f = distance_gp(f1_gp, p_int_gp) / distance_gp(f1_gp, f2_gp) u_t = distance_gp(t1_gp, p_int_gp) / distance_gp(t1_gp, t2_gp) return 0, loc, loc, u_f, u_t # No intersection, use last point of map segment (the assumption is the observations are far apart) # TODO: decide which point to use (see distance_segment_to_segment) p_int, u_t = _project_nvector(t1_gp, t2_gp, f2_gp) u_f = 1 d, _, _ = f2_gp.distance_and_azimuth(p_int) return d, (f1, f2), (p_int_gp.latitude_deg[0], p_int_gp.longitude_deg[0]), u_f, u_t def project(s1, s2, p, delta=0.0): s1 = frame.GeoPoint(s1[0], s1[1], degrees=True) s2 = frame.GeoPoint(s2[0], s2[1], degrees=True) p = frame.GeoPoint(p[0], p[1], degrees=True) p_int, ti = _project_nvector(s1, s2, p, delta=delta) return (p_int.latitude_deg[0], p_int.longitude_deg[0]), ti def _project_nvector(s1, s2, p, delta=0.0): path = nv.GeoPath(s1, s2) p_intr = _cross_track_point(path, p) pin = p_intr.to_nvector().normal s1n = s1.to_nvector().normal s2n = s2.to_nvector().normal ti = np.linalg.norm(pin - s1n) / np.linalg.norm(s2n - s1n) ti = max(delta, min(1 - delta, ti)) return path.interpolate(ti).to_geo_point(), ti def _cross_track_point(path, point): """Extend nvector package to find the projection point. The projection point is the closest point on path to the given point. Based on the nvector.cross_track_distance function. http://www.navlab.net/nvector/ :param path: GeoPath :param point: GeoPoint """ c_E = great_circle_normal(*path.nvector_normals()) n_EB_E = point.to_nvector().normal # type: np.array c_EP_E = np.cross(c_E, n_EB_E, axis=0) # Find intersection point C that is closest to point B frame = path.positionA.frame n_EA1_E = path.positionA.to_nvector().normal # should also be ok to use n_EB_C n_EC_E_tmp = unit(np.cross(c_E, c_EP_E, axis=0), norm_zero_vector=np.nan) n_EC_E = np.sign(np.dot(n_EC_E_tmp.T, n_EA1_E)) * n_EC_E_tmp if np.any(np.isnan(n_EC_E)): raise Exception('Paths are Equal. Intersection point undefined. NaN returned.') lat_C, long_C = n_E2lat_lon(n_EC_E, frame.R_Ee) return nv.GeoPoint(lat_C, long_C, frame=frame) def interpolate_path(path, dd): """ TODO: interplate time as third term :param path: (lat, lon) :param dd: Distance difference (meter) :return: """ path_new = [path[0]] for p1, p2 in zip(path, path[1:]): dist = distance(p1, p2) if dist > dd: s1 = frame.GeoPoint(p1[0], p1[1], degrees=True) s2 = frame.GeoPoint(p2[0], p2[1], degrees=True) segment = nv.GeoPath(s1, s2) dt = int(math.floor(dist / dd)) for dti in range(1, dt): p_new = segment.interpolate(dti/dt).to_geo_point() path_new.append((p_new.latitude_deg[0], p_new.longitude_deg[0])) path_new.append(p2) return path_new ================================================ FILE: leuvenmapmatching/util/evaluation.py ================================================ # encoding: utf-8 """ leuvenmapmatching.util.evaluation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Methods to help set up and evaluate experiments. :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging from dtaidistance.alignment import needleman_wunsch, best_alignment from . import dist_latlon MYPY = False if MYPY: from ..map.base import BaseMap from typing import List, Tuple, Optional, Callable logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") def route_mismatch_factor(map_con, path_pred, path_grnd, window=None, dist_fn=None, keep_mismatches=False): # type: (BaseMap, List[int], List[int], Optional[int], Optional[Callable], bool) -> Tuple[float, float, float, float, List[Tuple[int, int]], float, float] """Evaluation method from Newson and Krumm (2009). :math:`f = \frac{d_{-} + d_{+}}{d_0}` With :math:`d_{-}` the length that is erroneously subtracted, :math:`d_{+}` the length that is erroneously added, and :math:`d_0` the distance of the correct route. This function only supports connected states (thus not switching between states that are not connected (e.g. parallel roads). Also computes the Accuracy by Number (AN) and Accuracy by Length (AL) metrics from Zheng et al. (2009). """ if dist_fn is None: dist_fn = dist_latlon.distance _, matrix = needleman_wunsch(path_pred, path_grnd, window=window) print(matrix[:10, :10]) algn, _, _ = best_alignment(matrix) print(algn[:10]) d_plus = 0 # length erroneously added d_min = 0 # length erroneously subtracted d_zero = 0 # length of correct route cnt_matches = 0 # number of perfect matches cnt_mismatches = 0 mismatches = [] if keep_mismatches else None prev_grnd_pi = None for pred_pi, grnd_pi in algn: pred_p = path_pred[pred_pi] grnd_p = path_grnd[grnd_pi] grnd_d = map_con.path_dist(grnd_p) d_zero += grnd_d if pred_p == grnd_p: cnt_matches += 1 else: # print(f"Mismatch: {pred_p} != {grnd_p}") cnt_mismatches += 1 pred_d = map_con.path_dist(pred_p) d_plus += pred_d d_min += grnd_d if keep_mismatches: mismatches.append((pred_p, grnd_p)) prev_grnd_pi = grnd_pi factor = (d_min + d_plus) / d_zero an = cnt_matches / len(path_grnd) al = (d_zero - d_min) / d_zero return factor, cnt_matches, cnt_mismatches, d_zero, mismatches, an, al ================================================ FILE: leuvenmapmatching/util/gpx.py ================================================ # encoding: utf-8 """ leuvenmapmatching.util.gpx ~~~~~~~~~~~~~~~~~~~~~~~~~~ Some additional functions to interact with the gpx library. :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging import gpxpy import gpxpy.gpx logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") def gpx_to_path(gpx_file): gpx_fh = open(gpx_file) track = None try: gpx = gpxpy.parse(gpx_fh) if len(gpx.tracks) == 0: logger.error('No tracks found in GPX file ( tag missing?): {}'.format(gpx_file)) return None logger.info("Read gpx file: {} points, {} tracks, {} segments".format( gpx.get_points_no(), len(gpx.tracks), len(gpx.tracks[0].segments))) track = [(p.latitude, p.longitude, p.time) for p in gpx.tracks[0].segments[0].points] finally: gpx_fh.close() return track def path_to_gpx(path, filename=None): gpx = gpxpy.gpx.GPX() # Create first track in our GPX: gpx_track = gpxpy.gpx.GPXTrack() gpx.tracks.append(gpx_track) # Create first segment in our GPX track: gpx_segment = gpxpy.gpx.GPXTrackSegment() gpx_track.segments.append(gpx_segment) gpx_segment.points = [(gpxpy.gpx.GPXTrackPoint(lat, lon, time=time)) for (lat, lon, time) in path] if filename: with open(filename, 'w') as gpx_fh: gpx_fh.write(gpx.to_xml()) return gpx ================================================ FILE: leuvenmapmatching/util/kalman.py ================================================ # encoding: utf-8 """ leuvenmapmatching.util.kalman ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging from pykalman import KalmanFilter import numpy as np logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") def smooth_path(path, dt=1, obs_noise=1e-4, loc_var=1e-4, vel_var=1e-6, kf=None, rm_outliers=False, use_euclidean=True, n_iter=1000): """Apply Kalman filtering. Assumes data with a constant sample rate. Inspired by https://github.com/FlorianWilhelm/gps_data_with_python :param path: :param dt: Sample interval in seconds :param obs_noise: Observation noise (default=1e-4, approx 10-30m) :param loc_var: estimated location variance :param vel_var: estimated velocity variance :param kf: Trained Kalman filter :param rm_outliers: Remove outliers based on Kalman prediction True or 1 will be removal, 2 will also retrain after removal :param use_euclidean: :param n_iter: Kalman iterations :return: """ path = np.array(path) if kf is None: # state is (x, y, v_x, v_y) F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]]) # observations is (x, y) H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]]) R = np.diag([obs_noise, obs_noise]) ** 2 initial_state_mean = np.hstack([path[0, :2], 2 * [0.]]) initial_state_covariance = np.diag([loc_var, loc_var, vel_var, vel_var]) ** 2 kf = KalmanFilter(transition_matrices=F, observation_matrices=H, observation_covariance=R, initial_state_mean=initial_state_mean, initial_state_covariance=initial_state_covariance, em_vars=['transition_covariance']) if n_iter > 0: logger.debug("Start learning") kf = kf.em(path[:, :2], n_iter=n_iter) state_means, state_vars = kf.smooth(path[:, :2]) if use_euclidean: from .dist_euclidean import distance distance_f = distance else: from .dist_latlon import distance distance_f = distance if rm_outliers: path_ma = np.ma.asarray(path[:, :2]) for idx in range(path.shape[0]): d = distance_f(path[idx, :2], state_means[idx, :2]) if d > obs_noise * 2: logger.debug("Rm point {}".format(idx)) path_ma[idx] = np.ma.masked if rm_outliers == 2: logger.debug("Retrain") kf = kf.em(path_ma, n_iter=n_iter) state_means, state_vars = kf.smooth(path_ma) return state_means, state_vars, kf ================================================ FILE: leuvenmapmatching/util/openstreetmap.py ================================================ # encoding: utf-8 """ leuvenmapmatching.util.openstreetmap ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging from pathlib import Path import requests import tempfile import osmread logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") def locations_to_map(locations, map_con, filename=None): lats, lons = zip(*locations) lon_min, lon_max = min(lons), max(lons) lat_min, lat_max = min(lats), max(lats) bb = [lon_min, lat_min, lon_max, lat_max] return bb_to_map(bb, map_con, filename) def bb_to_map(bb, map_con, filename=None): """Download map from overpass-api.de. :param bb: [lon_min, lat_min, lon_max, lat_max] :param map: :param filename: :return: """ if filename: xml_file = Path(filename) else: xml_file = Path(tempfile.gettempdir()) / "osm.xml" if not xml_file.exists(): bb_str = ",".join(str(coord) for coord in bb) url = 'http://overpass-api.de/api/map?bbox='+bb_str logger.debug("Downloading {} from {} ...".format(xml_file, url)) r = requests.get(url, stream=True) with xml_file.open('wb') as ofile: for chunk in r.iter_content(chunk_size=1024): if chunk: ofile.write(chunk) logger.debug("... done") else: logger.debug("Reusing existing file: {}".format(xml_file)) return file_to_map(xml_file, map_con) def file_to_map(filename, map_con): logger.debug("Parse OSM file ...") for entity in osmread.parse_file(str(filename)): if isinstance(entity, osmread.Way) and 'highway' in entity.tags: for node_a, node_b in zip(entity.nodes, entity.nodes[1:]): map_con.add_edge(node_a, node_b) # Some roads are one-way. We'll add both directions. map_con.add_edge(node_b, node_a) if isinstance(entity, osmread.Node): map_con.add_node(entity.id, (entity.lat, entity.lon)) logger.debug("... done") logger.debug("Purging database ...") map_con.purge() logger.debug("... done") def download_map_xml(fn, bbox, force=False, verbose=False): """Download map from overpass-api.de based on a given bbox :param fn: Filename where to store the map as xml :param bbox: String or array with [lon_min, lat_min, lon_max, lat_max] :param force: Also download if file already exists :param verbose: Verbose output :return: """ fn = Path(fn) if type(bbox) is list: bb_str = ",".join(str(coord) for coord in bbox) elif type(bbox) is str: bb_str = bbox else: raise AttributeError('Unknown type for bbox: {}'.format(type(bbox))) if force or not fn.exists(): if verbose: print("Downloading {}".format(fn)) import requests url = f'http://overpass-api.de/api/map?bbox={bb_str}' r = requests.get(url, stream=True) with fn.open('wb') as ofile: for chunk in r.iter_content(chunk_size=1024): if chunk: ofile.write(chunk) else: if verbose: print("File already exists") def create_map_from_xml(fn, include_footways=False, include_parking=False, use_rtree=False, index_edges=False): """Create an InMemMap from an OpenStreetMap XML file. Used for testing routes on OpenStreetMap. """ from ..map.inmem import InMemMap map_con = InMemMap("map", use_latlon=True, use_rtree=use_rtree, index_edges=index_edges) cnt = 0 ways_filter = ['bridleway', 'bus_guideway', 'track'] if not include_footways: ways_filter += ['footway', 'cycleway', 'path'] parking_filter = ['driveway'] if not include_parking: parking_filter += ['parking_aisle'] for entity in osmread.parse_file(str(fn)): if isinstance(entity, osmread.Way): tags = entity.tags if 'highway' in tags \ and not (tags['highway'] in ways_filter) \ and not ('access' in tags and tags['access'] == 'private') \ and not ('landuse' in tags and tags['landuse'] == 'square') \ and not ('amenity' in tags and tags['amenity'] == 'parking') \ and not ('service' in tags and tags['service'] in parking_filter) \ and not ('area' in tags and tags['area'] == 'yes'): for node_a, node_b in zip(entity.nodes, entity.nodes[1:]): map_con.add_edge(node_a, node_b) # Some roads are one-way. We'll add both directions. map_con.add_edge(node_b, node_a) if isinstance(entity, osmread.Node): map_con.add_node(entity.id, (entity.lat, entity.lon)) map_con.purge() return map_con ================================================ FILE: leuvenmapmatching/util/projections.py ================================================ # encoding: utf-8 """ leuvenmapmatching.util.projections ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import math import logging import pyproj logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") def latlon2equirectangular(lat, lon, phi_er=0, lambda_er=0): """Naive equirectangular projection. This is the same as considering (lat,lon) == (y,x). This is a lot faster but only works if you are far enough from the poles and the dateline. :param lat: :param lon: :param phi_er: The standard parallels (north and south of the equator) where the scale of the projection is true :param lambda_er: The central meridian of the map """ x = (lon - lambda_er) * math.cos(phi_er) y = lat - phi_er return y, x def equirectangular2latlon(y, x, phi_er=0, lambda_er=0): """Naive equirectangular projection. This is the same as considering (lat,lon) == (y,x). This is a lot faster but only works if you are far enough from the poles and the dateline. :param phi_er: The standard parallels (north and south of the equator) where the scale of the projection is true :param lambda_er: The central meridian of the map """ lon = x / math.cos(phi_er) + lambda_er lat = y + phi_er return lat, lon def latlon2grs80(coordinates, lon_0=0.0, lat_ts=0.0, y_0=0, x_0=0.0, zone=31, **kwargs): """Given a list of (lon, lat) coordinates, create x-y coordinates in meter. :param coordinates: A list of lon-lat tuples :param lon_0: Longitude of projection center. :param lat_ts: Latitude of true scale. Defines the latitude where scale is not distorted. :param y_0: False northing :param x_0: False easting :param zone: UTM zone to use for projection (Defaults to 31) """ if zone is None: # https://en.wikipedia.org/wiki/Universal_Transverse_Mercator_coordinate_system zone = 31 other_options = " ".join(f"+{key}={val}" for key, val in kwargs.items()) proj = pyproj.Proj(f"+proj=utm +zone={zone} +ellps=GRS80 +units=m " f"+lon_0={lon_0} +lat_ts={lat_ts} +y_0={y_0} +x_0={x_0} " f"+no_defs {other_options}") for lon, lat in coordinates: x, y = proj(lon, lat) yield x, y ================================================ FILE: leuvenmapmatching/util/segment.py ================================================ # encoding: utf-8 """ leuvenmapmatching.util.segment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import logging logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") class Segment(object): """Segment in the graph and its interpolated point.""" __slots__ = ["l1", "p1", "l2", "p2", "_pi", "_ti"] def __init__(self, l1, p1, l2=None, p2=None, pi=None, ti=None): """Create a new segment. :param l1: Label of the node that is the start of the segment. :param p1: Point (coordinate) of the start node. :param l2: Label of the node that is the end of the segment. :param p2: Point (coordinate) of the end node. :param pi: Interpolated point. The point that is the best match and can be in between p1 and p2. :param ti: Position of pi on the segment [p1,p2], thus pi = p1+t1*(p2-p1). """ self.l1 = l1 # Start of segment, label self.p1 = p1 # point self.l2 = l2 # End of segment, if None the segment is a point self.p2 = p2 self.pi = pi # Interpolated point self.ti = ti # Position on segment p1-p2 @property def label(self): if self.l2 is None: return self.l1 return f"{self.l1}-{self.l2}" @property def rlabel(self): if self.l2 is None: return self.l1 return f"{self.l2}-{self.l1}" @property def key(self): if self.l2 is None: return self.l1 return f"{self.l1}-{self.l2}" @property def pi(self): if self.p2 is None: return self.p1 return self._pi @pi.setter def pi(self, value): if value is not None and len(value) > 2: self._pi = tuple(value[:2]) else: self._pi = value @property def ti(self): if self.p2 is None: return 0 return self._ti @ti.setter def ti(self, value): self._ti = value def is_point(self): return self.p2 is None def last_point(self): if self.p2 is None: return self.p1 return self.p2 def loc_to_str(self): if self.p2 is None: return f"{self.p1}" if self._pi is not None: return f"{self.p1}-{self.pi}/{self.ti}-{self.p2}" return f"{self.p1}-{self.p2}" def __str__(self): if self.p2 is None: return f"{self.l1}" if self._pi is not None: return f"{self.l1}-i-{self.l2}" return f"{self.l1}-{self.l2}" def __repr__(self): return "Segment<" + self.__str__() + ">" ================================================ FILE: leuvenmapmatching/visualization.py ================================================ # encoding: utf-8 """ leuvenmapmatching.visualization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import math import random import logging from itertools import islice import numpy as np import matplotlib.pyplot as plt from matplotlib import colors as mcolors import smopy logger = logging.getLogger("be.kuleuven.cs.dtai.mapmatching") graph_color = mcolors.CSS4_COLORS['darkmagenta'] match_color = mcolors.CSS4_COLORS['green'] match_ne_color = mcolors.CSS4_COLORS['olive'] lattice_color = mcolors.CSS4_COLORS['magenta'] nodes_color = mcolors.CSS4_COLORS['cyan'] path_color = mcolors.CSS4_COLORS['blue'] fontsize = 11 # 7 # 11 def plot_map(map_con, path=None, nodes=None, counts=None, ax=None, use_osm=False, z=None, bb=None, show_labels=False, matcher=None, show_graph=False, zoom_path=False, show_lattice=False, show_matching=False, filename=None, linewidth=2, coord_trans=None, figwidth=20, lattice_nodes=None): """Plot the db/graph and optionally include the observed path and inferred nodes. :param map_con: Map :param path: list[(lat, lon)] :param nodes: list[str] :param counts: Number of visits of a node in the lattice. dict[label, int] :param ax: Matplotlib axis :param use_osm: Use OpenStreetMap layer, the points should be latitude-longitude pairs. :param matcher: Matcher object (overrules given path, nodes and counts) :param filename: File to write image to :param show_graph: Plot the vertices and edges in the graph :return: None """ if matcher is not None: path = matcher.path counts = matcher.node_counts() nodes = None if lattice_nodes is None: lat_nodes = matcher.lattice_best else: lat_nodes = lattice_nodes if lat_nodes is None: lat_nodes = [] else: lat_nodes = [] if not bb: bb = map_con.bb() lat_min, lon_min, lat_max, lon_max = bb if path: plat, plon = islice(zip(*path), 2) lat_min, lat_max = min(lat_min, min(plat)), max(lat_max, max(plat)) lon_min, lon_max = min(lon_min, min(plon)), max(lon_max, max(plon)) bb = [lat_min, lon_min, lat_max, lon_max] logger.debug("bb = [{}, {}, {}, {}]".format(*bb)) if zoom_path and path: if type(zoom_path) is slice: plat, plon = islice(zip(*path[zoom_path]), 2) lat_min, lat_max = min(plat), max(plat) lon_min, lon_max = min(plon), max(plon) else: plat, plon = islice(zip(*path), 2) lat_min, lat_max = min(plat), max(plat) lon_min, lon_max = min(plon), max(plon) lat_d = lat_max - lat_min lon_d = lon_max - lon_min latlon_d = max(lat_d, lon_d) lat_max += max(latlon_d * 0.01, lat_d * 0.2) lon_min -= max(latlon_d * 0.01, lon_d * 0.2) lat_min -= max(latlon_d * 0.01, lat_d * 0.2) lon_max += max(latlon_d * 0.01, lon_d * 0.2) logger.debug("Setting bounding box to path") bb = [lat_min, lon_min, lat_max, lon_max] logger.debug("bb(zoom-path) = [{}, {}, {}, {}]".format(*bb)) bb_o = bb if coord_trans: logger.debug("Converting bounding box coordinates") if path: path = [coord_trans(lat, lon) for lat, lon in path] lat_min, lon_min, lat_max, lon_max = bb lat_min, lon_min = coord_trans(lat_min, lon_min) lat_max, lon_max = coord_trans(lat_max, lon_max) bb = [lat_min, lon_min, lat_max, lon_max] logger.debug("bb = [{}, {}, {}, {}]".format(*bb)) if use_osm: from .util import dist_latlon project = dist_latlon.project if z is None: z = 18 m = smopy.Map(bb, z=z, ax=ax) to_pixels = m.to_pixels x_max, y_max = to_pixels(lat_max, lon_max) x_min, y_min = to_pixels(lat_min, lon_min) height = figwidth / abs(x_max - x_min) * abs(y_max - y_min) if ax is None: ax = m.show_mpl(figsize=(figwidth, height)) else: ax = m.show_mpl(ax=ax) fig = None else: from .util import dist_euclidean project = dist_euclidean.project def to_pixels(lat, lon=None): if lon is None: lat, lon = lat[0], lat[1] return lon, lat x_max, y_max = to_pixels(lat_max, lon_max) x_min, y_min = to_pixels(lat_min, lon_min) height = figwidth / abs(lon_max - lon_min) * abs(lat_max - lat_min) if ax is None: fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(figwidth, height)) else: fig = None ax.set_xlim([x_min, x_max]) ax.set_ylim([y_min, y_max]) # if counts is None: # node_sizes = [10] * map_con.size() # else: # node_sizes = [counts[label]*100+5 for label in map_con.labels()] if show_graph: logger.debug('Plot vertices ...') cnt = 0 for key, coord in map_con.all_nodes(bb=bb_o): if coord_trans: coord = coord_trans(*coord) coord = to_pixels(coord) plt.plot(coord[0], coord[1], marker='o', markersize=2*linewidth, color=graph_color, alpha=0.75) if show_labels: key = str(key) if type(show_labels) is int: key = key[-show_labels:] xytext = ax.transLimits.transform(coord) xytext = (xytext[0]+0.001, xytext[1]+0.0) xytext = ax.transLimits.inverted().transform(xytext) # key = str(key)[-3:] # print(f'annotate: {key} {coord} {xytext}') ann = ax.annotate(key, xy=coord, xytext=xytext, # textcoords=('axes fraction', 'axes fraction'), # arrowprops=dict(arrowstyle='->'), color=graph_color, fontsize=fontsize) # ann.set_rotation(45) cnt += 1 logger.debug(f'... done, {cnt} nodes') logger.debug('Plot lines ...') cnt = 0 for row in map_con.all_edges(bb=bb_o): loc_a = row[1] loc_b = row[3] if coord_trans: loc_a = coord_trans(*loc_a) loc_b = coord_trans(*loc_b) x_a, y_a = to_pixels(*loc_a) x_b, y_b = to_pixels(*loc_b) ax.plot([x_a, x_b], [y_a, y_b], color=graph_color, linewidth=linewidth, markersize=linewidth) cnt += 1 logger.debug(f'... done, {cnt} edges') if show_lattice: if matcher is None: logger.warning("Matcher needs to be passed to show lattice. Not showing lattice.") plot_lattice(ax, to_pixels, matcher) if path: logger.debug('Plot path ...') if type(zoom_path) is slice: path_startidx = zoom_path.start path_slice = path[zoom_path] else: path_startidx = 0 path_slice = path px, py = zip(*[to_pixels(p[:2]) for p in path_slice]) ax.plot(px, py, linewidth=linewidth, markersize=linewidth * 2, alpha=0.75, linestyle="--", marker='o', color=path_color) if show_labels: for li, (lx, ly) in enumerate(zip(px, py)): # ax.text(lx, ly, f"O{li}", color=path_color) ann = ax.annotate(f"O{path_startidx + li}", xy=(lx, ly), color=path_color, fontsize=fontsize) ann.set_rotation(45) if nodes or matcher: logger.debug('Plot nodes ...') xs, ys, ls = [], [], [] prev = None node_locs = [] if nodes: for node in nodes: if type(node) == tuple: node = node[0] lat, lon = map_con.node_coordinates(node) node_locs.append((lat, lon, node)) elif lat_nodes is not None: prev_m = None for m in lat_nodes: if prev_m is not None and prev_m.edge_m.l2 == m.edge_m.l1 \ and prev_m.edge_m.l1 != m.edge_m.l2: lat, lon = m.edge_m.p1 node_locs.append((lat, lon, m.edge_m.l1)) lat, lon = m.edge_m.pi node_locs.append((lat, lon, m.edge_m.label)) prev_m = m for lat, lon, label in node_locs: if coord_trans: lat, lon = coord_trans(lat, lon) if bb[0] <= lat <= bb[2] and bb[1] <= lon <= bb[3]: if prev is not None: x, y = to_pixels(*prev) xs.append(x) ys.append(y) ls.append(label) prev = None x, y = to_pixels(lat, lon) xs.append(x) ys.append(y) ls.append(label) else: if prev is None: x, y = to_pixels(lat, lon) xs.append(x) ys.append(y) ls.append(label) prev = lat, lon ax.plot(xs, ys, 'o-', linewidth=linewidth * 3, markersize=linewidth * 3, alpha=0.75, color=nodes_color) # if show_labels: # for label, lx, ly in zip(ls, xs, ys): # ax.annotate(label, xy=(lx, ly), xytext=(lx - 30, ly), color=nodes_color) if matcher and show_matching: logger.debug('Plot matching path-nodes (using matcher) ...') for idx, m in enumerate(lat_nodes): lat, lon = m.edge_m.pi[:2] lat2, lon2 = m.edge_o.pi[:2] if coord_trans: lat, lon = coord_trans(lat, lon) lat2, lon2 = coord_trans(lat2, lon2) x, y = to_pixels(lat, lon) x2, y2 = to_pixels(lat2, lon2) if m.edge_o.is_point(): plt.plot(x, y, marker='x', markersize=2 * linewidth, color=match_color, alpha=0.75) plt.plot(x2, y2, marker='+', markersize=2 * linewidth, color=match_color, alpha=0.75) ax.plot((x, x2), (y, y2), '-', color=match_color, linewidth=linewidth, alpha=0.75) else: plt.plot(x, y, marker='x', markersize=2 * linewidth, color=match_ne_color, alpha=0.75) plt.plot(x2, y2, marker='+', markersize=2 * linewidth, color=match_ne_color, alpha=0.75) ax.plot((x, x2), (y, y2), '-', color=match_ne_color, linewidth=linewidth, alpha=0.75) # ax.plot((x, x2), (y, y2), '-', color=match_color, linewidth=10, alpha=0.1) # if show_labels: # ax.annotate(f"{m.obs}.{m.obs_ne}", xy=(x, y)) elif path and nodes and len(path) == len(nodes) and show_matching: logger.debug('Plot matching path-nodes (using sequence of nodes) ...') for idx, (loc, node) in enumerate(zip(path, nodes)): x, y = to_pixels(*loc) if type(node) == tuple and (len(node) == 4 or len(node) == 2): latlon2, latlon3 = map_con.node_coordinates(node[0]), map_con.node_coordinates(node[1]) if coord_trans: latlon2 = coord_trans(*latlon2) latlon3 = coord_trans(*latlon3) latlon4, _ = project(latlon2, latlon3, loc) x4, y4 = to_pixels(*latlon4) ax.plot((x, x4), (y, y4), '-', color=match_color, linewidth=linewidth, alpha=0.75) elif type(node) == tuple and len(node) == 3: lat2, lon2 = map_con.node_coordinates(node[0]) if coord_trans: lat2, lon2 = coord_trans(lat2, lon2) x2, y2 = to_pixels(lat2, lon2) ax.plot((x, x2), (y, y2), '-', color=match_color, linewidth=linewidth, alpha=0.75) elif type(node) == str or type(node) == int: lat2, lon2 = map_con.node_coordinates(node[0]) if coord_trans: lat2, lon2 = coord_trans(lat2, lon2) x2, y2 = to_pixels(lat2, lon2) ax.plot((x, x2), (y, y2), '-', color=match_color, linewidth=linewidth, alpha=0.75) else: raise Exception('Unknown node type: {} ({})'.format(node, type(node))) # if show_labels: # ax.annotate(str(idx), xy=(x, y)) if map_con.use_latlon: ax.set_xlabel('Longitude') ax.set_ylabel('Latitude') else: ax.set_xlabel('X') ax.set_ylabel('Y') ax.axis('equal') ax.set_aspect('equal') if filename is not None: plt.savefig(filename) if fig is not None: plt.close(fig) fig = None ax = None return fig, ax def plot_lattice(ax, to_pixels, matcher): for idx in range(len(matcher.lattice)): if len(matcher.lattice[idx]) == 0: continue for m in matcher.lattice[idx].values_all(): for mp in m.prev: if m.stop: alpha = 0.1 linewidth = 1 else: alpha = 0.3 linewidth = 3 if mp.edge_m.p2 is None: prv = mp.edge_m.p1 else: prv = mp.edge_m.p2 nxt = m.edge_m.p1 x1, y1 = to_pixels(*prv) x2, y2 = to_pixels(*nxt) ax.plot((x1, x2), (y1, y2), '.-', color=lattice_color, linewidth=linewidth, alpha=alpha) if m.edge_m.p2 is not None: x1, y1 = to_pixels(*m.edge_m.p1) x2, y2 = to_pixels(*m.edge_m.p2) ax.plot((x1, x2), (y1, y2), '.-', color=lattice_color, linewidth=linewidth, alpha=alpha) def plot_obs_noise_dist(obs_fn, obs_noise, min_dist=0, max_dist=10): """Plot the expected noise of an observation distribution. :param matcher: Matcher :return: """ x = np.linspace(min_dist, max_dist, 100) y = [obs_fn(xi) for xi in x] plt.plot(x, y) plt.xlabel("Distance") plt.ylabel("Probability") plt.xlim((min_dist, max_dist)) plt.ylim((0, 1)) plt.axvline(x=obs_noise, color='red', alpha=0.7) plt.annotate("Observation noise stddev", xy=(obs_noise, 0)) ================================================ FILE: setup.cfg ================================================ [metadata] name = leuvenmapmatching version = attr: leuvenmapmatching.__version__ author = Wannes Meert description = Match a trace of GPS positions to a locations and streets on a map license = Apache 2.0 long_description = file: README.md long_description_content_type = text/markdown url = https://github.com/wannesm/LeuvenMapMatching project_urls = Bug Tracker = https://github.com/wannesm/LeuvenMapMatching/issues classifiers = Programming Language :: Python :: 3 License :: OSI Approved :: Apache Software License Operating System :: OS Independent keywords = map, matching [options] packages = find: python_requires = >=3.6 install_requires = numpy scipy tests_requires = pytest-runner pytest [options.extras_require] vis = smopy; matplotlib>=2.0.0 db = rtree; pyproj all = requests; smopy; matplotlib>=2.0.0; rtree; pyproj; nvector==0.5.2; gpxpy; pykalman; pytest; pytest-runner; osmread; osmnx # In case of problems with osmread, use: "osmread @ git+https://github.com/dezhin/osmread" [aliases] test=pytest [tool:pytest] norecursedirs = .git venv* .eggs addopts = --verbose python_files = tests/*.py tests/*/*.py ================================================ FILE: setup.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 """ setup.py ~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2017-2021 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ from setuptools import setup, find_packages import re import os here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join('leuvenmapmatching', '__init__.py'), 'r') as fd: version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) if not version: raise RuntimeError('Cannot find version information') setup( name='leuvenmapmatching', version=version, packages=find_packages(), author='Wannes Meert', author_email='wannes.meert@cs.kuleuven.be', url='https://github.com/wannesm/LeuvenMapMatching', description='Match a trace of GPS positions to a locations and streets on a map', python_requires='>=3.6', license='Apache 2.0', classifiers=[ 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 3' ], keywords='map matching', ) ================================================ FILE: tests/examples/example_1_simple.py ================================================ from leuvenmapmatching.matcher.distance import DistanceMatcher from leuvenmapmatching.map.inmem import InMemMap map_con = InMemMap("mymap", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "F", "E", "K", "L"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.3, 3.5), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] matcher = DistanceMatcher(map_con, max_dist=2, obs_noise=1, min_prob_norm=0.5, max_lattice_width=5) states, _ = matcher.match(path) nodes = matcher.path_pred_onlynodes print("States\n------") print(states) print("Nodes\n------") print(nodes) print("") matcher.print_lattice_stats() ================================================ FILE: tests/examples/example_using_osmnx_and_geopandas.py ================================================ import os import sys import logging from pathlib import Path this_path = Path(os.path.realpath(__file__)).parent.parent / "rsrc" / "path_latlon" assert(this_path.exists()) path_to_mytrack_gpx = this_path / "route.gpx" assert(path_to_mytrack_gpx.exists()) import leuvenmapmatching as mm from leuvenmapmatching.map.inmem import InMemMap def run(): # Start example import osmnx as ox # Select map (all, drive, walk, ...) graph = ox.graph_from_place('Leuven, Belgium', network_type='all', simplify=False) graph_proj = ox.project_graph(graph) # Create GeoDataFrames # Approach 1: translate map to graph # DistanceMatcher uses edges, thus build index based on edges map_con = InMemMap("myosm", use_latlon=True, use_rtree=True, index_edges=True) nodes_proj, edges_proj = ox.graph_to_gdfs(graph_proj, nodes=True, edges=True) for nid, row in nodes_proj[['x', 'y']].iterrows(): map_con.add_node(nid, (row['x'], row['y'])) for eid, _ in edges_proj.iterrows(): map_con.add_edge(eid[0], eid[1]) # Approach 2: use a specific projection map_con = InMemMap("myosm", use_latlon=True, use_rtree=True, index_edges=True) nodes_proj, edges_proj = ox.graph_to_gdfs(graph_proj, nodes=True, edges=True) nodes_proj = nodes_proj.to_crs("EPSG:3395") # edges_proj = edges_proj.to_crs("EPSG:3395") for nid, row in nodes_proj.iterrows(): map_con.add_node(nid, (row['lat'], row['lon'])) # We can also extract edges also directly from networkx graph for nid1, nid2, _ in graph.edges: map_con.add_edge(nid1, nid2) # Perform matching from leuvenmapmatching.util.gpx import gpx_to_path from leuvenmapmatching.matcher.distance import DistanceMatcher track = gpx_to_path(path_to_mytrack_gpx) matcher = DistanceMatcher(map_con, max_dist=100, max_dist_init=50, # meter non_emitting_length_factor=0.75, obs_noise=50, obs_noise_ne=75, # meter dist_noise=50, # meter non_emitting_states=True, max_lattice_width=5) states, lastidx = matcher.match(track) print(states) # End example # import leuvenmapmatching.visualization as mm_viz # import matplotlib as mpl # mpl.use('MacOSX') # mm_viz.plot_map(map_con, matcher=matcher, use_osm=True, # zoom_path=True, show_graph=True, # filename=Path(os.environ.get('TESTDIR', Path(__file__).parent)) / "example.png") if __name__ == "__main__": mm.logger.setLevel(logging.INFO) mm.logger.addHandler(logging.StreamHandler(sys.stdout)) run() ================================================ FILE: tests/rsrc/bug2/readme.md ================================================ Test data ========= Download from https://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/leuvenmapmatching_testdata.zip - `edgesrl.csv` - `nodesrl.csv` - `path.csv` ================================================ FILE: tests/rsrc/newson_krumm_2009/readme.md ================================================ Newson Krum testdata ==================== Files will be downloaded from https://www.microsoft.com/en-us/research/publication/hidden-markov-map-matching-noise-sparseness/ - `gps_data.txt` - `road_network.txt` - `ground_truth_route.txt` ================================================ FILE: tests/rsrc/path_latlon/readme.md ================================================ Test data for path_latlon ========================= Download from https://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/leuvenmapmatching_testdata2.zip - `route.gpx` - `route2.gpx` - `osm_downloaded.xml` - `osm_downloaded2.xml` ================================================ FILE: tests/rsrc/path_latlon/route.gpx ================================================ START Leuven Stadswandeling 5 km TR 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 ================================================ FILE: tests/rsrc/path_latlon/route2.gpx ================================================ START Test route 1 2 3 4 5 6 7 8 ================================================ FILE: tests/test_bugs.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 """ tests.test_bugs ~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import os import sys import logging from pathlib import Path import csv import leuvenmapmatching as mm from leuvenmapmatching.map.inmem import InMemMap from leuvenmapmatching.map.sqlite import SqliteMap from leuvenmapmatching.matcher.simple import SimpleMatcher from leuvenmapmatching.matcher.distance import DistanceMatcher import leuvenmapmatching.visualization as mm_viz MYPY = False if MYPY: from typing import List, Tuple logger = mm.logger directory = None def test_bug1(): dist = 10 nb_steps = 20 map_con = InMemMap("map", graph={ "A": ((1, dist), ["B"]), "B": ((2, dist), ["A", "C", "CC"]), "C": ((3, 0), ["B", "D"]), "D": ((4 + dist, 0), ["C", "E"]), "CC": ((3, 2 * dist), ["B", "DD"]), "DD": ((4 + dist, 2 * dist), ["CC", "E"]), "E": ((5 + dist, dist), ["F", "D", "DD"]), "F": ((6 + dist, dist), ["E", ]), }, use_latlon=False) i = 10 path = [(1.1, 2*dist*i/nb_steps), (2.1, 2*dist*i/nb_steps), (5.1+dist, 2*dist*i/nb_steps), (6.1+dist, 2*dist*i/nb_steps) # (1, len*i/nb_steps), # (2, len*i/nb_steps), # (3, len*i/nb_steps) ] matcher = SimpleMatcher(map_con, max_dist=dist + 1, obs_noise=dist + 1, min_prob_norm=None, non_emitting_states=True) nodes = matcher.match(path, unique=False) print("Solution: ", nodes) if directory: import leuvenmapmatching.visualization as mm_vis matcher.print_lattice() matcher.print_lattice_stats() mm_vis.plot_map(map_con, path=path, nodes=nodes, counts=matcher.node_counts(), show_labels=True, filename=str(directory / "test_bugs_1.png")) def test_bug2(): this_path = Path(os.path.realpath(__file__)).parent / "rsrc" / "bug2" edges_fn = this_path / "edgesrl.csv" nodes_fn = this_path / "nodesrl.csv" path_fn = this_path / "path.csv" zip_fn = this_path / "leuvenmapmatching_testdata.zip" if not (edges_fn.exists() and nodes_fn.exists() and path_fn.exists()): import requests url = 'https://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/leuvenmapmatching_testdata.zip' logger.debug("Download testfiles from kuleuven.be") r = requests.get(url, stream=True) with zip_fn.open('wb') as ofile: for chunk in r.iter_content(chunk_size=1024): if chunk: ofile.write(chunk) import zipfile logger.debug("Unzipping leuvenmapmatching_testdata.zip") with zipfile.ZipFile(str(zip_fn), "r") as zip_ref: zip_ref.extractall(str(zip_fn.parent)) logger.debug(f"Reading map ...") mmap = SqliteMap("road_network", use_latlon=True, dir=this_path) path = [] with path_fn.open("r") as path_f: reader = csv.reader(path_f, delimiter=',') for row in reader: lat, lon = [float(coord) for coord in row] path.append((lat, lon)) node_cnt = 0 with nodes_fn.open("r") as nodes_f: reader = csv.reader(nodes_f, delimiter=',') for row in reader: nid, lonlat, _ = row nid = int(nid) lon, lat = [float(coord) for coord in lonlat[1:-1].split(",")] mmap.add_node(nid, (lat, lon), ignore_doubles=True, no_index=True, no_commit=True) node_cnt += 1 edge_cnt = 0 with edges_fn.open("r") as edges_f: reader = csv.reader(edges_f, delimiter=',') for row in reader: _eid, nid1, nid2, pid = [int(val) for val in row] mmap.add_edge(nid1, nid2, edge_type=0, path=pid, no_index=True, no_commit=True) edge_cnt += 1 logger.debug(f"... done: {node_cnt} nodes and {edge_cnt} edges") logger.debug("Indexing ...") mmap.reindex_nodes() mmap.reindex_edges() logger.debug("... done") matcher = DistanceMatcher(mmap, min_prob_norm=0.001, max_dist=200, obs_noise=4.07, non_emitting_states=True) # path = path[:2] nodes, idx = matcher.match(path, unique=True) path_pred = matcher.path_pred if directory: import matplotlib.pyplot as plt matcher.print_lattice_stats() logger.debug("Plotting post map ...") fig = plt.figure(figsize=(100, 100)) ax = fig.get_axes() mm_viz.plot_map(mmap, matcher=matcher, use_osm=True, ax=ax, show_lattice=False, show_labels=True, show_graph=False, zoom_path=True, show_matching=True) plt.savefig(str(directory / "test_bug1.png")) plt.close(fig) logger.debug("... done") if __name__ == "__main__": mm.logger.setLevel(logging.DEBUG) mm.logger.addHandler(logging.StreamHandler(sys.stdout)) directory = Path(os.environ.get('TESTDIR', Path(__file__).parent)) print(f"Saving files to {directory}") # test_bug1() test_bug2() ================================================ FILE: tests/test_conversion.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 import sys import logging from datetime import datetime import pytest import math import os from pathlib import Path import itertools import leuvenmapmatching as mm from leuvenmapmatching.util import dist_euclidean as de from leuvenmapmatching.util import dist_latlon as dll directory = None def test_path_to_gpx(): from leuvenmapmatching.util.gpx import path_to_gpx path = [(i, i, datetime.fromtimestamp(i)) for i in range(0, 10)] gpx = path_to_gpx(path) assert len(path) == len(gpx.tracks[0].segments[0].points) assert path[0][0] == pytest.approx(gpx.tracks[0].segments[0].points[0].latitude) assert path[0][1] == pytest.approx(gpx.tracks[0].segments[0].points[0].longitude) assert path[0][2] == gpx.tracks[0].segments[0].points[0].time def test_grs80(): from leuvenmapmatching.util.projections import latlon2grs80 coordinates = [(4.67878, 50.864), (4.68054, 50.86381), (4.68098, 50.86332), (4.68129, 50.86303), (4.6817, 50.86284), (4.68277, 50.86371), (4.68894, 50.86895), (4.69344, 50.86987), (4.69354, 50.86992), (4.69427, 50.87157), (4.69643, 50.87315), (4.69768, 50.87552), (4.6997, 50.87828)] points = latlon2grs80(coordinates, lon_0=coordinates[0][0], lat_ts=coordinates[0][1]) points = list(points) point = points[0] assert point[0] == pytest.approx(618139.9385518166) assert point[1] == pytest.approx(5636043.991970774) def test_distance1(): p1 = (38.898556, -77.037852) p2 = (38.897147, -77.043934) d = dll.distance(p1, p2) assert d == pytest.approx(549.1557912048178), f"Got: {d}" def test_distance2(): o_p1 = (6007539.987516373, -13607675.997610645) m_p1 = (6007518.475594072, -13607641.049711559) m_p2 = (6007576.295597112, -13607713.306589901) dist, proj_m, t_m = de.distance_point_to_segment(o_p1, m_p1, m_p2) assert dist == pytest.approx(5.038773480896327), f"dist = {dist}" assert t_m == pytest.approx(0.4400926470800718), f"t_m = {t_m}" def test_bearing1(): lat1, lon1 = math.radians(38.898556), math.radians(-77.037852) lat2, lon2 = math.radians(38.897147), math.radians(-77.043934) b = dll.bearing_radians(lat1, lon1, lat2, lon2) b = math.degrees(b) # assert b == pytest.approx(253.42138889), f"Got: {b}" assert b == pytest.approx(-106.5748183426045), f"Got: {b}" def test_destination1(): lat1, lon1 = math.radians(53.32055556), math.radians(1.72972222) bearing = math.radians(96.02166667) dist = 124800 lat2, lon2 = dll.destination_radians(lat1, lon1, bearing, dist) lat2, lon2 = (math.degrees(lat2), math.degrees(lon2)) assert lat2 == pytest.approx(53.188269553709034), f"Got: {lat2}" assert lon2 == pytest.approx(3.592721390871882), f"Got: {lon2}" def test_distance_segment_to_segment1(): f1 = (50.900393, 4.728607) f2 = (50.900389, 4.734047) t1 = (50.898538, 4.726107) t2 = (50.898176, 4.735463) d, pf, pt, u_f, u_t = dll.distance_segment_to_segment(f1, f2, t1, t2) if directory: plot_distance_segment_to_segment_latlon(f1, f2, t1, t2, pf, pt, "test_distance_segment_to_segment1") assert d == pytest.approx(216.60187728486514) assert pf == pytest.approx((50.900392999999994, 4.728607)) assert pt == pytest.approx((50.898448650708666, 4.728418070396815)) assert u_f == pytest.approx(0) assert u_t == pytest.approx(0.2470133466162735) def test_distance_segment_to_segment2(): f1 = (0, 0) f2 = (-0.43072496752146333, 381.4928613075559) t1 = (-206.26362055248765, -175.32538004745732) t2 = (-246.4746107556939, 480.8174213050763) d, pf, pt, u_f, u_t = de.distance_segment_to_segment(f1, f2, t1, t2) if directory: plot_distance_segment_to_segment_euc(f1, f2, t1, t2, pf, pt, "test_distance_segment_to_segment2") assert d == pytest.approx(216.60187728486514) assert pf == pytest.approx((0.0, 0.0)) assert pt == pytest.approx((-216.1962718133358, -13.249350827191222)) assert u_f == pytest.approx(0) assert u_t == pytest.approx(0.2470133466162735) def test_distance_segment_to_segment3(): f1 = (50.87205, 4.66089) f2 = (50.874550000000006, 4.672980000000001) t1 = (50.8740376, 4.6705204) t2 = (50.8741866999999, 4.67119980000001) d, pf, pt, u_f, u_t = dll.distance_segment_to_segment(f1, f2, t1, t2) if directory: plot_distance_segment_to_segment_latlon(f1, f2, t1, t2, pf, pt, "test_distance_segment_to_segment3") assert d == pytest.approx(0) assert pf == pytest.approx((50.87410572908839, 4.670830969750696)) assert pt == pytest.approx((50.87410575464133, 4.670830955670548)) assert u_f == pytest.approx(0.8222551304652699) assert u_t == pytest.approx(0.4571036354431931) def test_distance_segment_to_segment4(): f1 = (0, 0) f2 = (278.05674689789083, 848.3102386968303) t1 = (221.055090540802, 675.7367042826397) t2 = (237.6344733521503, 723.4080418578025) d, pf, pt, u_f, u_t = de.distance_segment_to_segment(f1, f2, t1, t2) if directory: plot_distance_segment_to_segment_euc(f1, f2, t1, t2, pf, pt, "test_distance_segment_to_segment4") assert d == pytest.approx(0) assert pf == pytest.approx((228.63358669727376, 697.5274459946864)) assert pt == pytest.approx((228.63358669727376, 697.5274459946864)) assert u_f == pytest.approx(0.8222551304652699) assert u_t == pytest.approx(0.4571036354431931) def test_distance_point_to_segment1(): locs = [ (47.6373, -122.0950167), (47.6369, -122.0950167), (47.6369, -122.0959167), (47.6369, -122.09422), (47.6369, -122.09400), (47.6375, -122.09505) ] loc_a = (47.6372498273849, -122.094900012016) loc_b = (47.6368394494057, -122.094280421734) segments = [] for lat_a, lat_b in itertools.product((loc_a[0], loc_b[0]), repeat=2): for lon_a, lon_b in itertools.product((loc_a[1], loc_b[1]), repeat=2): segments.append(((lat_a, lon_a), (lat_b, lon_b))) # segments = [(loc_a, loc_b)] for constrain in [True, False]: for loc_idx, loc in enumerate(locs): for seg_idx, (loc_a, loc_b) in enumerate(segments): dist1, pi1, ti1 = dll.distance_point_to_segment(loc, loc_a, loc_b, constrain=constrain) dist2, pi2, ti2 = dll.distance_point_to_segment(loc, loc_b, loc_a, constrain=constrain) if directory: plot_distance_point_to_segment_latlon(loc, loc_a, loc_b, pi1, f"point_to_segment_{loc_idx}_{seg_idx}_{constrain}.png") assert dist1 == pytest.approx(dist2), \ f"Locs[{loc_idx},{seg_idx},{constrain}]: Distances different, {dist1} != {dist2}" assert pi1[0] == pytest.approx(pi2[0]), \ f"Locs[{loc_idx},{seg_idx},{constrain}]: y coord different, {pi1[0]} != {pi2[0]}" assert pi1[1] == pytest.approx(pi2[1]), \ f"Locs[{loc_idx},{seg_idx},{constrain}]: y coord different, {pi1[1]} != {pi2[1]}" def plot_distance_point_to_segment_latlon(f, t1, t2, pt, fn): import smopy import matplotlib.pyplot as plt lat_min = min(f[0], t1[0], t2[0]) lat_max = max(f[0], t1[0], t2[0]) lon_min = min(f[1], t1[1], t2[1]) lon_max = max(f[1], t1[1], t2[1]) bb = [lat_min, lon_min, lat_max, lon_max] m = smopy.Map(bb) ax = m.show_mpl(figsize=(10, 10)) p1 = m.to_pixels(t1) p2 = m.to_pixels(t2) p3 = m.to_pixels(f) p4 = m.to_pixels(pt) ax.plot([p1[0], p2[0]], [p1[1], p2[1]], 'o-', color="black") ax.plot([p3[0]], [p3[1]], 'o-', color="black") ax.plot([p3[0], p4[0]], [p3[1], p4[1]], '--', color="red") plt.savefig(str(directory / fn)) plt.close(plt.gcf()) def plot_distance_segment_to_segment_latlon(f1, f2, t1, t2, pf, pt, fn): import smopy import matplotlib.pyplot as plt lat_min = min(f1[0], f2[0], t1[0], t2[0]) lat_max = max(f1[0], f2[0], t1[0], t2[0]) lon_min = min(f1[1], f2[1], t1[1], t2[1]) lon_max = max(f1[1], f2[1], t1[1], t2[1]) bb = [lat_min, lon_min, lat_max, lon_max] m = smopy.Map(bb) ax = m.show_mpl(figsize=(10, 10)) p1 = m.to_pixels(f1) p2 = m.to_pixels(f2) p3 = m.to_pixels(t1) p4 = m.to_pixels(t2) p5 = m.to_pixels(pf) p6 = m.to_pixels(pt) ax.plot([p1[0], p2[0]], [p1[1], p2[1]], 'o-') ax.plot([p3[0], p4[0]], [p3[1], p4[1]], 'o-') ax.plot([p5[0], p6[0]], [p5[1], p6[1]], 'x-') plt.savefig(str(directory / fn)) plt.close(plt.gcf()) def plot_distance_segment_to_segment_euc(f1, f2, t1, t2, pf, pt, fn): import matplotlib.pyplot as plt fig, ax = plt.subplots(1, 1, figsize=(10, 10)) ax.plot([f1[1], f2[1]], [f1[0], f2[0]], 'o-') ax.plot([t1[1], t2[1]], [t1[0], t2[0]], 'o-') ax.plot([pf[1], pt[1]], [pf[0], pt[0]], 'x-') ax.axis('equal') ax.set_aspect('equal') plt.savefig(str(directory / fn)) plt.close(plt.gcf()) if __name__ == "__main__": # mm.matching.logger.setLevel(logging.INFO) mm.logger.setLevel(logging.DEBUG) mm.logger.addHandler(logging.StreamHandler(sys.stdout)) directory = Path(os.environ.get('TESTDIR', Path(__file__).parent)) print(f"Saving files to {directory}") # test_path_to_gpx() test_grs80() # test_distance1() # test_bearing1() # test_destination1() # test_distance_segment_to_segment1() # test_distance_segment_to_segment2() # test_distance_segment_to_segment3() # test_distance_segment_to_segment4() # test_distance_point_to_segment1() ================================================ FILE: tests/test_examples.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 """ tests.test_examples ~~~~~~~~~~~~~~~~~~~ Run standalone python files that are a complete examples. Used to test the full examples in the documentation. :author: Wannes Meert :copyright: Copyright 2015-2022 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import sys import os import logging from pathlib import Path import subprocess as sp import pytest import leuvenmapmatching as mm logger = mm.logger examples_path = Path(os.path.realpath(__file__)).parent / "examples" def test_examples(): example_fns = examples_path.glob("*.py") for example_fn in example_fns: execute_file(example_fn.name) def importrun_file(fn, cmp_with_previous=False): import importlib fn = f"examples.{fn[:-3]}" print(f"Importing: {fn}") o = importlib.import_module(fn) o.run() def execute_file(fn, cmp_with_previous=False): print(f"Testing: {fn}") fn = examples_path / fn assert fn.exists() try: cmd = sp.run(["python3", fn], capture_output=True, check=True) except sp.CalledProcessError as exc: print(exc) print(exc.stderr.decode()) print(exc.stdout.decode()) raise exc if cmp_with_previous: # Not ready to be used in general testing, output contains floats result_data = cmd.stdout.decode() correct_fn = fn.with_suffix(".log") if correct_fn.exists(): with correct_fn.open("r") as correct_fp: correct_data = correct_fp.read() print(correct_data) print(result_data) assert correct_data == result_data, f"Logged output different for {fn}" else: with correct_fn.open("w") as correct_fp: correct_fp.write(result_data) if __name__ == "__main__": logger.setLevel(logging.WARNING) logger.addHandler(logging.StreamHandler(sys.stdout)) directory = Path(os.environ.get('TESTDIR', Path(__file__).parent)) print(f"Saving files to {directory}") # execute_file("example_1_simple.py", cmp_with_previous=True) execute_file("example_using_osmnx_and_geopandas.py", cmp_with_previous=True) # importrun_file("example_using_osmnx_and_geopandas.py", cmp_with_previous=True) ================================================ FILE: tests/test_newsonkrumm2009.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 """ tests.test_path_newsonkrumm2009 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Based on the data available at: https://www.microsoft.com/en-us/research/publication/hidden-markov-map-matching-noise-sparseness/ Notes: * There is a 'bug' in the map available from the website. Multiple segments (streets) in the map are not connected but have overlapping, but disconnected, nodes. For example, the following nodes are on the same location and should be connected because the given path runs over this road: - 884147801204 and 884148400033 - 884148100260 and 884148001002 * The path is missing a number of observations. For those parts non-emitting nodes are required. This occurs at: - 2770:2800 (index 2659 is start) - 2910:2929 :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import os import sys import logging import pickle from pathlib import Path import csv from datetime import datetime from itertools import product import pytest import leuvenmapmatching as mm from leuvenmapmatching.matcher import base from leuvenmapmatching.matcher.distance import DistanceMatcher from leuvenmapmatching.map.sqlite import SqliteMap import leuvenmapmatching.visualization as mm_viz MYPY = False if MYPY: from typing import List, Tuple logger = mm.logger this_path = Path(os.path.realpath(__file__)).parent / "rsrc" / "newson_krumm_2009" gps_data = this_path / "gps_data.txt" gps_data_pkl = gps_data.with_suffix(".pkl") ground_truth_route = this_path / "ground_truth_route.txt" road_network = this_path / "road_network.txt" road_network_zip = this_path / "road_network.zip" road_network_db = road_network.with_suffix(".sqlite") directory = None base.default_label_width = 34 def read_gps(route_fn): route = [] with route_fn.open("r") as route_f: reader = csv.reader(route_f, delimiter='\t') next(reader) for row in reader: date, time, lat, lon = row[:4] date_str = date + " " + time ts = datetime.strptime(date_str, '%d-%b-%Y %H:%M:%S') lat = float(lat) lon = float(lon) route.append((lat, lon, ts)) logger.debug(f"Read GPS trace of {len(route)} points") return route def read_paths(paths_fn): paths = [] with paths_fn.open("r") as paths_f: reader = csv.reader(paths_f, delimiter='\t') next(reader) for row in reader: pathid, trav = row[:2] pathid = int(pathid) trav = int(trav) paths.append((pathid, trav)) logger.debug(f"Read correct trace of {len(paths)} nodes") return paths def parse_linestring(line): # type: (str) -> List[Tuple[float, float]] line = line[line.index("(") + 1:line.index(")")] latlons = [] for lonlat in line.split(", "): lon, lat = lonlat.split(" ") latlons.append((float(lat), float(lon))) return latlons def read_map(map_fn): logger.debug(f"Reading map ...") mmap = SqliteMap("road_network", use_latlon=True, dir=this_path) node_cnt = 0 edge_cnt = 0 # new_node_id = 1000000000000 new_node_id = 1 with map_fn.open("r") as map_f: reader = csv.reader(map_f, delimiter='\t') next(reader) for row in reader: eid, nf, nt, twoway, speed, length, innernodes = row eid = int(eid) nf = int(nf) nt = int(nt) length = int(length) twoway = int(twoway) speed = float(speed) if twoway == 0: twoway = False elif twoway == 1: twoway = True else: raise Exception(f"Unknown value for twoway: {twoway}") innernodes = parse_linestring(innernodes) # Add nodes to map mmap.add_node(nf, innernodes[0], ignore_doubles=True, no_index=True, no_commit=True) mmap.add_node(nt, innernodes[-1], ignore_doubles=True, no_index=True, no_commit=True) node_cnt += 2 prev_node = nf assert(length < 1000) idx = 1 for innernode in innernodes[1:-1]: # innernode_id = nf * 1000 + idx innernode_id = new_node_id new_node_id += 1 mmap.add_node(innernode_id, innernode, no_index=True, no_commit=True) # Should not be double node_cnt += 1 mmap.add_edge(prev_node, innernode_id, speed=speed, edge_type=0, path=eid, pathnum=idx, no_index=True, no_commit=True) edge_cnt += 1 if twoway: mmap.add_edge(innernode_id, prev_node, speed=speed, edge_type=0, path=eid, pathnum=-idx, no_index=True, no_commit=True) edge_cnt += 1 prev_node = innernode_id idx += 1 mmap.add_edge(prev_node, nt, speed=speed, edge_type=0, path=eid, pathnum=idx, no_index=True, no_commit=True) edge_cnt += 1 if twoway: mmap.add_edge(nt, prev_node, speed=speed, edge_type=0, path=eid, pathnum=-idx, no_index=True, no_commit=True) edge_cnt += 1 if node_cnt % 100000 == 0: mmap.db.commit() logger.debug(f"... done: {node_cnt} nodes and {edge_cnt} edges") mmap.reindex_nodes() mmap.reindex_edges() assert(new_node_id < 100000000000) return mmap def correct_map(mmap): """Add edges between nodes with degree > 2 that are on the exact same location. This ignore that with bridges, the roads might not be connected. But we need a correct because the dataset has a number of interrupted paths. """ def correct_edge(labels): labels = [label for label in labels if label > 100000000000] logger.info(f"Add connections between {labels}") for l1, l2 in product(labels, repeat=2): mmap.add_edge(l1, l2, edge_type=1) mmap.find_duplicates(func=correct_edge) def load_data(): max_route_length = None # 200 # Paths if not ground_truth_route.exists(): import requests url = f'https://www.microsoft.com/en-us/research/uploads/prod/2017/07/ground_truth_route.txt' logger.debug("Download gound_truth_route.txt from microsoft.com") r = requests.get(url, stream=True) with ground_truth_route.open('wb') as ofile: for chunk in r.iter_content(chunk_size=1024): if chunk: ofile.write(chunk) paths = read_paths(ground_truth_route) # Map if road_network_db.exists(): map_con = SqliteMap.from_file(road_network_db) logger.debug(f"Read road network from db file {road_network_db} ({map_con.size()} nodes)") else: if not road_network.exists(): import requests url = f'https://www.microsoft.com/en-us/research/uploads/prod/2017/07/road_network.zip' logger.debug("Download road_network.zip from microsoft.com") r = requests.get(url, stream=True) with road_network_zip.open('wb') as ofile: for chunk in r.iter_content(chunk_size=1024): if chunk: ofile.write(chunk) import zipfile logger.debug("Unzipping road_network.zip") with zipfile.ZipFile(str(road_network_zip), "r") as zip_ref: zip_ref.extractall(str(road_network_zip.parent)) map_con = read_map(road_network) correct_map(map_con) logger.debug(f"Create road network to db file {map_con.db_fn} ({map_con.size()} nodes)") # Route if gps_data_pkl.exists(): with gps_data_pkl.open("rb") as ifile: route = pickle.load(ifile) logger.debug(f"Read gps route from file ({len(route)} points)") else: if not gps_data.exists(): import requests url = f'https://www.microsoft.com/en-us/research/uploads/prod/2017/07/gps_data.txt' logger.debug("Download gps_data.txt from microsoft.com") r = requests.get(url, stream=True) with gps_data.open('wb') as ofile: for chunk in r.iter_content(chunk_size=1024): if chunk: ofile.write(chunk) route = read_gps(gps_data) if max_route_length: route = route[:max_route_length] with gps_data_pkl.open("wb") as ofile: pickle.dump(route, ofile) route = [(lat, lon) for lat, lon, _ in route] return paths, map_con, route def test_route_slice1(): if directory: import matplotlib.pyplot as plt nodes, map_con, route = load_data() zoom_path = True matcher = DistanceMatcher(map_con, min_prob_norm=0.001, max_dist=200, dist_noise=6, dist_noise_ne=12, obs_noise=30, obs_noise_ne=150, non_emitting_states=True) route_slice = route[2657:2662] matcher.match(route_slice) path_pred = matcher.path_pred_onlynodes path_sol = [172815, 172816, 172817, 172818, 172819, 172820, 172821, 172822, 172823, 172824, 172825, 172826, 172827, 172828, 172829, 172830, 884148100261, 172835, 172836, 172837, 884148100254, 172806, 884148100255, 172807] # Can change when building db assert len(path_pred) == len(path_sol) def test_bug1(): map_con = SqliteMap("map", use_latlon=True) map_con.add_nodes([ (1, (47.590439915657, -122.238368690014)), (2, (47.5910192728043, -122.239519357681)), (3, (47.5913706421852, -122.240168452263)) ]) map_con.add_edges([ (1, 2), (2, 3) ]) path = [ # (47.59043333, -122.2384167), (47.59058333, -122.2387), (47.59071667, -122.2389833), (47.59086667, -122.2392667), (47.59101667, -122.23955), (47.59115, -122.2398333) ] path_sol = [(1, 2), (2, 3)] matcher = DistanceMatcher(map_con, min_prob_norm=0.001, max_dist=200, obs_noise=4.07, non_emitting_states=True) matcher.match(path, unique=True) path_pred = matcher.path_pred if directory: import matplotlib.pyplot as plt matcher.print_lattice_stats() logger.debug("Plotting post map ...") fig = plt.figure(figsize=(100, 100)) ax = fig.get_axes() mm_viz.plot_map(map_con, matcher=matcher, use_osm=True, ax=ax, show_lattice=False, show_labels=True, show_graph=True, zoom_path=True, show_matching=True) plt.savefig(str(directory / "test_newson_bug1.png")) plt.close(fig) logger.debug("... done") assert path_pred == path_sol, f"Edges not equal:\n{path_pred}\n{path_sol}" @pytest.mark.skip(reason="Takes a long time") def test_route(): if directory: import matplotlib.pyplot as plt else: plt = None paths, map_con, route = load_data() route = [(lat, lon) for lat, lon, _ in route] zoom_path = True # zoom_path = slice(2645, 2665) slice_route = None # slice_route = slice(650, 750) # slice_route = slice(2657, 2662) # First location where some observations are missing # slice_route = slice(2770, 2800) # Observations are missing # slice_route = slice(2910, 2950) # Interesting point # slice_route = slice(2910, 2929) # Interesting point # slice_route = slice(6825, 6833) # Outlier observation # if directory is not None: # logger.debug("Plotting pre map ...") # mm_viz.plot_map(map_con_latlon, path=route_latlon, use_osm=True, # show_lattice=False, show_labels=False, show_graph=False, zoom_path=zoom_path, # filename=str(directory / "test_newson_route.png")) # logger.debug("... done") matcher = DistanceMatcher(map_con, min_prob_norm=0.0001, max_dist=200, dist_noise=15, dist_noise_ne=30, obs_noise=30, obs_noise_ne=150, non_emitting_states=True) if slice_route is None: pkl_fn = this_path / "nodes_pred.pkl" if pkl_fn.exists(): with pkl_fn.open("rb") as pkl_file: logger.debug(f"Reading predicted nodes from pkl file") route_nodes = pickle.load(pkl_file) else: matcher.match(route) route_nodes = matcher.path_pred_onlynodes with pkl_fn.open("wb") as pkl_file: pickle.dump(route_nodes, pkl_file) from leuvenmapmatching.util.evaluation import route_mismatch_factor print(route_nodes[:10]) # route_edges = map_con.nodes_to_paths(route_nodes) # print(route_edges[:10]) grnd_paths, _ = zip(*paths) print(grnd_paths[:10]) route_paths = map_con.nodes_to_paths(route_nodes) print(route_paths[:10]) logger.debug(f"Compute route mismatch factor") factor, cnt_matches, cnt_mismatches, total_length, mismatches = \ route_mismatch_factor(map_con, route_paths, grnd_paths,window=None, keep_mismatches=True) logger.debug(f"factor = {factor}, " f"cnt_matches = {cnt_matches}/{cnt_mismatches} of {len(grnd_paths)}/{len(route_paths)}, " f"total_length = {total_length}\n" f"mismatches = " + " | ".join(str(v) for v in mismatches)) else: _, last_idx = matcher.match(route[slice_route]) logger.debug(f"Last index = {last_idx}") # matcher.match(route[2657:2662]) # First location where some observations are missing # matcher.match(route[2770:2800]) # Observations are missing # matcher.match(route[2910:2950]) # Interesting point # matcher.match(route[2910:2929]) # Interesting point # matcher.match(route[6000:]) path_pred = matcher.path_pred_onlynodes if directory: matcher.print_lattice_stats() logger.debug("Plotting post map ...") fig = plt.figure(figsize=(200, 200)) ax = fig.get_axes() mm_viz.plot_map(map_con, matcher=matcher, use_osm=True, ax=ax, show_lattice=False, show_labels=True, zoom_path=zoom_path, show_matching=True, show_graph=False) plt.savefig(str(directory / "test_newson_route_matched.png")) plt.close(fig) logger.debug("... done") logger.debug("Best path:") for m in matcher.lattice_best: logger.debug(m) print(path_pred) @pytest.mark.skip(reason="Takes a too long") def test_bug2(): from leuvenmapmatching.util.openstreetmap import locations_to_map map_con = SqliteMap("map", use_latlon=True, dir=directory) path = [ (50.87205, 4.66089), (50.874550000000006, 4.672980000000001), (50.87538000000001, 4.67698), (50.875800000000005, 4.6787600000000005), (50.876520000000006, 4.6818), (50.87688000000001, 4.683280000000001), (50.87814, 4.68733), (50.87832, 4.68778), (50.87879, 4.68851), (50.87903000000001, 4.68895), (50.879560000000005, 4.689170000000001), (50.87946, 4.6900900000000005), (50.879290000000005, 4.6909600000000005), (50.87906, 4.6921800000000005), (50.87935, 4.6924), (50.879720000000006, 4.69275), (50.88002, 4.6930700000000005), (50.880430000000004, 4.693440000000001), (50.880660000000006, 4.69357), (50.880660000000006, 4.6936100000000005), (50.88058, 4.694640000000001), (50.88055000000001, 4.69491), (50.88036, 4.696160000000001), (50.88009, 4.697550000000001), (50.87986, 4.6982800000000005), (50.879720000000006, 4.698790000000001), (50.87948, 4.699730000000001), (50.87914000000001, 4.6996400000000005), (50.87894000000001, 4.6995000000000005), (50.878800000000005, 4.699350000000001), (50.8785, 4.6991000000000005), (50.87841, 4.6990300000000005) ] locations_to_map(path, map_con, filename=directory / "osm.xml") path_sol = [(5777282112, 2633552218), (2633552218, 5777282111), (5777282111, 5777282110), (5777282110, 1642021707), (1642021707, 71361087), (71361087, 71364203), (71364203, 1151697757), (1151697757, 1647339017), (1647339017, 1647339030), (1647339030, 2058510349), (2058510349, 2633552212), (2633552212, 1380538577), (1380538577, 1439572271), (1439572271, 836434313), (836434313, 2633771041), (2633771041, 5042874484), (5042874484, 5042874485), (5042874485, 2518922583), (2518922583, 2659762546), (2659762546, 5777282063), (5777282063, 2633771037), (2633771037, 2633771035), (2633771035, 2633771033), (2633771033, 1151668705), (1151668705, 2633771094), (2633771094, 1151668722), (1151668722, 1151668724), (1151668724, 5543948222), (5543948222, 2058481517), (2058481517, 16933576), (16933576, 5543948221), (5543948221, 2518923620), (2518923620, 5543948020), (5543948020, 5543948019), (5543948019, 18635886), (18635886, 18635887), (18635887, 1036909153), (1036909153, 2658942230), (2658942230, 1001099975), (1001099975, 16933574), (16933574, 1125604152), (1125604152, 5543948238), (5543948238, 1125604150), (1125604150, 1125604148), (1125604148, 2634195334), (2634195334, 2087854243), (2087854243, 5543948237), (5543948237, 160226603), (160226603, 180130266), (180130266, 5543948227), (5543948227, 5543948226), (5543948226, 1195681902), (1195681902, 101135392), (101135392, 2606704673), (2606704673, 18635977), (18635977, 1026111708), (1026111708, 1026111631), (1026111631, 16571375), (16571375, 2000680621), (2000680621, 999580042), (999580042, 16571370), (16571370, 2000680620), (2000680620, 5078692402), (5078692402, 5543948008), (5543948008, 16571371), (16571371, 999579936), (999579936, 2639836143), (2639836143, 5543948014), (5543948014, 5222992316), (5222992316, 30251323), (30251323, 159701080), (159701080, 3173217124), (3173217124, 1165209673), (1165209673, 1380538689), (1380538689, 2878334668), (2878334668, 2871137399), (2871137399, 2876902981), (2876902981, 2873624508), (2873624508, 2873624509), (2873624509, 2899666507), (2899666507, 2899666518), (2899666518, 2899666513), (2899666513, 2903073945), (2903073945, 2903073951), (2903073951, 1380538681), (1380538681, 2914810627), (2914810627, 2914810618), (2914810618, 2914810607), (2914810607, 2914810604), (2914810604, 2914810483), (2914810483, 2914810462), (2914810462, 2914810464), (2914810464, 1312433523), (1312433523, 20918594), (20918594, 2634267817), (2634267817, 2967425445), (2967425445, 3201523879), (3201523879, 157217466), (157217466, 2963305939), (2963305939, 3201523877), (3201523877, 3889275909), (3889275909, 3889275897), (3889275897, 157255077), (157255077, 30251882), (30251882, 157245624), (157245624, 1150903673), (1150903673, 4504936404)] matcher = DistanceMatcher(map_con, min_prob_norm=0.001, max_dist=200, obs_noise=4.07, non_emitting_states=True) nodes, idx = matcher.match(path, unique=True) path_pred = matcher.path_pred if directory: import matplotlib.pyplot as plt matcher.print_lattice_stats() logger.debug("Plotting post map ...") fig = plt.figure(figsize=(100, 100)) ax = fig.get_axes() mm_viz.plot_map(map_con, matcher=matcher, use_osm=True, ax=ax, show_lattice=False, show_labels=True, show_graph=False, zoom_path=True, show_matching=True) plt.savefig(str(directory / "test_newson_bug1.png")) plt.close(fig) logger.debug("... done") assert path_pred == path_sol, f"Edges not equal:\n{path_pred}\n{path_sol}" if __name__ == "__main__": logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) directory = Path(os.environ.get('TESTDIR', Path(__file__).parent)) print(f"Saving files to {directory}") # test_route() test_route_slice1() # test_bug1() # test_bug2() ================================================ FILE: tests/test_nonemitting.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 """ tests.test_nonemitting ~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2017-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import sys import os import logging from pathlib import Path try: import leuvenmapmatching as mm except ImportError: sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) import leuvenmapmatching as mm from leuvenmapmatching.matcher.distance import DistanceMatcher, DistanceMatching from leuvenmapmatching.matcher.simple import SimpleMatcher from leuvenmapmatching.map.inmem import InMemMap MYPY = False if MYPY: from typing import Tuple logger = mm.logger directory = None def setup_map(): path1 = [(1.8, 0.1), (1.8, 3.5), (3.0, 4.9)] # More nodes than observations path2 = [(1.8, 0.1), (1.8, 2.0), (1.8, 3.5), (3.0, 4.9)] path_sol = ['X', 'C', 'D', 'F'] mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "E", "K", "L", "F"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) return mapdb, path1, path2, path_sol def visualize_map(pathnb=1): mapdb, path1, path2, path_sol = setup_map() import leuvenmapmatching.visualization as mm_vis if pathnb == 2: path = path2 else: path = path1 mm_vis.plot_map(mapdb, path=path, show_labels=True, filename=(directory / "test_nonemitting_map.png")) def test_path1(): mapdb, path1, path2, path_sol = setup_map() matcher = SimpleMatcher(mapdb, max_dist_init=1, min_prob_norm=0.5, obs_noise=0.5, non_emitting_states=True, only_edges=False) matcher.match(path1, unique=True) path_pred = matcher.path_pred_onlynodes if directory: from leuvenmapmatching import visualization as mmviz matcher.print_lattice_stats() matcher.print_lattice() with (directory / 'lattice_path1.gv').open('w') as ofile: matcher.lattice_dot(file=ofile) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_nonemitting_test_path1.png")) assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" def test_path1_inc(): mapdb, path1, path2, path_sol = setup_map() matcher = SimpleMatcher(mapdb, max_dist_init=1, in_prob_norm=0.5, obs_noise=0.5, non_emitting_states=True, only_edges=False, max_lattice_width=1) print('## PHASE 1 ##') matcher.match(path1, unique=True) path_pred = matcher.path_pred_onlynodes if directory: from leuvenmapmatching import visualization as mmviz matcher.print_lattice_stats() matcher.print_lattice() with (directory / 'lattice_path1_inc1.gv').open('w') as ofile: matcher.lattice_dot(file=ofile, precision=2, render=True) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_nonemitting_test_path1_inc1.png")) print('## PHASE 2 ##') matcher.increase_max_lattice_width(3, unique=True) path_pred = matcher.path_pred_onlynodes if directory: from leuvenmapmatching import visualization as mmviz matcher.print_lattice_stats() matcher.print_lattice() with (directory / 'lattice_path1_inc2.gv').open('w') as ofile: matcher.lattice_dot(file=ofile, precision=2, render=True) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_nonemitting_test_path1_inc2.png")) assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" def test_path1_dist(): mapdb, path1, path2, path_sol = setup_map() matcher = DistanceMatcher(mapdb, max_dist_init=1, min_prob_norm=0.5, obs_noise=0.5, non_emitting_states=True, only_edges=True) matcher.match(path1, unique=True) path_pred = matcher.path_pred_onlynodes if directory: from leuvenmapmatching import visualization as mmviz matcher.print_lattice_stats() matcher.print_lattice() print("LATTICE BEST") for m in matcher.lattice_best: print(m) with (directory / 'lattice_path1.gv').open('w') as ofile: matcher.lattice_dot(file=ofile) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_nonemitting_test_path1_dist.png")) assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" def test_path2(): mapdb, path1, path2, path_sol = setup_map() matcher = SimpleMatcher(mapdb, max_dist_init=1, min_prob_norm=0.5, obs_noise=0.5, non_emitting_states=True, only_edges=False) matcher.match(path2, unique=True) path_pred = matcher.path_pred_onlynodes dists = matcher.path_all_distances() if directory: from leuvenmapmatching import visualization as mmviz matcher.print_lattice_stats() matcher.print_lattice() with (directory / 'lattice_path2.gv').open('w') as ofile: matcher.lattice_dot(file=ofile) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, filename=str(directory / "test_nonemitting_test_path2.png")) assert path_pred == path_sol, "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) def test_path2_dist(): mapdb, path1, path2, path_sol = setup_map() matcher = DistanceMatcher(mapdb, max_dist_init=1, min_prob_norm=0.5, obs_noise=0.5, dist_noise=0.5, non_emitting_states=True) matcher.match(path2, unique=True) path_pred = matcher.path_pred_onlynodes if directory: from leuvenmapmatching import visualization as mmviz matcher.print_lattice_stats() matcher.print_lattice() # with (directory / 'lattice_path2.gv').open('w') as ofile: # matcher.lattice_dot(file=ofile) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, filename=str(directory / "test_nonemitting_test_path2_dist.png")) assert path_pred == path_sol, "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) def test_path2_incremental(): mapdb, path1, path2, path_sol = setup_map() matcher = SimpleMatcher(mapdb, max_dist_init=1, min_prob_norm=0.5, obs_noise=0.5, non_emitting_states=True, only_edges=False) matcher.match(path2[:2]) path_pred_1 = matcher.path_pred_onlynodes matcher.match(path2, expand=True) path_pred = matcher.path_pred_onlynodes if directory: from leuvenmapmatching import visualization as mmviz matcher.print_lattice_stats() matcher.print_lattice() with (directory / 'lattice_path2.gv').open('w') as ofile: matcher.lattice_dot(file=ofile) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, filename=str(directory / "test_nonemitting_test_path2.png")) assert path_pred_1 == path_sol[:2], "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) assert path_pred == path_sol, "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) def test_path_duplicate(): from datetime import datetime # A path with two identical points path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.1, 3.3), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C"]), "B": ((1, 3), ["A", "C", "D"]), "C": ((2, 2), ["A", "B", "D", "E"]), "D": ((2, 4), ["B", "C", "D", "E"]), "E": ((3, 3), ["C", "D", "F"]), "F": ((3, 5), ["D", "E"]) }, use_latlon=False) matcher = SimpleMatcher(mapdb, max_dist=None, min_prob_norm=None, non_emitting_states = True, only_edges=False) #Matching with and without timestamps signed to the points path_pred = matcher.match(path, unique=False) path = [(p1, p2, datetime.fromtimestamp(i)) for i, (p1, p2) in enumerate(path)] path_pred_time = matcher.match(path, unique=False) if directory: from leuvenmapmatching import visualization as mmviz matcher.print_lattice_stats() matcher.print_lattice() mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, filename=str(directory / "test_nonemitting_test_path_duplicate.png")) # The path should be identical regardless of the timestamps assert path_pred == path_pred_time, f"Nodes not equal:\n{path_pred}\n{path_pred_time}" def test_path3_many_obs(): path = [(1, 0), (3, -0.1), (3.7, 0.6), (4.5, 0.7), (5.5, 1.2), (6.5, 0.88), (7.5, 0.65), (8.5, -0.1), (9.8, 0.1),(10.1, 1.9)] path_sol = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I'] mapdb = InMemMap("map", graph={ "A": ((1, 0.00), ["B"]), "B": ((3, 0.00), ["A", "C"]), "C": ((4, 0.70), ["B", "D"]), "D": ((5, 1.00), ["C", "E"]), "E": ((6, 1.00), ["D", "F"]), "F": ((7, 0.70), ["E", "G"]), "G": ((8, 0.00), ["F", "H"]), "H": ((10, 0.0), ["G", "I"]), "I": ((10, 2.0), ["H"]) }, use_latlon=False) matcher = SimpleMatcher(mapdb, max_dist_init=0.2, obs_noise=1, obs_noise_ne=10, non_emitting_states=True) matcher.match(path) path_pred = matcher.path_pred_onlynodes if directory: matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, linewidth=10, show_graph=True, show_lattice=True, filename=str(directory / "test_test_path_ne_3_mo.png")) assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" def test_path3_few_obs_en(): path = [(1, 0), (7.5, 0.65), (10.1, 1.9)] path_sol = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I'] mapdb = InMemMap("map", graph={ "A": ((1, 0.00), ["B"]), "B": ((3, 0.00), ["A", "C"]), "C": ((4, 0.70), ["B", "D"]), "D": ((5, 1.00), ["C", "E"]), "E": ((6, 1.00), ["D", "F"]), "F": ((7, 0.70), ["E", "G"]), "G": ((8, 0.00), ["F", "H"]), "H": ((10, 0.0), ["G", "I"]), "I": ((10, 2.0), ["H"]) }, use_latlon=False) matcher = SimpleMatcher(mapdb, max_dist_init=0.2, obs_noise=1, obs_noise_ne=10, non_emitting_states=True, only_edges=False) matcher.match(path) path_pred = matcher.path_pred_onlynodes if directory: matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, linewidth=10, filename=str(directory / "test_test_path_ne_3_fo.png")) assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" def test_path3_few_obs_e(): path = [(1, 0), (7.5, 0.65), (10.1, 1.9)] path_sol = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I'] mapdb = InMemMap("map", graph={ "A": ((1, 0.00), ["B"]), "B": ((3, 0.00), ["A", "C"]), "C": ((4, 0.70), ["B", "D"]), "D": ((5, 1.00), ["C", "E"]), "E": ((6, 1.00), ["D", "F"]), "F": ((7, 0.70), ["E", "G"]), "G": ((8, 0.00), ["F", "H"]), "H": ((10, 0.0), ["G", "I"]), "I": ((10, 2.0), ["H"]) }, use_latlon=False) matcher = SimpleMatcher(mapdb, max_dist_init=0.2, obs_noise=1, obs_noise_ne=10, non_emitting_states=True, only_edges=True) matcher.match(path) path_pred = matcher.path_pred_onlynodes if directory: matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, linewidth=10, filename=str(directory / "test_test_path_e_3_fo.png")) assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" def test_path3_dist(): path = [(0, 1), (0.65, 7.5), (1.9, 10.1)] path_sol = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I'] mapdb = InMemMap("map", graph={ "A": ((0.00, 1), ["B"]), "B": ((0.00, 3), ["A", "C"]), "C": ((0.70, 3), ["B", "D"]), "D": ((1.00, 5), ["C", "E"]), "E": ((1.00, 6), ["D", "F"]), "F": ((0.70, 7), ["E", "G"]), "G": ((0.00, 8), ["F", "H"]), "H": ((0.0, 10), ["G", "I"]), "I": ((2.0, 10), ["H"]) }, use_latlon=False) matcher = DistanceMatcher(mapdb, max_dist_init=0.2, obs_noise=0.5, obs_noise_ne=2, dist_noise=0.5, non_emitting_states=True) states, lastidx = matcher.match(path) path_pred = matcher.path_pred_onlynodes if directory: from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, linewidth=2, filename=str(directory / "test_path_3_dist.png")) assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" for obs_idx, m in enumerate(matcher.lattice_best): # type: Tuple[int, DistanceMatching] state = m.shortkey # tuple indicating edge ne_str = "e" if m.is_emitting() else "ne" # state is emitting or not p1_str = "{:>5.2f}-{:<5.2f}".format(*m.edge_m.pi) # best matching location on graph p2_str = "{:>5.2f}-{:<5.2f}".format(*m.edge_o.pi) # best matching location on track print(f"{obs_idx:<2} | {state} | {ne_str:<2} | {p1_str} | {p2_str}") if __name__ == "__main__": # mm.matching.logger.setLevel(logging.INFO) logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) directory = Path(os.environ.get('TESTDIR', Path(__file__).parent)) print(f"Saving files to {directory}") # visualize_map(pathnb=1) # test_path1() # test_path1_inc() # test_path1_dist() test_path2() # test_path2_dist() # test_path2_incremental() # test_path_duplicate() # test_path3_many_obs() # test_path3_few_obs_en() # test_path3_few_obs_e() # test_path3_dist() ================================================ FILE: tests/test_nonemitting_circle.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 """ tests.test_nonemitting_circle ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import sys, os import logging import math from pathlib import Path import numpy as np try: import leuvenmapmatching as mm except ImportError: sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) import leuvenmapmatching as mm from leuvenmapmatching.matcher.simple import SimpleMatcher from leuvenmapmatching.matcher.distance import DistanceMatcher from leuvenmapmatching.map.inmem import InMemMap directory = None def setup_map(disconnect=True): theta = np.linspace(0, 2 * math.pi, 4 * 5 + 1)[:-1] ox = 0.1 + np.cos(theta * 0.95) oy = np.sin(theta * 1) path1 = list(zip(ox, oy)) # all observations path2 = [(x, y) for x, y in zip(ox, oy) if x > -0.60] nx = np.cos(theta) ny = np.sin(theta) nl = [f"N{i}" for i in range(len(nx))] graph = {} for i, (x, y, l) in enumerate(zip(nx, ny, nl)): if disconnect: edges = [] if i != len(nx) - 1: edges.append(nl[(i + 1) % len(nl)]) if i != 0: edges.append(nl[(i - 1) % len(nl)]) else: edges = [nl[(i - 1) % len(nl)], nl[(i + 1) % len(nl)]] graph[l] = ((x, y), edges) graph["M"] = ((0, 0), ["N5", "N15"]) graph["N5"][1].append("M") graph["N15"][1].append("M") print(graph) path_sol = nl if not disconnect: path_sol += ["N0"] mapdb = InMemMap("map", graph=graph, use_latlon=False) return mapdb, path1, path2, path_sol def visualize_map(): if directory is None: return import matplotlib.pyplot as plt import leuvenmapmatching.visualization as mm_vis mapdb, path1, path2, path_sol = setup_map() fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(10, 5)) mm_vis.plot_map(mapdb, path=path1, ax=ax, show_labels=True) fig.savefig(str(directory / 'test_nonemitting_circle_map_path1.png')) plt.close(fig) fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(10, 10)) mm_vis.plot_map(mapdb, path=path2, ax=ax, show_labels=True) fig.savefig(str(directory / 'test_nonemitting_circle_map_path2.png')) plt.close(fig) def visualize_path(matcher, mapdb, name="test"): import matplotlib.pyplot as plt from leuvenmapmatching import visualization as mmviz fig, ax = plt.subplots(1, 1, figsize=(10, 10)) mmviz.plot_map(mapdb, matcher=matcher, ax=ax, show_labels=True, show_matching=True, show_graph=True, linewidth=2) fn = directory / f"test_nonemitting_circle_{name}_map.png" fig.savefig(str(fn)) plt.close(fig) print(f"saved to {fn}") def test_path1(): mapdb, path1, path2, path_sol = setup_map() matcher = SimpleMatcher(mapdb, max_dist_init=1, min_prob_norm=0.5, obs_noise=0.5, non_emitting_states=True) matcher.match(path1, unique=True) path_pred = matcher.path_pred_onlynodes if directory: visualize_path(matcher, mapdb, name="testpath1") assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" def test_path1_dist(): mapdb, path1, path2, path_sol = setup_map() matcher = DistanceMatcher(mapdb, max_dist_init=1, min_prob_norm=0.8, obs_noise=0.5, non_emitting_states=True) matcher.match(path1, unique=True) path_pred = matcher.path_pred_onlynodes if directory: visualize_path(matcher, mapdb, name="test_path1_dist") assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" def test_path2(): mapdb, path1, path2, _ = setup_map() path_sol = [f"N{i}" for i in range(20)] matcher = SimpleMatcher(mapdb, max_dist_init=0.2, min_prob_norm=0.1, obs_noise=0.1, obs_noise_ne=1, non_emitting_states=True, only_edges=True) path_pred = matcher.match(path2, unique=True) print(path_pred) path_pred = matcher.path_pred_onlynodes if directory: matcher.print_lattice_stats(verbose=True) matcher.print_lattice() print("Best path through lattice:") for m in matcher.lattice_best: print(m) visualize_path(matcher, mapdb, name="testpath2") assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" def test_path2_dist(): mapdb, path1, path2, _ = setup_map() path_sol = [f"N{i}" for i in range(20)] matcher = DistanceMatcher(mapdb, max_dist_init=0.2, min_prob_norm=0.1, obs_noise=0.1, obs_noise_ne=1, non_emitting_states=True) matcher.match(path2) path_pred = matcher.path_pred_onlynodes if directory: visualize_path(matcher, mapdb, name="test_path2_dist") assert path_pred == path_sol, f"Nodes not equal:\n{path_pred}\n{path_sol}" if __name__ == "__main__": # mm.matching.logger.setLevel(logging.INFO) mm.logger.setLevel(logging.DEBUG) mm.logger.addHandler(logging.StreamHandler(sys.stdout)) directory = Path(os.environ.get('TESTDIR', Path(__file__).parent)) print(f"Saving files to {directory}") # visualize_map() test_path1() # test_path1_dist() # test_path2() # test_path2_dist() ================================================ FILE: tests/test_parallelroads.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 """ tests.test_parallelroads ~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import sys import os import logging from pathlib import Path import leuvenmapmatching as mm from leuvenmapmatching.map.sqlite import SqliteMap from leuvenmapmatching.matcher.distance import DistanceMatcher from leuvenmapmatching.util.dist_euclidean import lines_parallel logger = mm.logger directory = None def create_map1(): db = SqliteMap("map", use_latlon=False, dir=directory) logger.debug(f"Initialized db: {db}") db.add_nodes([ (1, (1, 1)), (2, (1, 2.9)), (22, (1, 3.0)), (3, (2, 2)), (33, (2, 2.1)), (4, (2, 4)), (5, (3, 3)), (6, (3, 5)) ]) db.add_edges([ (1, 2), (1, 3), (2, 22), (2, 1), (22, 2), (22, 33), (22, 4), (3, 33), (3, 1), (3, 2), (3, 5), (33, 3), (4, 22), (4, 33), (4, 5), (4, 6), (5, 3), (5, 4), (5, 6), (6, 4), (6, 5) ]) logger.debug(f"Filled db: {db}") return db def create_path1(): return [(0.9, 2.5), (1.1, 2.75), (1.25, 2.6), (1.4, 2.5), (1.5, 2.4), (1.6, 2.5), (1.4, 2.7), (1.2, 2.9), (1.1, 3.0), (1.3, 3.2)] def test_parallel(): result = lines_parallel((1, 2.9), (2, 2), (1, 3.0), (2, 2.1), d=0.1) assert result is True def test_bb1(): mapdb = create_map1() if directory: from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, show_labels=True, show_graph=True, filename=str(directory / "test_bb.png")) mapdb.connect_parallelroads() assert mapdb.size() == 8 coord = mapdb.node_coordinates(2) assert coord == (1, 2.9) nodes = mapdb.all_nodes(bb=[0.5, 2.5, 1.5, 3.5]) node_ids = set([nid for nid, _ in nodes]) assert node_ids == {2, 22} edges = mapdb.all_edges(bb=[0.5, 2.5, 1.5, 3.5]) edge_tuples = set((nid1, nid2) for nid1, _, nid2, _ in edges) assert edge_tuples == {(1, 2), (2, 1), (3, 2), (2, 22), (22, 2), (22, 33), (22, 4), (4, 22)} nodes = mapdb.nodes_nbrto(2) node_ids = set([nid for nid, _ in nodes]) assert node_ids == {1, 22} edges = mapdb.edges_nbrto((1, 2)) edge_tuples = set((nid1, nid2) for nid1, _, nid2, _ in edges) assert edge_tuples == {(2, 22), (2, 1)} edges = mapdb.edges_nbrto((3, 2)) edge_tuples = set((nid1, nid2) for nid1, _, nid2, _ in edges) assert edge_tuples == {(22, 33), (2, 22), (2, 1)} def test_merge1(): mapdb = create_map1() mapdb.connect_parallelroads() if directory: from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, show_labels=True, show_graph=True, filename=str(directory / "test_parallel_merge.png")) def test_path1(): mapdb = create_map1() mapdb.connect_parallelroads() path = create_path1() states_sol = [(1, 2), (2, 22), (22, 33), (22, 33), (22, 33), (3, 2), (3, 2), (3, 2), (2, 22), (22, 4)] matcher = DistanceMatcher(mapdb, max_dist_init=0.2, obs_noise=0.5, obs_noise_ne=2, dist_noise=0.5, non_emitting_states=True) states, _ = matcher.match(path) if directory: from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_graph=True, show_matching=True, filename=str(directory / "test_parallel_merge.png")) assert states == states_sol, f"Unexpected states: {states}" if __name__ == "__main__": logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) directory = Path(os.environ.get('TESTDIR', Path(__file__).parent)) print(f"Saving files to {directory}") # test_parallel() # test_merge1() # test_path1() test_bb1() ================================================ FILE: tests/test_path.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 """ tests.test_path ~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2017-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import sys import os import logging from pathlib import Path import leuvenmapmatching as mm from leuvenmapmatching.map.inmem import InMemMap from leuvenmapmatching.matcher.simple import SimpleMatcher from leuvenmapmatching.matcher.distance import DistanceMatcher logger = mm.logger directory = None def test_path1(): path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.1, 3.3), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] # path_sol = ['A', ('A', 'B'), 'B', ('B', 'D'), 'D', ('D', 'E'), 'E', ('E', 'F')] path_sol_nodes = ['A', 'B', 'D', 'E', 'F'] mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C"]), "B": ((1, 3), ["A", "C", "D"]), "C": ((2, 2), ["A", "B", "D", "E"]), "D": ((2, 4), ["B", "C", "D", "E"]), "E": ((3, 3), ["C", "D", "F"]), "F": ((3, 5), ["D", "E"]) }, use_latlon=False) matcher = SimpleMatcher(mapdb, max_dist=None, min_prob_norm=None, non_emitting_states=False, only_edges=False) path_pred, _ = matcher.match(path, unique=True) if directory: matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, show_lattice=True, filename=str(directory / "test_path1.png")) # assert path_pred == path_sol, f"Paths not equal:\n{path_pred}\n{path_sol}" nodes_pred = matcher.path_pred_onlynodes assert nodes_pred == path_sol_nodes, f"Nodes not equal:\n{nodes_pred}\n{path_sol_nodes}" def test_path1_dist(): path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.1, 3.3), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] # path_sol = ['A', ('A', 'B'), 'B', ('B', 'D'), 'D', ('D', 'E'), 'E', ('E', 'F')] path_sol_nodes = ['A', 'B', 'D', 'E', 'F'] mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C"]), "B": ((1, 3), ["A", "C", "D"]), "C": ((2, 2), ["A", "B", "D", "E"]), "D": ((2, 4), ["B", "C", "D", "E"]), "E": ((3, 3), ["C", "D", "F"]), "F": ((3, 5), ["D", "E"]) }, use_latlon=False) matcher = DistanceMatcher(mapdb, max_dist=None, min_prob_norm=None, obs_noise=0.5, non_emitting_states=False) matcher.match(path) if directory: from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_path1_dist.png")) nodes_pred = matcher.path_pred_onlynodes assert nodes_pred == path_sol_nodes, f"Nodes not equal:\n{nodes_pred}\n{path_sol_nodes}" def test_path2(): path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.1, 3.3), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] # path_sol = ['A', ('A', 'B'), 'B', ('B', 'D'), 'D', ('D', 'E'), 'E', ('E', 'F')] path_sol_nodes = ['A', 'B', 'D', 'E', 'F'] mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "F", "E", "K", "L"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) matcher = SimpleMatcher(mapdb, max_dist=None, min_prob_norm=0.001, non_emitting_states=False, only_edges=False, max_lattice_width=3) path_pred, _ = matcher.match(path, unique=True) if directory: matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_lattice=True, show_graph=True, filename=str(directory / "test_path2.png")) # assert path_pred == path_sol, "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) nodes_pred = matcher.path_pred_onlynodes assert nodes_pred == path_sol_nodes, f"Nodes not equal:\n{nodes_pred}\n{path_sol_nodes}" def test_path2_inc(): path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.1, 3.3), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] # path_sol = ['A', ('A', 'B'), 'B', ('B', 'D'), 'D', ('D', 'E'), 'E', ('E', 'F')] path_sol_nodes = ['A', 'B', 'D', 'E', 'F'] mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "F", "E", "K", "L"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) ## Phase 1 print('=== PHASE 1 ===') matcher = SimpleMatcher(mapdb, max_dist=None, min_prob_norm=0.001, non_emitting_states=False, only_edges=False, max_lattice_width=1) path_pred, _ = matcher.match(path, unique=True) if directory: matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz with (directory / 'test_path2_inc_1.gv').open('w') as ofile: matcher.lattice_dot(file=ofile, precision=2, render=True) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_lattice=True, show_graph=True, filename=str(directory / "test_path2_inc_1.png")) ## Next phases for phase_nb, phase_width in enumerate([2, 3]): print(f'=== PHASE {phase_nb + 2} ===') path_pred, _ = matcher.increase_max_lattice_width(phase_width, unique=True) if directory: matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz with (directory / f'test_path2_inc_{phase_nb + 2}.gv').open('w') as ofile: matcher.lattice_dot(file=ofile, precision=2, render=True) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_lattice=True, show_graph=True, filename=str(directory / f"test_path2_inc_{phase_nb + 2}.png")) # assert path_pred == path_sol, "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) nodes_pred = matcher.path_pred_onlynodes assert nodes_pred == path_sol_nodes, f"Nodes not equal:\n{nodes_pred}\n{path_sol_nodes}" def test_path2_dist(): path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.1, 3.3), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] path_sol_nodes = ['X', 'A', 'B', 'D', 'E', 'F'] mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "F", "E", "K", "L"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) matcher = DistanceMatcher(mapdb, max_dist=None, min_prob_norm=0.001, obs_noise=0.5, non_emitting_states=False) matcher.match(path, unique=True) if directory: from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_path2_dist.png")) nodes_pred = matcher.path_pred_onlynodes assert nodes_pred == path_sol_nodes, f"Nodes not equal:\n{nodes_pred}\n{path_sol_nodes}" def test_path_outlier(): path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.1, 3.3), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] path_sol = ['A', 'B', 'D', 'C', 'D', 'E', 'F'] path.insert(13, (2.3, 1.8)) mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "F", "E", "K", "L"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) matcher = SimpleMatcher(mapdb, max_dist=None, min_prob_norm=0.0001, max_dist_init=1, obs_noise=0.5, obs_noise_ne=10, non_emitting_states=True) _, last_idx = matcher.match(path, unique=True) path_pred = matcher.path_pred_onlynodes if directory: matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz with (directory / 'lattice.gv').open('w') as ofile: matcher.lattice_dot(file=ofile) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, filename=str(directory / "test_path_outlier.png")) print("Path through lattice:\n" + "\n".join(m.label for m in matcher.lattice_best)) assert last_idx == len(path) - 1 assert path_pred == path_sol, "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) def test_path_outlier2(): path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.1, 3.3), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] path.insert(13, (2.3, -3.0)) mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "F", "E", "K", "L"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) matcher = DistanceMatcher(mapdb, max_dist=None, min_prob_norm=0.1, max_dist_init=1, obs_noise=0.25, obs_noise_ne=1, non_emitting_states=True) _, last_idx = matcher.match(path, unique=True) if directory: # matcher.print_lattice_stats() # matcher.print_lattice() from leuvenmapmatching import visualization as mmviz # with (directory / 'lattice.gv').open('w') as ofile: # matcher.lattice_dot(file=ofile) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, filename=str(directory / "test_path_outlier2.png")) assert last_idx == 12 def test_path_outlier_dist(): path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.1, 3.3), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] path_sol = ['A', 'B', 'D', 'C', 'E', 'F'] path.insert(13, (2.3, 1.8)) mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "F", "E", "K", "L"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) matcher = DistanceMatcher(mapdb, max_dist=None, min_prob_norm=0.0001, max_dist_init=1, obs_noise=0.5, obs_noise_ne=10, non_emitting_states=True) matcher.match(path) path_pred = matcher.path_pred_onlynodes if directory: from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_path_outlier_dist.png")) # TODO: Smoothing the observation distances could eliminate the outlier assert path_pred == path_sol, "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) def test_path3(): path = [(3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] path_sol = ['E', 'F'] mapdb = InMemMap("map", graph={ "E": ((3, 3), ["F"]), "F": ((3, 5), ["E"]), }, use_latlon=False) matcher = SimpleMatcher(mapdb, max_dist=None, min_prob_norm=0.0001, max_dist_init=1, obs_noise=0.25, obs_noise_ne=10, non_emitting_states=True) matcher.match(path, unique=True) path_pred = matcher.path_pred_onlynodes if directory: matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz with (directory / 'lattice.gv').open('w') as ofile: matcher.lattice_dot(file=ofile) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, filename=str(directory / "test_path3.png")) print("Path through lattice:\n" + "\n".join(m.label for m in matcher.lattice_best)) assert path_pred == path_sol, "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) def test_path3_dist(): path = [(3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] path_sol = ['E', 'F'] mapdb = InMemMap("map", graph={ "E": ((3, 3), ["F"]), "F": ((3, 5), ["E"]), }, use_latlon=False) matcher = DistanceMatcher(mapdb, max_dist=None, min_prob_norm=0.0001, max_dist_init=1, obs_noise=0.25, obs_noise_ne=10, non_emitting_states=True) matcher.match(path, unique=True) path_pred = matcher.path_pred_onlynodes if directory: from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, filename=str(directory / "test_path3_dist.png")) print("Path through lattice:\n" + "\n".join(m.label for m in matcher.lattice_best)) assert path_pred == path_sol, "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) def test_path4_dist_inc(): map_con = InMemMap("mymap", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C", "D", "K"]), "C": ((2, 2), ["A", "B", "D", "E", "X", "Y"]), "D": ((2, 4), ["B", "C", "D", "E", "K", "L"]), "E": ((3, 3), ["C", "D", "F", "Y"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C", "E"]), "K": ((1, 5), ["B", "D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.3, 3.5), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] matcher = DistanceMatcher(map_con, max_dist=2, obs_noise=1, min_prob_norm=0.5) matcher.match(path[:5]) if directory: import matplotlib # matplotlib.use('macosx') # print(matplotlib.matplotlib_fname()) # import matplotlib.pyplot as plt # print(plt.get_backend()) from leuvenmapmatching import visualization as mmviz mmviz.plot_map(map_con, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_path4_dist_inc_1.png")) matcher.match(path, expand=True) nodes = matcher.path_pred_onlynodes if directory: from leuvenmapmatching import visualization as mmviz mmviz.plot_map(map_con, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_path4_dist_inc_2.png")) nodes_sol = ['X', 'A', 'B', 'D', 'E', 'F'] assert nodes == nodes_sol, "Nodes not equal:\n{}\n{}".format(nodes, nodes_sol) def test_path4_dist_inc_missing(): map_con = InMemMap("mymap", graph={ "A": ((1, 1), ["B", "C", "X"]), "B": ((1, 3), ["A", "C"]), "C": ((2, 2), ["A", "B", "X", "Y"]), "D": ((2, 4), ["E", "K", "L"]), "E": ((3, 3), ["D", "F"]), "F": ((3, 5), ["D", "E", "L"]), "X": ((2, 0), ["A", "C", "Y"]), "Y": ((3, 1), ["X", "C"]), "K": ((1, 5), ["D", "L"]), "L": ((2, 6), ["K", "D", "F"]) }, use_latlon=False) path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.3, 3.5), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] matcher = DistanceMatcher(map_con, max_dist=0.5, obs_noise=1, min_prob_norm=0.5, max_lattice_width=3) matcher.match(path) if directory: import matplotlib # matplotlib.use('macosx') # print(matplotlib.matplotlib_fname()) # import matplotlib.pyplot as plt # print(plt.get_backend()) from leuvenmapmatching import visualization as mmviz mmviz.plot_map(map_con, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_path4_dist_inc_missing_1.png")) matcher.continue_with_distance() # return matcher.match(path, expand=True) nodes = matcher.path_pred_onlynodes_withjumps if directory: from leuvenmapmatching import visualization as mmviz mmviz.plot_map(map_con, matcher=matcher, show_labels=True, show_matching=True, show_graph=True, filename=str(directory / "test_path4_dist_inc_missing_2.png")) nodes_sol = ['A', 'B', 'C', 'D', 'E', 'F'] assert nodes == nodes_sol, "Nodes not equal:\n{}\n{}".format(nodes, nodes_sol) if __name__ == "__main__": logger.setLevel(logging.INFO) logger.addHandler(logging.StreamHandler(sys.stdout)) directory = Path(os.environ.get('TESTDIR', Path(__file__).parent)) print(f"Saving files to {directory}") # test_path1() test_path1_dist() # test_path2() # test_path2_inc() # test_path2_dist() # test_path_outlier() # test_path_outlier2() # test_path_outlier_dist() # test_path3() # test_path3_dist() # test_path4_dist_inc() # test_path4_dist_inc_missing() ================================================ FILE: tests/test_path_latlon.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 """ tests.test_path_latlon ~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2015-2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import sys import os import pickle import logging from pathlib import Path import pytest import leuvenmapmatching as mm import leuvenmapmatching.visualization as mm_viz from leuvenmapmatching.util.gpx import gpx_to_path from leuvenmapmatching.util.dist_latlon import interpolate_path from leuvenmapmatching.util.openstreetmap import create_map_from_xml, download_map_xml from leuvenmapmatching.matcher.distance import DistanceMatcher from leuvenmapmatching.map.inmem import InMemMap logger = mm.logger this_path = Path(os.path.realpath(__file__)).parent / "rsrc" / "path_latlon" osm_fn = this_path / "osm_downloaded.xml" osm2_fn = this_path / "osm_downloaded2.xml" osm3_fn = this_path / "osm_downloaded3.xml" track_fn = this_path / "route.gpx" # http://users.telenet.be/meirenwi/Leuven%20Stadswandeling%20-%205%20km%20RT.zip track2_fn = this_path / "route2.gpx" track3_fn = this_path / "route3.pgx" zip_fn = this_path / "leuvenmapmatching_testdata2.zip" directory = None def prepare_files(verbose=False, force=False, download_from_osm=False): if download_from_osm: download_map_xml(osm_fn, '4.694933,50.870047,4.709256000000001,50.879628', force=force, verbose=verbose) download_map_xml(osm2_fn, '4.6997666,50.8684188,4.7052813,50.8731718', force=force, verbose=verbose) download_map_xml(osm3_fn, '4.69049,50.86784,4.71604,50.88784', force=force, verbose=verbose) else: if not (osm_fn.exists() and osm2_fn.exists() and osm3_fn.exists() and track_fn.exists() and track2_fn.exists()): import requests url = 'https://people.cs.kuleuven.be/wannes.meert/leuvenmapmatching/leuvenmapmatching_testdata2.zip' logger.debug("Download road_network.zip from kuleuven.be") r = requests.get(url, stream=True) with zip_fn.open('wb') as ofile: for chunk in r.iter_content(chunk_size=1024): if chunk: ofile.write(chunk) import zipfile logger.debug("Unzipping road_network.zip") with zipfile.ZipFile(str(zip_fn), "r") as zip_ref: zip_ref.extractall(str(zip_fn.parent)) def test_path1(use_rtree=False): prepare_files() track = gpx_to_path(track_fn) track = [loc[:2] for loc in track] track = track[:5] track_int = interpolate_path(track, 5) map_con = create_map_from_xml(osm_fn, use_rtree=use_rtree, index_edges=True) matcher = DistanceMatcher(map_con, max_dist=50, obs_noise=50, min_prob_norm=0.1) states, last_idx = matcher.match(track_int) if directory: # matcher.print_lattice_stats() mm_viz.plot_map(map_con, matcher=matcher, use_osm=True, zoom_path=True, show_graph=True, filename=str(directory / "test_path_latlon_path1.png")) assert len(states) == len(track_int), f"Path ({len(track_int)}) not fully matched by best path ({len(states)}), " + \ f"last index = {last_idx}" states_sol = [(2963305939, 249348325), (2963305939, 249348325), (2963305939, 249348325), (2963305939, 249348325), (2963305939, 249348325), (2963305939, 249348325), (249348325, 1545679243), (249348325, 1545679243), (1545679243, 3663115134), (1545679243, 3663115134), (1545679243, 3663115134), (3663115134, 1545679251), (1545679251, 20910628), (1545679251, 20910628), (1545679251, 20910628), (1545679251, 20910628), (20910628, 3663115130)] assert states == states_sol, f"Got states: {states}" def test_path1_serialization(use_rtree=False): prepare_files() track = gpx_to_path(track_fn) track = [loc[:2] for loc in track] track = track[:5] track_int = interpolate_path(track, 5) map_con = create_map_from_xml(osm_fn, use_rtree=use_rtree, index_edges=True) to_serialize = map_con.serialize() map_con.dir = this_path map_con.dump() map_con2 = InMemMap.from_pickle(filename = map_con.dir / (map_con.name + ".pkl")) matcher = DistanceMatcher(map_con2, max_dist=50, obs_noise=50, min_prob_norm=0.1) states, last_idx = matcher.match(track_int) if directory: # matcher.print_lattice_stats() mm_viz.plot_map(map_con2, matcher=matcher, use_osm=True, zoom_path=True, show_graph=True, filename=str(directory / "test_path_latlon_path1.png")) assert len(states) == len(track_int), f"Path ({len(track_int)}) not fully matched by best path ({len(states)}), " + \ f"last index = {last_idx}" states_sol = [(2963305939, 249348325), (2963305939, 249348325), (2963305939, 249348325), (2963305939, 249348325), (2963305939, 249348325), (2963305939, 249348325), (249348325, 1545679243), (249348325, 1545679243), (1545679243, 3663115134), (1545679243, 3663115134), (1545679243, 3663115134), (3663115134, 1545679251), (1545679251, 20910628), (1545679251, 20910628), (1545679251, 20910628), (1545679251, 20910628), (20910628, 3663115130)] assert states == states_sol, f"Got states: {states}" @pytest.mark.skip(reason="Takes a long time") def test_path1_full(): prepare_files() track = gpx_to_path(track_fn) track = [loc[:2] for loc in track] track_int = interpolate_path(track, 5) map_con = create_map_from_xml(osm_fn, include_footways=True, include_parking=True) matcher = DistanceMatcher(map_con, max_dist=50, obs_noise=50, min_prob_norm=0.1) states, last_idx = matcher.match(track_int) if directory: # matcher.print_lattice_stats() mm_viz.plot_map(map_con, matcher=matcher, use_osm=True, zoom_path=True, show_graph=True, filename=str(directory / "test_path_latlon_path1.png")) assert len(states) == len(track_int), f"Path ({len(track_int)}) not fully matched by best path ({len(states)}), " + \ f"last index = {last_idx}" def test_path2_proj(): prepare_files() map_con_latlon = create_map_from_xml(osm2_fn) map_con = map_con_latlon.to_xy() track = [map_con.latlon2yx(p[0], p[1]) for p in gpx_to_path(track2_fn)] matcher = DistanceMatcher(map_con, max_dist=300, max_dist_init=25, min_prob_norm=0.0001, non_emitting_length_factor=0.95, obs_noise=50, obs_noise_ne=50, dist_noise=50, max_lattice_width=5, non_emitting_states=True) states, last_idx = matcher.match(track, unique=False) nodes = matcher.path_pred_onlynodes if directory: matcher.print_lattice_stats() mm_viz.plot_map(map_con, matcher=matcher, path=track, use_osm=False, show_graph=True, show_matching=True, show_labels=5, filename=str(directory / "test_path_latlon_path2_proj.png")) nodes_sol = [2634474831, 1096512242, 3051083902, 1096512239, 1096512241, 1096512240, 1096508366, 1096508372, 16483861, 1096508360, 159656075, 1096508382, 16483862, 3051083898, 16526535, 3060597381, 3060515059, 16526534, 16526532, 1274158119, 16526540, 3060597377, 16526541, 16424220, 1233373340, 613125597, 1076057753] nodes_sol2 = [1096512242, 3051083902, 1096512239, 1096512241, 1096512240, 159654664, 1096508373, 1096508381, 16483859, 1096508369, 159654663, 1096508363, 16483862, 3051083898, 16526535, 3060597381, 3060515059, 16526534, 16526532, 611867918, 3060725817, 16483866, 3060725817, 611867918, 16526532, 1274158119, 16526540, 3060597377, 16526541, 16424220, 1233373340, 613125597, 1076057753] assert (nodes == nodes_sol) or (nodes == nodes_sol2), f"Nodes do not match: {nodes}" def test_path2(): prepare_files() map_con = create_map_from_xml(osm2_fn) track = [(p[0], p[1]) for p in gpx_to_path(track2_fn)] matcher = DistanceMatcher(map_con, max_dist=300, max_dist_init=25, min_prob_norm=0.0001, non_emitting_length_factor=0.95, obs_noise=50, obs_noise_ne=50, dist_noise=50, max_lattice_width=5, non_emitting_states=True) states, last_idx = matcher.match(track, unique=False) nodes = matcher.path_pred_onlynodes if directory: mm_viz.plot_map(map_con, matcher=matcher, nodes=nodes, path=track, z=17, use_osm=True, show_graph=True, show_matching=True, filename=str(directory / "test_path_latlon_path2.png")) nodes_sol = [2634474831, 1096512242, 3051083902, 1096512239, 1096512241, 1096512240, 1096508366, 1096508372, 16483861, 3051083900, 16483864, 16483865, 3060515058, 16526534, 16526532, 1274158119, 16526540, 3060597377, 16526541, 16424220, 1233373340, 613125597, 1076057753] nodes_sol2 = [2634474831, 1096512242, 3051083902, 1096512239, 1096512241, 1096512240, 159654664, 1096508373, 1096508381, 16483859, 1096508369, 159654663, 1096508363, 16483862, 3051083898, 16526535, 3060597381, 3060515059, 16526534, 16526532, 1274158119, 16526540, 3060597377, 16526541, 16424220, 1233373340, 613125597, 1076057753] assert (nodes == nodes_sol) or (nodes == nodes_sol2), f"Nodes do not match: {nodes}" def test_path3(): prepare_files() track = [(50.87881, 4.698930000000001), (50.87899, 4.69836), (50.87905000000001, 4.698110000000001), (50.879000000000005, 4.69793), (50.87903000000001, 4.69766), (50.87906, 4.697500000000001), (50.87908, 4.6973), (50.879110000000004, 4.69665), (50.87854, 4.696420000000001), (50.878440000000005, 4.696330000000001), (50.878370000000004, 4.696140000000001), (50.8783, 4.69578), (50.87832, 4.69543), (50.87767, 4.695530000000001), (50.87763, 4.695080000000001), (50.87758, 4.6948300000000005), (50.877480000000006, 4.69395), (50.877500000000005, 4.693700000000001), (50.877520000000004, 4.69343), (50.877610000000004, 4.692670000000001), (50.87776, 4.6917800000000005), (50.87783, 4.69141), (50.87744000000001, 4.6908900000000004), (50.87736, 4.690790000000001), (50.877300000000005, 4.69078), (50.876650000000005, 4.6907000000000005), (50.87597, 4.69066), (50.875820000000004, 4.69068), (50.87561, 4.6907700000000006), (50.874430000000004, 4.69136), (50.874210000000005, 4.691490000000001), (50.87413, 4.69151), (50.87406000000001, 4.69151), (50.87397000000001, 4.69148), (50.87346, 4.6913800000000005), (50.87279, 4.691260000000001), (50.872490000000006, 4.69115), (50.87259, 4.6908900000000004), (50.87225, 4.690650000000001), (50.872080000000004, 4.6904900000000005), (50.871550000000006, 4.69125), (50.87097000000001, 4.69216), (50.87033, 4.69324), (50.87017, 4.6935400000000005), (50.87012000000001, 4.69373), (50.86997, 4.69406), (50.86981, 4.694520000000001), (50.86943, 4.69585), (50.868970000000004, 4.697500000000001), (50.868770000000005, 4.698130000000001), (50.86863, 4.6985), (50.86844000000001, 4.69899), (50.868140000000004, 4.69977), (50.86802, 4.70023), (50.867920000000005, 4.70078), (50.86787, 4.701180000000001), (50.86784, 4.70195), (50.86786000000001, 4.702310000000001), (50.86791, 4.702870000000001), (50.86836, 4.7052700000000005), (50.86863, 4.7064900000000005), (50.86880000000001, 4.707210000000001), (50.869220000000006, 4.708410000000001), (50.869400000000006, 4.70891), (50.86959, 4.709350000000001), (50.86995, 4.71004), (50.87006, 4.71021), (50.870900000000006, 4.7112300000000005), (50.872260000000004, 4.712890000000001), (50.87308, 4.71389), (50.873430000000006, 4.714300000000001), (50.873560000000005, 4.71441), (50.873740000000005, 4.714530000000001), (50.874280000000006, 4.714740000000001), (50.876250000000006, 4.71544), (50.876490000000004, 4.7155700000000005), (50.876900000000006, 4.7158500000000005), (50.87709, 4.71598), (50.877190000000006, 4.716010000000001), (50.87751, 4.7160400000000005), (50.87782000000001, 4.7160400000000005), (50.87832, 4.71591), (50.87894000000001, 4.71567), (50.87975, 4.71536), (50.88004, 4.71525), (50.8804, 4.715070000000001), (50.88163, 4.71452), (50.881750000000004, 4.71447), (50.8819, 4.714390000000001), (50.882200000000005, 4.71415), (50.882470000000005, 4.7138800000000005), (50.883480000000006, 4.7127300000000005), (50.88552000000001, 4.710470000000001), (50.88624, 4.70966), (50.88635000000001, 4.7096100000000005), (50.886520000000004, 4.709580000000001), (50.88664000000001, 4.7095400000000005), (50.886750000000006, 4.709280000000001), (50.88684000000001, 4.70906), (50.886970000000005, 4.70898), (50.88705, 4.70887), (50.88714, 4.70868), (50.88743, 4.7079), (50.887840000000004, 4.7069), (50.88776000000001, 4.70687), (50.88765, 4.706790000000001), (50.887100000000004, 4.70627), (50.88702000000001, 4.70619), (50.886950000000006, 4.706040000000001), (50.886950000000006, 4.7058800000000005), (50.886970000000005, 4.705620000000001), (50.88711000000001, 4.70417), (50.88720000000001, 4.70324), (50.88723, 4.7027600000000005), (50.88709000000001, 4.70253), (50.886480000000006, 4.70148), (50.88636, 4.70131), (50.886050000000004, 4.70101), (50.88593, 4.70092), (50.885810000000006, 4.700880000000001), (50.88539, 4.7008600000000005), (50.88497, 4.70082), (50.88436, 4.70089), (50.88398, 4.70094), (50.883250000000004, 4.7010700000000005), (50.88271, 4.701160000000001), (50.88136, 4.70159), (50.881130000000006, 4.701790000000001), (50.880930000000006, 4.7020100000000005), (50.88078, 4.70223), (50.88046000000001, 4.70146), (50.88015000000001, 4.70101), (50.880030000000005, 4.700880000000001), (50.87997000000001, 4.70078), (50.879900000000006, 4.70061), (50.87984, 4.70052), (50.879960000000004, 4.70026)] track = track[:30] map_con = create_map_from_xml(osm3_fn) matcher = DistanceMatcher(map_con, max_dist_init=30, max_dist=50, min_prob_norm=0.1, obs_noise=10, obs_noise_ne=20, dist_noise=10, non_emitting_states=True) states, last_idx = matcher.match(track) if directory: # matcher.print_lattice_stats() mm_viz.plot_map(map_con, matcher=matcher, use_osm=True, zoom_path=True, show_graph=False, show_matching=True, filename=str(directory / "test_path_latlon_path3.png")) nodes = matcher.path_pred_onlynodes nodes_sol = [3906576303, 1150903750, 4506996820, 4506996819, 4506996798, 3906576457, 130147477, 3906576346, 231974072, 231974123, 1180606706, 19792164, 19792172, 1180606683, 1180606709, 5236409057, 19792169, 5236409056, 180241961, 180241975, 4506996259, 19792156, 5236409048, 180241625, 180241638, 231953030, 241928030, 241928031, 83796665, 231953028, 1125556965, 1380538625, 1824115892, 4909655515, 16571387, 16737662, 16571388, 179425214, 3705540990, 4567021046] assert nodes == nodes_sol, f"Nodes do not match: {nodes}" if __name__ == "__main__": logger.setLevel(logging.INFO) logger.addHandler(logging.StreamHandler(sys.stdout)) directory = Path(os.environ.get('TESTDIR', Path(__file__).parent)) print(f"Saving files to {directory}") import matplotlib as mpl mpl.use('MacOSX') # test_path1(use_rtree=True) test_path1_serialization(use_rtree=True) # test_path1_full() # test_path2_proj() # test_path2() # test_path3() ================================================ FILE: tests/test_path_onlyedges.py ================================================ #!/usr/bin/env python3 # encoding: utf-8 """ tests.test_path_onlyedges ~~~~~~~~~~~~~~~~~~~~~~~~~ :author: Wannes Meert :copyright: Copyright 2018 DTAI, KU Leuven and Sirris. :license: Apache License, Version 2.0, see LICENSE for details. """ import sys import os import logging from pathlib import Path import leuvenmapmatching as mm from leuvenmapmatching.map.inmem import InMemMap from leuvenmapmatching.matcher.simple import SimpleMatcher logger = mm.logger directory = None def test_path1(): path = [(0.8, 0.7), (0.9, 0.7), (1.1, 1.0), (1.2, 1.5), (1.2, 1.6), (1.1, 2.0), (1.1, 2.3), (1.3, 2.9), (1.2, 3.1), (1.5, 3.2), (1.8, 3.5), (2.0, 3.7), (2.1, 3.3), (2.4, 3.2), (2.6, 3.1), (2.9, 3.1), (3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] path_sol = ['A', 'B', 'D', 'E', 'F'] mapdb = InMemMap("map", graph={ "A": ((1, 1), ["B", "C"]), "B": ((1, 3), ["A", "C", "D"]), "C": ((2, 2), ["A", "B", "D", "E"]), "D": ((2, 4), ["B", "C", "D", "E"]), "E": ((3, 3), ["C", "D", "F"]), "F": ((3, 5), ["D", "E"]) }, use_latlon=False) matcher = SimpleMatcher(mapdb, max_dist=None, min_prob_norm=None, only_edges=True, non_emitting_states=False) matcher.match(path, unique=True) path_pred = matcher.path_pred_onlynodes if directory: print("Lattice best:") for m in matcher.lattice_best: print(m) matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, filename=str(directory / "test_onlyedges_path1.png")) assert path_pred == path_sol, f"Paths not equal:\n{path_pred}\n{path_sol}" def test_path3(): path = [(3.0, 3.2), (3.1, 3.8), (3.0, 4.0), (3.1, 4.3), (3.1, 4.6), (3.0, 4.9)] path_sol = ['E', 'F'] mapdb = InMemMap("map", graph={ "E": ((3, 3), ["F"]), "F": ((3, 5), ["E"]), }, use_latlon=False) matcher = SimpleMatcher(mapdb, max_dist=None, min_prob_norm=0.0001, max_dist_init=1, obs_noise=0.25, obs_noise_ne=10, non_emitting_states=True, only_edges=True) matcher.match(path, unique=True) path_pred = matcher.path_pred_onlynodes if directory: matcher.print_lattice_stats() matcher.print_lattice() from leuvenmapmatching import visualization as mmviz with (directory / 'lattice.gv').open('w') as ofile: matcher.lattice_dot(file=ofile) mmviz.plot_map(mapdb, matcher=matcher, show_labels=True, show_matching=True, filename=str(directory / "test_onlyedges_path3.png")) print("Path through lattice:\n" + "\n".join(m.label for m in matcher.lattice_best)) assert path_pred == path_sol, "Nodes not equal:\n{}\n{}".format(path_pred, path_sol) if __name__ == "__main__": logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) directory = Path(os.environ.get('TESTDIR', Path(__file__).parent)) print(f"Saving files to {directory}") test_path1() # test_path3()