[ { "path": ".github/workflows/check_black.yml", "content": "# This workflow will test the code base using the LATEST version of black\n\n# IMPORTANT: Black is under development. Hence, minor fommatting changes between\n# different version are expected.\n# If this test fails, install the latest version of black and then run black.\n# Preferably, run black only on the files that you have modified.\n# This will faciliate the revision of the proposed changes.\n\nname: Check Black\n\non:\n # Triggers the workflow on push or pull request events but only for the master branch\n push:\n branches: [ master ]\n pull_request:\n branches: [ master ]\n\njobs:\n build:\n\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python 3.11\n uses: actions/setup-python@v5\n with:\n python-version: \"3.11\"\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install black\n\n - name: Black version\n run: black --version\n\n - name: Black check\n working-directory: ${{github.workspace}}\n run: black --check .\n" }, { "path": ".github/workflows/python-publish.yml", "content": "# This workflows will upload a Python Package using Twine when a release is created\n# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries\n\nname: Upload Python Package\n\non:\n release:\n types: [published]\n\njobs:\n deploy:\n\n runs-on: ubuntu-latest\n permissions:\n id-token: write\n\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: '3.x'\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install setuptools wheel numpy cython\n - name: Build\n run: |\n python setup.py sdist\n - name: Publish\n uses: pypa/gh-action-pypi-publish@release/v1\n" }, { "path": ".github/workflows/test_pysteps.yml", "content": "name: Test pysteps\n\non:\n # Triggers the workflow on push or pull request events to the master branch\n push:\n branches:\n - master\n - pysteps-v2\n pull_request:\n branches:\n - master\n - pysteps-v2\n\njobs:\n unit_tests:\n name: Unit Tests (${{ matrix.python-version }}, ${{ matrix.os }})\n runs-on: ${{ matrix.os }}\n\n strategy:\n fail-fast: false\n matrix:\n os: [ \"ubuntu-latest\", \"macos-latest\", \"windows-latest\" ]\n python-version: [\"3.11\", \"3.13\"]\n max-parallel: 6\n\n defaults:\n run:\n shell: bash -l {0}\n\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-python@v5\n with:\n python-version: ${{ matrix.python-version }}\n\n # need headless opencv on Linux, see https://github.com/conda-forge/opencv-feedstock/issues/401\n - name: Install mamba and create environment for Linux\n if: matrix.os == 'ubuntu-latest'\n uses: mamba-org/setup-micromamba@v1\n with:\n # https://github.com/mamba-org/setup-micromamba/issues/225\n micromamba-version: 1.5.10-0\n environment-file: ci/ci_test_env.yml\n environment-name: test_environment\n generate-run-shell: false\n create-args: >-\n python=${{ matrix.python-version }}\n libopencv=*=headless*\n\n - name: Install mamba and create environment (not Linux)\n if: matrix.os != 'ubuntu-latest'\n uses: mamba-org/setup-micromamba@v1\n with:\n # https://github.com/mamba-org/setup-micromamba/issues/225\n micromamba-version: 1.5.10-0\n environment-file: ci/ci_test_env.yml\n environment-name: test_environment\n generate-run-shell: false\n create-args: python=${{ matrix.python-version }}\n\n - name: Install pygrib (not win)\n if: matrix.os != 'windows-latest'\n run: mamba install --quiet pygrib\n\n - name: Install pysteps for MacOS\n if: matrix.os == 'macos-latest'\n working-directory: ${{github.workspace}}\n env:\n CC: gcc-13\n CXX: g++-13\n CXX1X: g++-13\n HOMEBREW_NO_INSTALL_CLEANUP: 1\n run: |\n brew update-reset\n brew update\n gcc-13 --version || brew install gcc@13\n pip install .\n\n - name: Install pysteps\n if: matrix.os != 'macos-latest'\n working-directory: ${{github.workspace}}\n run: pip install .\n\n - name: Download pysteps data\n env:\n PYSTEPS_DATA_PATH: ${{github.workspace}}/pysteps_data\n working-directory: ${{github.workspace}}/ci\n run: python fetch_pysteps_data.py\n\n - name: Check imports\n working-directory: ${{github.workspace}}/pysteps_data\n run: |\n python --version\n python -c \"import pysteps; print(pysteps.__file__)\"\n python -c \"from pysteps import motion\"\n python -c \"from pysteps.motion import vet\"\n python -c \"from pysteps.motion import proesmans\"\n\n - name: Run tests and coverage report\n working-directory: ${{github.workspace}}/pysteps_data\n env:\n PYSTEPSRC: ${{github.workspace}}/pysteps_data/pystepsrc\n run: pytest --pyargs pysteps --cov=pysteps --cov-report=xml --cov-report=term -ra\n\n - name: Upload coverage to Codecov (Linux only)\n if: matrix.os == 'ubuntu-latest'\n uses: codecov/codecov-action@v4\n env:\n OS: ${{ matrix.os }}\n PYTHON: ${{ matrix.python-version }}\n with:\n token: ${{ secrets.CODECOV_TOKEN }}\n files: ${{github.workspace}}/pysteps_data/coverage.xml\n flags: unit_tests\n env_vars: OS,PYTHON\n fail_ci_if_error: true\n verbose: true\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n*.c\n\n# Distribution / packaging\n.Python\n.tox\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Sphinx documentation\ndocs/_build/\ndoc/_build/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# pyenv\n.python-version\n\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Pycharm\n.idea\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# VSCode\n.vscode\n\n# Rope project settings\n.ropeproject\n\n# mypy\n.mypy_cache/\n\n# Mac OS Stuff\n.DS_Store\n\n# Running local tests\n/tmp\n/pysteps/tests/tmp/\n" }, { "path": ".pre-commit-config.yaml", "content": "repos:\n- repo: https://github.com/psf/black\n rev: 26.1.0\n hooks:\n - id: black\n language_version: python3\n" }, { "path": ".readthedocs.yml", "content": "# Read the Docs configuration file\n# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details\n\nversion: 2\n\n# the build.os and build.tools section is mandatory\nbuild:\n os: \"ubuntu-22.04\"\n tools:\n python: \"3.11\"\n\nsphinx:\n configuration: doc/source/conf.py\n\nformats:\n - htmlzip\n\npython:\n install:\n - requirements: requirements.txt\n - requirements: doc/requirements.txt\n - method: pip\n path: .\n" }, { "path": "CITATION.bib", "content": "@Article{gmd-12-4185-2019,\n AUTHOR = {Pulkkinen, S. and Nerini, D. and P\\'erez Hortal, A. A. and Velasco-Forero, C. and Seed, A. and Germann, U. and Foresti, L.},\n TITLE = {Pysteps: an open-source Python library for probabilistic precipitation nowcasting (v1.0)},\n JOURNAL = {Geoscientific Model Development},\n VOLUME = {12},\n YEAR = {2019},\n NUMBER = {10},\n PAGES = {4185--4219},\n URL = {https://gmd.copernicus.org/articles/12/4185/2019/},\n DOI = {10.5194/gmd-12-4185-2019}\n}\n@article{qj.4461,\n AUTHOR = {Imhoff, Ruben O. and De Cruz, Lesley and Dewettinck, Wout and Brauer, Claudia C. and Uijlenhoet, Remko and van Heeringen, Klaas-Jan and Velasco-Forero, Carlos and Nerini, Daniele and Van Ginderachter, Michiel and Weerts, Albrecht H.},\n TITLE = {Scale-dependent blending of ensemble rainfall nowcasts and NWP in the open-source pysteps library},\n JOURNAL = {Quarterly Journal of the Royal Meteorological Society},\n VOLUME = {n/a},\n NUMBER = {n/a},\n YEAR = {2023},\n PAGES ={1--30},\n DOI = {https://doi.org/10.1002/qj.4461},\n URL = {https://rmets.onlinelibrary.wiley.com/doi/abs/10.1002/qj.4461},\n}\n\n" }, { "path": "CONTRIBUTING.rst", "content": "Contributing to pysteps\n=======================\n\nWelcome! Pysteps is a community-driven initiative for developing and\nmaintaining an easy to use, modular, free and open-source Python\nframework for short-term ensemble prediction systems.\n\nThere are many ways to contribute to pysteps:\n\n* contributing bug reports and feature requests\n* contributing documentation\n* code contributions, new features, or bug fixes\n* contribute with usage examples\n\nWorkflow for code contributions\n-------------------------------\n\nWe welcome all kinds of contributions, like documentation updates, bug fixes, or new features.\nThe workflow for the contibutions uses the usual\n`GitHub pull-request flow `_.\n\nIf you have ideas for new contributions to the project, feel free to get in touch with the pysteps community on our\n`pysteps slack `__.\nTo get access to it, you need to ask for an invitation or you can use the automatic invitation page\n`here `__.\nOur slack channel is a great place for preliminary discussions about new features or functionalities.\nAnother place where you can report bugs and suggest new enhancements is the\n`project's issue tracker `_.\n\n\nFirst Time Contributors\n-----------------------\n\nIf you are interested in helping to improve pysteps,\nthe best way to get started is by looking for \"Good First Issue\" in the\n`issue tracker `_.\n\nIn a nutshell, the main steps to follow for contributing to pysteps are:\n\n* Setting up the development environment\n* Fork the repository\n* Install pre-commit hooks\n* Create a new branch for each contribution\n* Read the Code Style guide\n* Work on your changes\n* Test your changes\n* Push to your fork repository and create a new PR in GitHub.\n\n\nSetting up the Development environment\n--------------------------------------\n\nThe recommended way to setup up the developer environment is the Anaconda\n(commonly referred to as Conda).\nConda quickly installs, runs, and updates packages and their dependencies.\nIt also allows you to create, save, load, and switch between different environments on your local computer.\n\nBefore continuing, Mac OSX users also need to install a more recent compiler.\nSee instructions `here `__.\n\nThe developer environment can be created from the file\n`environment_dev.yml `_\nin the project's root directory by running the command::\n\n conda env create -f environment_dev.yml\n\nThis will create the **pysteps_dev** environment that can be activated using::\n\n conda activate pysteps_dev\n\nOnce the environment is activated, the latest version of pysteps can be installed\nin development mode, in such a way that the project appears to be installed,\nbut yet is still editable from the source tree::\n\n pip install -e \n\nTo test if the installation went fine, you can try importing pysteps from the python interpreter by running::\n\n python -c \"import pysteps\"\n\n\nFork the repository\n~~~~~~~~~~~~~~~~~~~\n\nOnce you have set the development environment, the next step is creating your local copy of the repository, where you will commit your modifications.\nThe steps to follow are:\n\n#. Set up Git on your computer.\n#. Create a GitHub account (if you don't have one).\n#. Fork the repository in your GitHub.\n#. Clone a local copy of your fork. For example::\n\n git clone https://github.com//pysteps.git\n\nDone!, now you have a local copy of pysteps git repository.\nIf you are new to GitHub, below you can find a list of helpful tutorials:\n\n- http://rogerdudler.github.io/git-guide/index.html\n- https://www.atlassian.com/git/tutorials\n\nInstall pre-commit hooks\n~~~~~~~~~~~~~~~~~~~~~~~~\n\nAfter setting up your development environment, install the git pre-commit hook by executing the following command in the repository's\nroot::\n\n pre-commit install\n\nThe pre-commit hooks are scripts executed automatically in every commit to identify simple issues with the code.\nWhen an issue is identified (the pre-commit script exits with non-zero status), the hook aborts the commit and prints the error.\nCurrently, pysteps only tests that the code to be committed complies with black's format style.\nIn case that the commit is aborted, you only need to run black in the entire source code.\nThis can be done by running :code:`black .` or :code:`pre-commit run --all-files`.\nThe latter is recommended since it indicates if the commit contained any formatting errors (that are automatically corrected).\nBlack's configuration is stored in the `pyproject.toml` file to ensure that the same configuration is used in every development environment.\nThis configuration is automatically loaded when black is run from any directory in the\npysteps project.\n\nIMPORTANT: Periodically update the black version used in the pre-commit hook by running::\n\n pre-commit autoupdate\n\nFor more information about git hooks and the pre-commit package, see:\n\n- https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks\n- https://pre-commit.com/\n\n\nCreate a new branch\n~~~~~~~~~~~~~~~~~~~\n\nAs a collaborator, all the new contributions you want should be made in a new branch under your forked repository.\nWorking on the master branch is reserved for Core Contributors only.\nCore Contributors are developers that actively work and maintain the repository.\nThey are the only ones who accept pull requests and push commits directly to the pysteps repository.\n\nFor more information on how to create and work with branches, see\n`\"Branches in a Nutshell\" `__ in the Git documentation\n\n\nCode Style\n----------\n\nWe strongly suggest following the\n`PEP8 coding standards `__.\nSince PEP8 is a set of recommendations, these are the most important good coding practices for the pysteps project:\n\n* Always use four spaces for indentation (don’t use tabs).\n* Max line-length: 88 characters (note that we don't use the PEP8's 79 value). Enforced by `black`.\n* Always indent wrapped code for readability. Enforced by `black`.\n* Avoid extraneous whitespace. Enforced by `black`.\n* Don’t use whitespace to line up assignment operators (=, :). Enforced by `black`.\n* Avoid writing multiple statements in the same line.\n* Naming conventions should follow the recomendations from\n the `Google's python style guide `__, summarized as follows:\n\n .. raw:: html\n\n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n\n
TypePublicInternal
Packageslower_with_under
Moduleslower_with_under_lower_with_under
ClassesCapWords_CapWords
ExceptionsCapWords
Functionslower_with_under()_lower_with_under()
Global/Class ConstantsCAPS_WITH_UNDER_CAPS_WITH_UNDER
Global/Class Variableslower_with_under_lower_with_under
Instance Variableslower_with_under_lower_with_under (protected)
Method Nameslower_with_under()_lower_with_under() (protected)
Function/Method Parameterslower_with_under
Local Variableslower_with_under
\n\n(source: `Section 3.16.4, Google's python style guide `__)\n\n- If you need to ignore part of the variables returned by a function,\n use \"_\" (single underscore) or __ (double underscore)::\n\n precip, __, metadata = import_bom_rf3('example_file.bom')\n precip2, _, metadata2 = import_bom_rf3('example_file2.bom')\n\n\n- Zen of Python (`PEP 20 `__), the guiding principles for Python’s\n design::\n\n >>> import this\n The Zen of Python, by Tim Peters\n\n Beautiful is better than ugly.\n Explicit is better than implicit.\n Simple is better than complex.\n Complex is better than complicated.\n Flat is better than nested.\n Sparse is better than dense.\n Readability counts.\n Special cases aren't special enough to break the rules.\n Although practicality beats purity.\n Errors should never pass silently.\n Unless explicitly silenced.\n In the face of ambiguity, refuse the temptation to guess.\n There should be one-- and preferably only one --obvious way to do it.\n Although that way may not be obvious at first unless you're Dutch.\n Now is better than never.\n Although never is often better than *right* now.\n If the implementation is hard to explain, it's a bad idea.\n If the implementation is easy to explain, it may be a good idea.\n Namespaces are one honking great idea -- let's do more of those!\n\nFor more suggestions on good coding practices for python, check these guidelines:\n\n- `The Hitchhiker's Guide to Python `__\n- `Google's python style guide `__\n- `PEP8 `__\n\n\n**Using Black auto-formatter**\n\nTo ensure a minimal style consistency, we use\n`black `__ to auto-format to the source code.\nThe black configuration used in the pysteps project is defined in the pyproject.toml,\nand it is automatically detected by black.\n\nBlack can be installed using any of the following::\n\n conda install black\n\n #For the latest version:\n conda install -c conda-forge black\n\n pip install black\n\nCheck the `official documentation `__\nfor more information.\n\n**Docstrings**\n\nEvery module, function, or class must have a docstring that describe its\npurpose and how to use it. The docstrings follows the conventions described in the\n`PEP 257 `__\nand the\n`Numpy's docstrings format `__.\n\nHere is a summary of the most important rules:\n\n- Always use triple quotes for doctrings, even if it fits a single line.\n- For one-line docstring, end the phrase with a period.\n- Use imperative mood for all docstrings (\"\"\"Return some value.\"\"\") rather than descriptive mood\n (\"\"\"Returns some value.\"\"\").\n\nHere is an example of a docstring::\n\n def adjust_lag2_corrcoef1(gamma_1, gamma_2):\n \"\"\"\n A simple adjustment of lag-2 temporal autocorrelation coefficient to\n ensure that the resulting AR(2) process is stationary when the parameters\n are estimated from the Yule-Walker equations.\n\n Parameters\n ----------\n gamma_1 : float\n Lag-1 temporal autocorrelation coeffient.\n gamma_2 : float\n Lag-2 temporal autocorrelation coeffient.\n\n Returns\n -------\n out : float\n The adjusted lag-2 correlation coefficient.\n \"\"\"\n\n\nContributions guidelines\n------------------------\n\nThe collaborator guidelines used in pysteps were largely inspired by those of the\n`MyPy project `__.\n\nCollaborators guidelines\n~~~~~~~~~~~~~~~~~~~~~~~~\n\nAs a collaborator, all your new contributions should be made in a new branch under your forked repository.\nWorking on the master branch is reserved for Core Contributors only to submit small changes only.\nCore Contributors are developers that actively work and maintain the repository.\nThey are the only ones who accept pull requests and push commits directly to\nthe **pysteps** repository.\n\n**IMPORTANT**\nHowever, for contribution requires a significant amount of work, we strongly suggest opening a new issue with\nthe **enhancement** or **discussion** tag to encourage discussions.\nThe discussions will help clarify the best way to approach the suggested changes or raise potential concerns.\n\nFor code contributions, collaboratos can use the usual\n`GitHub pull-request flow `__.\nOnce your proposed changes are ready, you need to create a pull request (PR) from your fork in your GitHub account.\nAfterward, core contributors will review your proposed changes, provide feedback in the PR discussion, and maybe,\nrequest changes to the code. Once the PR is ready, a Core Developer will merge the changes into the main branch.\n\n**Important:**\nIt is strongly suggested that each PR only addresses a single objective (e.g., fix a bug, improve documentation, etc.).\nThis will help to reduce the time needed to process the PR. For changes outside the PR's objectives, we highly\nrecommend opening a new PR.\n\n\nTesting your changes\n~~~~~~~~~~~~~~~~~~~~\n\nBefore committing changes or creating pull requests, check that all the tests in the pysteps suite pass.\nSee the `Testing pysteps `__\nfor detailed instruction to run the tests.\n\nAlthough it is not strictly needed, we suggest creating minimal tests for new contributions to ensure that it achieves\nthe desired behavior. Pysteps uses the pytest framework that it is easy to use and also supports complex functional\ntesting for applications and libraries.\nCheck the `pytests official documentation `_ for more information.\n\nThe tests should be placed under the\n`pysteps.tests `_\nmodule.\nThe file should follow the **test_*.py** naming convention and have a\ndescriptive name.\n\nA quick way to get familiar with the pytest syntax and the testing procedures\nis checking the python scripts present in the pysteps test module.\n\nCore developer guidelines\n~~~~~~~~~~~~~~~~~~~~~~~~~\n\nWorking directly on the master branch is discouraged and is reserved only\nfor small changes and updates that do not compromise the stability of the code.\nThe *master* branch is a production branch that is ready to be deployed\n(cloned, installed, and ready to use).\nIn consequence, this master branch is meant to be stable.\n\nThe pysteps repository uses the GitHub Actions service to run tests every time you commit to GitHub.\nIn that way, your modifications along with the entire library are tested.\n\nPushing untested or work-in-progress changes to the master branch can potentially introduce bugs or break the stability of the package.\nSince the tests triggered by a commit to the master branch take around 20 minutes, any errors introduced there\nwill be noticed after the stablility of the master branch was compromised.\nIn addition, other developers start working on a new feature from master from a potentially broken state.\n\nInstead, it is recommended to work on each new feature in its own branch, which can be pushed to the central repository\nfor backup/collaboration. When you’re done with the feature's development work, you can merge the feature branch into the\nmaster or submit a Pull Request. This approach has two main advantages:\n\n- Every commit on the feature branch is tested via GitHub Actions.\n If the tests fail, they do not affect the **master** branch.\n\n- Once the changes are finished and the tests passed, the commits history can be squashed into a single commit and\n then merged into the master branch. Squashing the commits helps to keep a clean commit history in the main branch.\n\n\nProcessing pull requests\n~~~~~~~~~~~~~~~~~~~~~~~~\n\n.. _`Squash and merge`: https://github.com/blog/2141-squash-your-commits\n\nTo process the pull request, we follow similar rules to those used in the\n`mypy developer guidelines `_:\n\n* Always wait for tests to pass before merging PRs.\n* Always use \"`Squash and merge`_\" to merge PRs.\n* Make sure that the subject of the commit message summarizes the objective of the PR and does not finish with a dot.\n* Write a new commit message before merging that provides a detailed description of the changes introduced by the PR.\n Try to keep the maximum line length under 80 characters, spplitting lines if necessary.\n **IMPORTANT:** Make sure that the commit message doesn't contain the branch's commit history!\n Also, if the PR fixes an issue, mention this explicitly.\n* Use the imperative mood in the subject line (e.g. \"Fix typo in README\").\n\nAfter the PR is merged, the merged branch can be safely deleted.\n\nPreparing a new release\n~~~~~~~~~~~~~~~~~~~~~~~\n\nCore developers should follow the steps to prepare a new release (version):\n\n1. Before creating the actual release in GitHub, be sure that every item in the following checklist was followed:\n\n * In the file setup.py, update the **version=\"X.X.X\"** keyword in the setup\n function.\n * Update the version in PKG-INFO file.\n * If new dependencies were added to pysteps since the last release, add\n them to the **environment.yml, requirements.txt**, and\n **requirements_dev.txt** files.\n#. Create a new release in GitHub following\n `these guidelines `_.\n Include a detailed changelog in the release.\n#. Generating the source distribution for new pysteps version and upload it to\n the `Python Package Index `_ (PyPI).\n See `Packaging the pysteps project `__\n for a detailed description of this process.\n#. Update the conda-forge pysteps-feedstock following this guidelines:\n `Updating the conda-forge pysteps-feedstock `__\n\n\nCredits\n-------\n\nThis document was based on contributors guides of two Python\nopen-source projects:\n\n* Py-Art_: Copyright (c) 2013, UChicago Argonne, LLC.\n `License `_.\n* mypy_: Copyright (c) 2015-2016 Jukka Lehtosalo and contributors.\n `MIT License `_.\n* Official github documentation (https://help.github.com)\n\n.. _Py-Art: https://github.com/ARM-DOE/pyart\n.. _mypy: https://github.com/python/mypy\n" }, { "path": "LICENSE", "content": "BSD 3-Clause License\n\nCopyright (c) 2019, PySteps developers\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n* Neither the name of the copyright holder nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "MANIFEST.in", "content": "include LICENSE *.txt *.rst\ninclude pysteps/pystepsrc\ninclude pysteps/pystepsrc_schema.json\ninclude pysteps/io/mch_lut_8bit_Metranet_AZC_V104.txt\ninclude pysteps/io/mch_lut_8bit_Metranet_v103.txt\nrecursive-include pysteps *.pyx\ninclude pyproject.toml\n\n " }, { "path": "PKG-INFO", "content": "Metadata-Version: 1.2\nName: pysteps\nVersion: 1.20.0\nSummary: Python framework for short-term ensemble prediction systems\nHome-page: http://pypi.python.org/pypi/pysteps/\nLicense: LICENSE\nDescription: =======\npySteps\n=======\nThe pysteps initiative is a community that develops and maintains an easy to\nuse, modular, free and open-source python framework for short-term ensemble\nprediction systems.\n\nThe focus is on probabilistic nowcasting of radar precipitation fields,\nbut pysteps is designed to allow a wider range of uses.\n\nPlatform: UNKNOWN\n" }, { "path": "README.rst", "content": "pysteps - Python framework for short-term ensemble prediction systems\n=====================================================================\n\n.. start-badges\n\n.. list-table::\n :stub-columns: 1\n :widths: 10 90\n\n * - docs\n - |stable| |colab| |gallery|\n * - status\n - |test| |docs| |codecov| |codacy| |black|\n * - package\n - |github| |conda| |pypi| |zenodo|\n * - community\n - |contributors| |downloads| |license|\n\n\n.. |docs| image:: https://readthedocs.org/projects/pysteps/badge/?version=latest\n :alt: Documentation Status\n :target: https://pysteps.readthedocs.io/\n\n.. |test| image:: https://github.com/pySTEPS/pysteps/workflows/Test%20pysteps/badge.svg\n :alt: Test pysteps\n :target: https://github.com/pySTEPS/pysteps/actions?query=workflow%3A\"Test+Pysteps\"\n\n.. |black| image:: https://github.com/pySTEPS/pysteps/workflows/Check%20Black/badge.svg\n :alt: Check Black\n :target: https://github.com/pySTEPS/pysteps/actions?query=workflow%3A\"Check+Black\"\n\n.. |codecov| image:: https://codecov.io/gh/pySTEPS/pysteps/branch/master/graph/badge.svg\n :alt: Coverage\n :target: https://codecov.io/gh/pySTEPS/pysteps\n\n.. |github| image:: https://img.shields.io/github/release/pySTEPS/pysteps.svg\n :target: https://github.com/pySTEPS/pysteps/releases/latest\n :alt: Latest github release\n\n.. |conda| image:: https://anaconda.org/conda-forge/pysteps/badges/version.svg\n :target: https://anaconda.org/conda-forge/pysteps\n :alt: Anaconda Cloud\n\n.. |pypi| image:: https://badge.fury.io/py/pysteps.svg\n :target: https://pypi.org/project/pysteps/\n :alt: Latest PyPI version\n\n.. |license| image:: https://img.shields.io/badge/License-BSD%203--Clause-blue.svg\n :alt: License\n :target: https://opensource.org/licenses/BSD-3-Clause\n\n.. |contributors| image:: https://img.shields.io/github/contributors/pySTEPS/pysteps\n :alt: GitHub contributors\n :target: https://github.com/pySTEPS/pysteps/graphs/contributors\n\n.. |downloads| image:: https://img.shields.io/conda/dn/conda-forge/pysteps\n :alt: Conda downloads\n :target: https://anaconda.org/conda-forge/pysteps\n\n.. |colab| image:: https://colab.research.google.com/assets/colab-badge.svg\n :alt: My first nowcast\n :target: https://colab.research.google.com/github/pySTEPS/pysteps/blob/master/examples/my_first_nowcast.ipynb\n\n.. |gallery| image:: https://img.shields.io/badge/example-gallery-blue.svg\n :alt: pysteps example gallery\n :target: https://pysteps.readthedocs.io/en/stable/auto_examples/index.html\n \n.. |stable| image:: https://img.shields.io/badge/docs-stable-blue.svg\n :alt: pysteps documentation\n :target: https://pysteps.readthedocs.io/en/stable/\n \n.. |codacy| image:: https://api.codacy.com/project/badge/Grade/6cff9e046c5341a4afebc0347362f8de\n :alt: Codacy Badge\n :target: https://app.codacy.com/gh/pySTEPS/pysteps?utm_source=github.com&utm_medium=referral&utm_content=pySTEPS/pysteps&utm_campaign=Badge_Grade\n\n.. |zenodo| image:: https://zenodo.org/badge/140263418.svg\n :alt: DOI\n :target: https://zenodo.org/badge/latestdoi/140263418\n\n.. end-badges\n\nWhat is pysteps?\n================\n\nPysteps is an open-source and community-driven Python library for probabilistic precipitation nowcasting, i.e. short-term ensemble prediction systems.\n\nThe aim of pysteps is to serve two different needs. The first is to provide a modular and well-documented framework for researchers interested in developing new methods for nowcasting and stochastic space-time simulation of precipitation. The second aim is to offer a highly configurable and easily accessible platform for practitioners ranging from weather forecasters to hydrologists.\n\nThe pysteps library supports standard input/output file formats and implements several optical flow methods as well as advanced stochastic generators to produce ensemble nowcasts. In addition, it includes tools for visualizing and post-processing the nowcasts and methods for deterministic, probabilistic, and neighbourhood forecast verification.\n\n\nQuick start\n-----------\n\nUse pysteps to compute and plot a radar extrapolation nowcast in Google Colab with `this interactive notebook `_.\n\nInstallation\n============\n\nThe recommended way to install pysteps is with `conda `_ from the conda-forge channel::\n\n $ conda install -c conda-forge pysteps\n\nMore details can be found in the `installation guide `_.\n\nUsage\n=====\n\nHave a look at the `gallery of examples `__ to get a good overview of what pysteps can do.\n\nFor a more detailed description of all the available methods, check the `API reference `_ page.\n\nExample data\n============\n\nA set of example radar data is available in a separate repository: `pysteps-data `_.\nMore information on how to download and install them is available `here `_.\n\nContributions\n=============\n\n*We welcome contributions!*\n\nFor feedback, suggestions for developments, and bug reports please use the dedicated `issues page `_.\n\nFor more information, please read our `contributors guidelines `_.\n\n\nReference publications\n======================\n\nThe overall library is described in\n\nPulkkinen, S., D. Nerini, A. Perez Hortal, C. Velasco-Forero, U. Germann,\nA. Seed, and L. Foresti, 2019: Pysteps: an open-source Python library for\nprobabilistic precipitation nowcasting (v1.0). *Geosci. Model Dev.*, **12 (10)**,\n4185–4219, doi:`10.5194/gmd-12-4185-2019 `_.\n\nWhile the more recent blending module is described in\n\nImhoff, R.O., L. De Cruz, W. Dewettinck, C.C. Brauer, R. Uijlenhoet, K-J. van Heeringen, \nC. Velasco-Forero, D. Nerini, M. Van Ginderachter, and A.H. Weerts, 2023:\nScale-dependent blending of ensemble rainfall nowcasts and NWP in the open-source\npysteps library. *Q J R Meteorol Soc.*, 1-30,\ndoi: `10.1002/qj.4461 `_.\n\n\nContributors\n============\n\n.. image:: https://contrib.rocks/image?repo=pySTEPS/pysteps\n :target: https://github.com/pySTEPS/pysteps/graphs/contributors\n" }, { "path": "ci/ci_test_env.yml", "content": "# pysteps development environment\nname: test_environment\nchannels:\n - conda-forge\n - defaults\ndependencies:\n - python>=3.11\n - pip\n - mamba\n # Minimal dependencies\n - numpy\n - cython\n - jsmin\n - jsonschema\n - matplotlib\n - netCDF4\n - opencv\n - pillow\n - pyproj\n - scipy\n # Optional dependencies\n - dask\n - pyfftw\n - cartopy\n - h5py\n - PyWavelets\n - pandas\n - scikit-image\n - scikit-learn\n - rasterio\n - gdal\n # Test dependencies\n - pytest\n - pytest-cov\n - pip:\n - cookiecutter\n" }, { "path": "ci/fetch_pysteps_data.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nScript used to install the pysteps data in a test environment and set a pystepsrc\nconfiguration file that points to that data.\n\nThe test data is downloaded in the `PYSTEPS_DATA_PATH` environmental variable.\n\nAfter this script is run, the `PYSTEPSRC` environmental variable should be set to\nPYSTEPSRC=$PYSTEPS_DATA_PATH/pystepsrc for pysteps to use that configuration file.\n\"\"\"\n\nimport os\n\nfrom pysteps.datasets import create_default_pystepsrc, download_pysteps_data\n\ntox_test_data_dir = os.environ[\"PYSTEPS_DATA_PATH\"]\n\ndownload_pysteps_data(tox_test_data_dir, force=True)\n\ncreate_default_pystepsrc(\n tox_test_data_dir, config_dir=tox_test_data_dir, file_name=\"pystepsrc\"\n)\n" }, { "path": "ci/test_plugin_support.py", "content": "# -*- coding: utf-8 -*-\n\n\"\"\"\nScript to test the plugin support.\n\nThis script assumes that a package created with the default pysteps plugin template\n(and using the default values) is installed.\n\nhttps://github.com/pySTEPS/cookiecutter-pysteps-plugin\n\n\"\"\"\n\nfrom pysteps import io\n\nprint(\"Testing plugin support: \", end=\"\")\nassert hasattr(io.importers, \"import_institution_name\")\n\nassert \"institution_name\" in io.interface._importer_methods\n\nfrom pysteps.io.importers import import_institution_name\n\nimport_institution_name(\"filename\")\nprint(\"PASSED\")\n" }, { "path": "doc/.gitignore", "content": "_build/\ngenerated\nauto_examples" }, { "path": "doc/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nSPHINXPROJ = pysteps\nSOURCEDIR = source\nBUILDDIR = _build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n" }, { "path": "doc/_static/pysteps.css", "content": "\n.section h1 {\n border-bottom: 2px solid #0099ff;\n display: inline-block;\n}\n\n.section h2 {\n border-bottom: 2px solid #ccebff;\n display: inline-block;\n}\n\n/* override table width restrictions */\n@media screen and (min-width: 767px) {\n\n .wy-table-responsive table td {\n /* !important prevents the common CSS stylesheets from overriding\n this as on RTD they are loaded after this stylesheet */\n white-space: normal !important;\n }\n\n .wy-table-responsive {\n overflow: visible !important;\n }\n}" }, { "path": "doc/_templates/layout.html", "content": "{% extends \"!layout.html\" %}\n{% set css_files = css_files + [\"_static/pysteps.css\"] %}" }, { "path": "doc/make.bat", "content": "@ECHO OFF\r\n\r\npushd %~dp0\r\n\r\nREM Command file for Sphinx documentation\r\n\r\nif \"%SPHINXBUILD%\" == \"\" (\r\n\tset SPHINXBUILD=sphinx-build\r\n)\r\nset SOURCEDIR=.\r\nset BUILDDIR=_build\r\nset SPHINXPROJ=pysteps\r\n\r\nif \"%1\" == \"\" goto help\r\n\r\n%SPHINXBUILD% >NUL 2>NUL\r\nif errorlevel 9009 (\r\n\techo.\r\n\techo.The 'sphinx-build' command was not found. Make sure you have Sphinx\r\n\techo.installed, then set the SPHINXBUILD environment variable to point\r\n\techo.to the full path of the 'sphinx-build' executable. Alternatively you\r\n\techo.may add the Sphinx directory to PATH.\r\n\techo.\r\n\techo.If you don't have Sphinx installed, grab it from\r\n\techo.http://sphinx-doc.org/\r\n\texit /b 1\r\n)\r\n\r\n%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%\r\ngoto end\r\n\r\n:help\r\n%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%\r\n\r\n:end\r\npopd\r\n" }, { "path": "doc/rebuild_docs.sh", "content": "# Build documentation from scratch.\n\nrm -r source/generated &> /dev/null\nrm -r source/auto_examples &> /dev/null\n\nmake clean\n\nmake html\n" }, { "path": "doc/requirements.txt", "content": "# Additional requirements related to the documentation build only\nsphinx\nsphinxcontrib.bibtex\nsphinx-book-theme\nsphinx_gallery\nscikit-image\nscikit-learn\npandas\ngit+https://github.com/pySTEPS/pysteps-nwp-importers.git@main#egg=pysteps_nwp_importers\npygrib\nh5py\n" }, { "path": "doc/source/conf.py", "content": "# -*- coding: utf-8 -*-\n\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\nimport os\nimport subprocess\nimport sys\nfrom datetime import datetime\n\nimport json\nfrom jsmin import jsmin\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\nif \"READTHEDOCS\" not in os.environ:\n sys.path.insert(1, os.path.abspath(\"../../\"))\n\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#\nneeds_sphinx = \"1.6\"\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n \"sphinx.ext.autodoc\",\n \"sphinx.ext.autosummary\",\n \"sphinx.ext.coverage\",\n \"sphinx.ext.mathjax\",\n \"sphinx.ext.napoleon\",\n \"sphinxcontrib.bibtex\",\n \"sphinx_gallery.gen_gallery\",\n]\n\nbibtex_bibfiles = [\"references.bib\"]\n\n# numpydoc_show_class_members = False\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = [\"_templates\"]\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n#\n# source_suffix = ['.rst', '.md']\nsource_suffix = \".rst\"\n\n# The master toctree document.\nmaster_doc = \"index\"\n\n# General information about the project.\nproject = \"pysteps\"\ncopyright = f\"2018-{datetime.now():%Y}, pysteps developers\"\nauthor = \"pysteps developers\"\n\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\ndef get_version():\n \"\"\"Returns project version as string from 'git describe' command.\"\"\"\n\n from subprocess import check_output\n\n _version = check_output([\"git\", \"describe\", \"--tags\", \"--always\"])\n\n if _version:\n return _version.decode(\"utf-8\")\n else:\n return \"X.Y\"\n\n\n# The short X.Y version.\nversion = get_version().lstrip(\"v\").rstrip().split(\"-\")[0]\n\n# The full version, including alpha/beta/rc tags.\nrelease = version\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = \"en\"\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This patterns also effect to html_static_path and html_extra_path\nexclude_patterns = [\"_build\", \"Thumbs.db\", \".DS_Store\"]\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = \"sphinx\"\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = False\n\n\n# -- Read the Docs build --------------------------------------------------\n\n\ndef set_root():\n fn = os.path.abspath(os.path.join(\"..\", \"..\", \"pysteps\", \"pystepsrc\"))\n with open(fn, \"r\") as f:\n rcparams = json.loads(jsmin(f.read()))\n\n for key, value in rcparams[\"data_sources\"].items():\n new_path = os.path.join(\"..\", \"..\", \"pysteps-data\", value[\"root_path\"])\n new_path = os.path.abspath(new_path)\n\n value[\"root_path\"] = new_path\n\n fn = os.path.abspath(os.path.join(\"..\", \"..\", \"pystepsrc.rtd\"))\n with open(fn, \"w\") as f:\n json.dump(rcparams, f, indent=4)\n\n\nif \"READTHEDOCS\" in os.environ:\n repourl = \"https://github.com/pySTEPS/pysteps-data.git\"\n dir = os.path.join(os.getcwd(), \"..\", \"..\", \"pysteps-data\")\n dir = os.path.abspath(dir)\n subprocess.check_call([\"rm\", \"-rf\", dir])\n subprocess.check_call([\"git\", \"clone\", repourl, dir])\n os.environ[\"PYSTEPS_DATA_PATH\"] = dir\n set_root()\n pystepsrc = os.path.abspath(os.path.join(\"..\", \"..\", \"pystepsrc.rtd\"))\n os.environ[\"PYSTEPSRC\"] = pystepsrc\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\n# html_theme = 'alabaster'\n# html_theme = 'classic'\nhtml_theme = \"sphinx_book_theme\"\nhtml_title = \"\"\n\nhtml_context = {\n \"github_user\": \"pySTEPS\",\n \"github_repo\": \"pysteps\",\n \"github_version\": \"master\",\n \"doc_path\": \"doc\",\n}\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#\nhtml_theme_options = {\n \"repository_url\": \"https://github.com/pySTEPS/pysteps\",\n \"repository_branch\": \"master\",\n \"path_to_docs\": \"doc/source\",\n \"use_edit_page_button\": True,\n \"use_repository_button\": True,\n \"use_issues_button\": True,\n}\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\nhtml_logo = \"../_static/pysteps_logo.png\"\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = [\"../_static\"]\nhtml_css_files = [\"../_static/pysteps.css\"]\n\n# Custom sidebar templates, must be a dictionary that maps document names\n# to template names.\n#\n# This is required for the alabaster theme\n# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars\n# html_sidebars = {\n# \"**\": [\n# \"relations.html\", # needs 'show_related': True theme option to display\n# \"searchbox.html\",\n# ]\n# }\n\nhtml_domain_indices = True\n\nautosummary_generate = True\n\n# -- Options for HTMLHelp output ------------------------------------------\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = \"pystepsdoc\"\n\n# -- Options for LaTeX output ---------------------------------------------\n\n# This hack is taken from numpy (https://github.com/numpy/numpy/blob/master/doc/source/conf.py).\nlatex_preamble = r\"\"\"\n \\usepackage{amsmath}\n \\DeclareUnicodeCharacter{00A0}{\\nobreakspace}\n \n % In the parameters section, place a newline after the Parameters\n % header\n \\usepackage{expdlist}\n \\let\\latexdescription=\\description\n \\def\\description{\\latexdescription{}{} \\breaklabel}\n \n % Make Examples/etc section headers smaller and more compact\n \\makeatletter\n \\titleformat{\\paragraph}{\\normalsize\\py@HeaderFamily}%\n {\\py@TitleColor}{0em}{\\py@TitleColor}{\\py@NormalColor}\n \\titlespacing*{\\paragraph}{0pt}{1ex}{0pt}\n \\makeatother\n \n % Fix footer/header\n \\renewcommand{\\chaptermark}[1]{\\markboth{\\MakeUppercase{\\thechapter.\\ #1}}{}}\n \\renewcommand{\\sectionmark}[1]{\\markright{\\MakeUppercase{\\thesection.\\ #1}}}\n\"\"\"\n\nlatex_elements = {\n \"papersize\": \"a4paper\",\n \"pointsize\": \"10pt\",\n \"preamble\": latex_preamble,\n # Latex figure (float) alignment\n #\n # 'figure_align': 'htbp',\n}\n\nlatex_domain_indices = False\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, \"pysteps.tex\", \"pysteps reference\", author, \"manual\"),\n]\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [(master_doc, \"pysteps\", \"pysteps reference\", [author], 1)]\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (\n master_doc,\n \"pysteps\",\n \"pysteps reference\",\n author,\n \"pysteps\",\n \"One line description of project.\",\n \"Miscellaneous\",\n ),\n]\n\n# -- Options for Sphinx-Gallery -------------------------------------------\n\n# The configuration dictionary for Sphinx-Gallery\n\nsphinx_gallery_conf = {\n \"examples_dirs\": \"../../examples\", # path to your example scripts\n \"gallery_dirs\": \"auto_examples\", # path where to save gallery generated examples\n \"filename_pattern\": r\"/*\\.py\", # Include all the files in the examples dir\n}\n" }, { "path": "doc/source/developer_guide/build_the_docs.rst", "content": ".. _build_the_docs:\n\n=================\nBuilding the docs\n=================\n\nThe pysteps documentations is build using\n`Sphinx `_,\na tool that makes it easy to create intelligent and beautiful documentation\n\nThe documentation is located in the **doc** folder in the pysteps repo.\n\nAutomatic build\n---------------\n\nThe simplest way to build the documentation is using tox and the tox-conda\nplugin (conda needed).\nTo install these packages activate your conda development environment and run::\n\n conda install -c conda-forge tox tox-conda\n\nThen, to build the documentation, from the repo's root run::\n\n tox -e docs\n\nThis will create a conda environment will all the necessary dependencies and the\ndata needed to create the examples.\n\n\nManual build\n------------\nTo build the docs you need to need to satisfy a few more dependencies\nrelated to Sphinx that are specified in the doc/requirements.txt file:\n\n- sphinx\n- numpydoc\n- sphinxcontrib.bibtex\n- sphinx-book-theme\n- sphinx_gallery\n\nYou can install these packages running `pip install -r doc/requirements.txt`.\n\nIn addition to this requirements, to build the example gallery in the\ndocumentation the example pysteps-data is needed. To download and install this\ndata see the installation instructions in the :ref:`example_data` section.\n\nOnce these requirements are met, to build the documentation, in the **doc**\nfolder run::\n\n make html\n\nThis will build the documentation along with the example gallery.\n\nThe build documentation (html web page) will be available in\n**doc/_build/html/**.\nTo correctly visualize the documentation, you need to set up and run a local\nHTTP server. To do that, in the **doc/_build/html/** directory run::\n\n python -m http.server\n\nThis will set up a local HTTP server on 0.0.0.0 port 8000.\nTo see the built documentation open the following url in the browser:\nhttp://0.0.0.0:8000/\n" }, { "path": "doc/source/developer_guide/contributors_guidelines.rst", "content": ".. _contributor_guidelines:\n\n.. include:: ../../../CONTRIBUTING.rst" }, { "path": "doc/source/developer_guide/importer_plugins.rst", "content": ".. _importer-plugins:\n\n===========================\nCreate your importer plugin\n===========================\n\nSince version 1.4, pysteps allows the users to add new importers by installing external\npackages, called plugins, without modifying the pysteps installation. These plugins need\nto follow a particular structure to allow pysteps to discover and integrate the new\nimporters to the pysteps interface without any user intervention.\n\n.. contents:: Table of Contents\n :local:\n :depth: 3\n\nHow do the plugins work?\n========================\n\nWhen the plugin is installed, it advertises the new importers to other packages (in our\ncase, pysteps) using the python `entry points specification`_.\nThese new importers are automatically discovered every time that the pysteps library is\nimported. The discovered importers are added as attributes to the io.importers module\nand also registered to the io.get_method interface without any user intervention.\nIn addition, since the installation of the plugins does not modify the actual pysteps\ninstallation (i.e., the pysteps sources), the pysteps library can be updated without\nreinstalling the plugin.\n\n.. _`entry points specification`: https://packaging.python.org/specifications/entry-points/\n\n\nCreate your plugin\n==================\n\nThere are two ways of creating a plugin. The first one is building the importers plugin\nfrom scratch. However, this can be a daunting task if you are creating your first plugin.\nTo facilitate the creating of new plugins, we provide a `Cookiecutter`_ template, in a\nseparate project, that creates a template project to be used as a starting point to build\nthe plugin.\n\nThe template for the pysteps plugins is maintained as a separate project at\n`cookiecutter-pysteps-plugin `_.\nFor detailed instruction on how to create a plugin, `check the template's documentation`_.\n\n.. _`check the template's documentation`: https://cookiecutter-pysteps-plugin.readthedocs.io/en/latest\n\n.. _Cookiecutter: https://cookiecutter.readthedocs.io\n" }, { "path": "doc/source/developer_guide/pypi.rst", "content": ".. _pypi_relase:\n\n=============================\nPackaging the pysteps project\n=============================\n\nThe `Python Package Index `_ (PyPI) is a software\nrepository for the Python programming language. PyPI helps you find and\ninstall software developed and shared by the Python community.\n\nThe following guide to package pysteps was adapted from the\n`PyPI `_\nofficial documentation.\n\nGenerating the source distribution\n==================================\n\nThe first step is to generate a `source distribution\n(sdist) `_\nfor the pysteps library. These are archives that are uploaded to the\n`Package Index `_ and can be installed by pip.\n\nTo create the sdist package we need the **setuptools** package\ninstalled.\n\nThen, from the root folder of the pysteps source run::\n\n python setup.py sdist\n\nOnce this command is completed, it should generate a tar.gz (source\narchive) file the **dist** directory::\n\n dist/\n pysteps-a.b.c.tar.gz\n\nwhere a.b.c denote the version number.\n\nUploading the source distribution to the archive\n================================================\n\nThe last step is to upload your package to the `Python Package\nIndex `_.\n\n**Important**\n\nBefore we actually upload the distribution to the Python Index, we will\ntest it in `Test PyPI `_. Test PyPI is a\nseparate instance of the package index that allows us to try the\ndistribution without affecting the real index (PyPi). Because TestPyPI\nhas a separate database from the actual PyPI, you’ll need a separate\nuser account for specifically for TestPyPI. You can register your\naccount in https://test.pypi.org/account/register/.\n\nOnce you are registered, you can use\n`twine `_\nto upload the distribution packages. Alternatively, the package can be\nuploaded manually from the **Test PyPI** page.\n\nIf Twine is not installed, you can install it by running\n``pip install twine`` or ``conda install twine``.\n\n\nTest PyPI\n^^^^^^^^^\n\nTo upload the recently created source distribution\n(**dist/pysteps-a.b.c.tar.gz**) under the **dist** directory run::\n\n twine upload --repository-url https://test.pypi.org/legacy/ dist/pysteps-a.b.c.tar.gz\n\nwhere a.b.c denote the version number.\n\nYou will be prompted for the username and password you registered with\nTest PyPI. After the command completes, you should see output similar to\nthis::\n\n Uploading distributions to https://test.pypi.org/legacy/\n Enter your username: [your username]\n Enter your password:\n Uploading pysteps-a.b.c.tar.gz\n 100%|█████████████████████| 4.25k/4.25k [00:01<00:00, 3.05kB/s]\n\nOnce uploaded your package should be viewable on TestPyPI, for example,\nhttps://test.pypi.org/project/pysteps\n\nTest the uploaded package\n-------------------------\n\nBefore uploading the package to the official `Python Package\nIndex `_, test that the package can be installed\nusing pip.\n\nAutomatic test\n^^^^^^^^^^^^^^\n\nThe simplest way to hat the package can be installed using pip is using tox\nand the tox-conda plugin (conda needed).\nTo install these packages activate your conda development environment and run::\n\n conda install -c conda-forge tox tox-conda\n\nThen, to test the installation in a minimal and an environment with all the\ndependencies (full env), run::\n\n tox -r -e pypi_test # Test the installation in a minimal env\n tox -r -e pypi_test_full # Test the installation in an full env\n\n\nManual test\n^^^^^^^^^^^\n\nTo manually test the installation on new environment,\ncreate a copy of the basic development environment using the\n`environment_dev.yml `_\nfile in the root folder of the pysteps project::\n\n conda env create -f environment_dev.yml -n pysteps_test\n\nThen we activate the environment::\n\n source activate pysteps_test\n\nor::\n\n conda activate pysteps_test\n\nIf the environment pysteps_test was already created, remove any version of\npysteps already installed::\n\n pip uninstall pysteps\n\nNow, install the pysteps package from test.pypi.org. \nSince not all the dependecies are available in the Test PyPI repository, we need to add the official repo as an extra index to pip. By doing so, pip will look first in the Test PyPI index and then in the official PyPI::\n\n pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url=https://pypi.org/simple/ pysteps\n\nTo test that the installation was successful, from a folder different\nthan the pysteps source, run::\n\n pytest --pyargs pysteps\n\n\nIf any test didn't pass, check the sources or consider creating a new release\nfixing those bugs.\n\n\nUpload package to PyPi\n----------------------\n\nOnce the\n`sdist `_\npackage was tested, we can safely upload it to the Official PyPi\nrepository with::\n\n twine upload dist/pysteps-a.b.c.tar.gz\n\nNow, **pysteps** can be installed by simply running::\n\n pip install pysteps\n\nAs an extra sanity measure, it is recommended to test the pysteps package\ninstalled from the Official PyPi repository\n(instead of the test PyPi).\n\nAutomatic test\n^^^^^^^^^^^^^^\n\nSimilarly to the `Test the uploaded package`_ section, to test the\ninstallation from PyPI in a clean environment, run::\n\n tox -r -e pypi\n\nManual test\n^^^^^^^^^^^\n\nFollow test instructions in `Test PyPI`_ section.\n\n" }, { "path": "doc/source/developer_guide/test_pysteps.rst", "content": ".. _testing_pysteps:\n\n===============\nTesting pysteps\n===============\n\nThe pysteps distribution includes a small test suite for some of the\nmodules. To run the tests the `pytest `__\npackage is needed. To install it, in a terminal run::\n\n pip install pytest\n\n\nAutomatic testing\n=================\n\nThe simplest way to run the pysteps' test suite is using tox and the tox-conda\nplugin (conda needed).\nTo install these packages activate your conda development environment and run::\n\n conda install -c conda-forge tox tox-conda\n\nThen, to run the tests, from the repo's root run::\n\n tox # Run pytests\n tox -e install # Test package installation\n tox -e black # Test for black formatting warnings\n\n\nManual testing\n==============\n\n\nExample data\n------------\n\nThe build-in tests require the pysteps example data installed.\nSee the installation instructions in the :ref:`example_data` section.\n\nTest an installed package\n-------------------------\n\nAfter the package is installed, you can launch the test suite from any\ndirectory by running::\n\n pytest --pyargs pysteps\n\nTest from sources\n-----------------\n\nBefore testing the package directly from the sources, we need to build\nthe extensions in-place. To do that, from the root pysteps folder run::\n\n python setup.py build_ext -i\n\nNow, the package sources can be tested in-place using the **pytest**\ncommand on the root of the pysteps source directory. E.g.::\n\n pytest -v --tb=line\n\n" }, { "path": "doc/source/developer_guide/update_conda_forge.rst", "content": ".. _update_conda_feedstock:\n\n==========================================\nUpdating the conda-forge pysteps-feedstock\n==========================================\n\n\n.. _pysteps-feedstock: https://github.com/conda-forge/pysteps-feedstock\n.. _`conda-forge/pysteps-feedstock`: https://github.com/conda-forge/pysteps-feedstock\n\nHere we will describe the steps to update the pysteps conda-forge feedstock.\nThis tutorial is intended for the core developers listed as maintainers of the\nconda recipe in the `conda-forge/pysteps-feedstock`_.\n\nExamples for needing to update the pysteps-feedstock are:\n\n* New release\n* Fix errors pysteps package errors\n\n**The following tutorial was adapted from the official conda-forge.org documentation, released\nunder CC4.0 license**\n\nWhat is a “conda-forge”\n=======================\n\nConda-forge is a community effort that provides conda packages for a wide range of software.\nThe conda team from Anaconda packages a multitude of packages and provides them to all users\nfree of charge in their default channel.\n\n**conda-forge** is a community-led conda channel of installable packages that allows users to share software\nthat is not included in the official Anaconda repository. The main advantages of **conda-forge** are:\n\n- all packages are shared in a single channel named conda-forge\n- care is taken that all packages are up-to-date\n- common standards ensure that all packages have compatible versions\n- by default, packages are built for macOS, linux amd64 and windows amd64\n\nIn order to provide high-quality builds, the process has been automated into the conda-forge GitHub organization.\nThe conda-forge organization contains one repository for each of the installable packages.\nSuch a repository is known as a **feedstock**.\n\nThe actual pysteps feedstock is https://github.com/conda-forge/pysteps-feedstock\n\nA feedstock is made up of a conda recipe (the instructions on what and how to build the package) and the\nnecessary configurations for automatic building using freely available continuous integration services.\n\nSee the official `conda-forge documentation `_ for more details.\n\n\nMaintain pysteps conda-forge package\n====================================\n\nPysteps core developers that are maintainers of the pysteps feedstock.\n\nAll pysteps developers listed as maintainers of the pysteps feedstock are given push access to the feedstock repository.\nThis means that a maintainer can create branches in the main repository.\n\nEvery time that a new commit is pushed/merged in the feedstock repository, conda-forge runs Continuous Integration (CI)\nsystem that run quality checks, builds the pysteps recipe on Windows, OSX, and Linux, and publish the built recipes in\nthe conda-forge channel.\n\nImportant\n---------\n\nFor updates, using a branch in the main repo and a subsequent Pull Request (PR) to the master branch is discouraged because:\n- CI is run on both the branch and on the Pull Request (if any) associated with that branch. This wastes CI resources.\n- Branches are automatically published by the CI system. This mean that a for every push, the packages will be published\nbefore the PR is actually merged.\n\nFor these reasons, to update the feedstock, the maintainers need to fork the feedstock, create a new branch in that\nfork, push to that branch in the fork, and then open a PR to the conda-forge repo.\n\n\nWorkflow for updating a pysteps-feedstock\n-----------------------------------------\n\n\nThe mandatory steps to update the pysteps-feedstock_ are:\n\n1. Forking the pysteps-feedstock_.\n\n * Clone the forked repository in your computer::\n\n git clone https://github.com//pysteps-feedstock\n\n#. Syncing your fork with the pysteps feedstock. This step is only needed if your local repository is not up to date\n the pysteps-feedstock_. If you just cloned the forked pysteps-feedstock_, you can ignore this step.\n\n * Make sure you are on the master branch::\n\n git checkout master\n\n * Register conda-forge’s feedstock with::\n\n git remote add upstream https://github.com/conda-forge/pysteps-feedstock\n\n * Fetch the latest updates with git fetch upstream::\n\n git fetch upstream\n\n * Pull in the latest changes into your master branch::\n\n git rebase upstream/master\n\n#. Create a new branch::\n\n git checkout -b \n\n#. Update the recipe and push changes in this new branch\n\n * See next section \"Updating recipes\" for more details\n * Push changes::\n\n git commit -m \n\n#. Pushing your changes to GitHub::\n\n git push origin \n\n#. Propose a Pull Request\n\n * Create a pull request via the web interface\n\n\nUpdating pysteps recipe\n=======================\n\nThe pysteps-feedstock_ should be updated when:\n\n* We release a new pysteps version\n* Need to fix errors in the pysteps package\n\nNew release\n-----------\n\nWhen a new pysteps version is released, before update the pysteps feedstock, the new version needs to be uploaded\nto the Python Package Index (PyPI) (see :ref:`pypi_relase` for more details).\nThis step is needed because the conda recipe uses the PyPI to build the pysteps conda package.\n\nOnce the new version is available in the PyPI, the conda recipe in pysteps-feedstock/recipe/meta.yaml\nneeds to be updated by:\n\n1. Updating version and hash\n\n#. Checking the dependencies\n\n#. Bumping the build number\n\n - When the package version changes, reset the build number back to 0.\n - The build number is increased when the source code for the package has\n not changed but you need to make a new build.\n - In case that the recipe must be updated, increase by 1 the\n **build_number** in the conda recipe in\n `pysteps-feedstock/recipe/meta.yaml `_.\n\n Some examples for needing to increase the build number are:\n\n - updating the pinned dependencies\n - Fixing wrong dependencies\n\n#. Rerendering feedstocks\n\n - Rerendering is conda-forge’s way to update the files common to\n all feedstocks (e.g. README, CI configuration, pinned dependencies).\n\n - When to rerender:\n\n We need to re-render when there are changes the following parts of the\n feedstock:\n\n - the platform configuration (skip sections)\n - the yum_requirements.txt\n - updates in the build matrix due to new versions of Python, NumPy,\n PERL, R, etc.\n - updates in conda-forge pinning that affect the feedstock\n - build issues that a feedstock configuration update will fix\n\n - To rerender the feedstock, the first step is to install **conda-smithy**\n in your root environment::\n\n conda install -c conda-forge conda-smithy\n\n - Commit all changes and from the root directory of the feedstock, type::\n\n conda smithy rerender -c auto\n\n Optionally one can commit the changes manually.\n To do this drop *-c auto* from the command.\n\nMore information on https://conda-forge.org/docs/maintainer/updating_pkgs.html#dev-rerender-local\n\n\nconda-forge autotick bot\n------------------------\n\nThe conda-forge autotick bot is now a central part of the conda-forge\necosystem.\nThe conda-forge autotick bot was created to track out-of-date feedstocks and\nissue pull requests with updated recipes.\nThe bot tracks and updates out-of-date feedstocks in four steps:\n\n- Find the names of all feedstocks on conda-forge.\n- Compute the dependency graph of packages on conda-forge found in step 1.\n- Find the most recent version of each feedstock’s source code.\n- Open a PR into each out-of-date feedstock updating the meta.yaml for the most recent upstream release.\n\nThese steps are run automatically every six hours.\n\nHence, when a new pysteps version is upload to PyPI, this bot will\nautomatically update the recipe and submit a PR.\nIf the tests in the PR pass, then it can be merger into the\nfeedstock's master branch.\n\n" }, { "path": "doc/source/index.rst", "content": "pysteps -- The nowcasting initiative\n====================================\n\nPysteps is a community-driven initiative for developing and maintaining an easy\nto use, modular, free and open source Python framework for short-term ensemble\nprediction systems.\n\nThe focus is on probabilistic nowcasting of radar precipitation fields,\nbut pysteps is designed to allow a wider range of uses.\n\nPysteps is actively developed on GitHub__, while a more thorough description\nof pysteps is available in the pysteps reference publications:\n\n.. note::\n Pulkkinen, S., D. Nerini, A. Perez Hortal, C. Velasco-Forero, U. Germann,\n A. Seed, and L. Foresti, 2019: Pysteps: an open-source Python library for\n probabilistic precipitation nowcasting (v1.0). *Geosci. Model Dev.*, **12 (10)**,\n 4185–4219, doi:`10.5194/gmd-12-4185-2019 `_.\n\n Imhoff, R.O., L. De Cruz, W. Dewettinck, C.C. Brauer, R. Uijlenhoet, K-J. van Heeringen, \n C. Velasco-Forero, D. Nerini, M. Van Ginderachter, and A.H. Weerts, 2023:\n Scale-dependent blending of ensemble rainfall nowcasts and NWP in the open-source\n pysteps library. *Q J R Meteorol Soc.*, 1-30,\n doi: `doi:10.1002/qj.4461 `_.\n\n__ https://github.com/pySTEPS/pysteps\n\n.. toctree::\n :maxdepth: 1\n :hidden:\n :caption: For users\n\n Installation \n Gallery <../auto_examples/index>\n My first nowcast (Colab Notebook) \n API Reference \n Example data \n Configuration file (pystepsrc) \n Machine learning applications \n Bibliography \n\n.. toctree::\n :maxdepth: 1\n :hidden:\n :caption: For developers\n\n Contributing Guide \n Importer plugins \n Testing \n Building the docs \n Packaging \n Publishing to conda-forge \n GitHub repository \n" }, { "path": "doc/source/pysteps_reference/blending.rst", "content": "================\npysteps.blending\n================\n\nImplementation of blending methods for blending (ensemble) nowcasts with Numerical Weather Prediction (NWP) models.\n\n.. automodule:: pysteps.blending.interface\n.. automodule:: pysteps.blending.clim\n.. automodule:: pysteps.blending.ens_kalman_filter_methods\n.. automodule:: pysteps.blending.linear_blending\n.. automodule:: pysteps.blending.pca_ens_kalman_filter\n.. automodule:: pysteps.blending.skill_scores\n.. automodule:: pysteps.blending.steps\n.. automodule:: pysteps.blending.utils\n" }, { "path": "doc/source/pysteps_reference/cascade.rst", "content": "===============\npysteps.cascade\n===============\n\nMethods for constructing bandpass filters and decomposing 2d precipitation\nfields into different spatial scales.\n\n.. automodule:: pysteps.cascade.interface\n.. automodule:: pysteps.cascade.bandpass_filters\n.. automodule:: pysteps.cascade.decomposition\n\n\n" }, { "path": "doc/source/pysteps_reference/datasets.rst", "content": ".. automodule:: pysteps.datasets" }, { "path": "doc/source/pysteps_reference/decorators.rst", "content": ".. automodule:: pysteps.decorators\n" }, { "path": "doc/source/pysteps_reference/downscaling.rst", "content": "===================\npysteps.downscaling\n===================\n\nImplementation of deterministic and ensemble downscaling methods.\n\n\n.. automodule:: pysteps.downscaling.interface\n.. automodule:: pysteps.downscaling.rainfarm\n\n" }, { "path": "doc/source/pysteps_reference/extrapolation.rst", "content": "=====================\npysteps.extrapolation\n=====================\n\nExtrapolation module functions and interfaces.\n\n.. automodule:: pysteps.extrapolation.interface\n.. automodule:: pysteps.extrapolation.semilagrangian\n\n" }, { "path": "doc/source/pysteps_reference/feature.rst", "content": "===============\npysteps.feature\n===============\n\nImplementations of feature detection methods.\n\n\n.. automodule:: pysteps.feature.interface\n.. automodule:: pysteps.feature.blob\n.. automodule:: pysteps.feature.tstorm\n.. automodule:: pysteps.feature.shitomasi\n\n" }, { "path": "doc/source/pysteps_reference/index.rst", "content": ".. _pysteps-reference:\n\nAPI Reference\n=============\n\n:Release: |version|\n:Date: |today|\n\nThis page gives an comprehensive description of all the modules and functions\navailable in pysteps.\n\n.. toctree::\n :maxdepth: 2\n :caption: API Reference\n\n pysteps\n blending\n cascade\n decorators\n extrapolation\n datasets\n downscaling\n feature\n io\n motion\n noise\n nowcasts\n postprocessing\n timeseries\n tracking\n utils\n verification\n visualization\n\n.. only:: html\n\n Indices and tables\n ==================\n\n * :ref:`genindex`\n * :ref:`modindex`\n * :ref:`search`\n\n.. only:: html\n\n Bibliography\n ------------\n\n * :ref:`bibliography`\n" }, { "path": "doc/source/pysteps_reference/io.rst", "content": "==========\npysteps.io\n==========\n\nMethods for browsing data archives, reading 2d precipitation fields and writing \nforecasts into files.\n\n.. automodule:: pysteps.io.interface\n.. automodule:: pysteps.io.archive\n.. automodule:: pysteps.io.importers\n.. automodule:: pysteps.io.nowcast_importers\n.. automodule:: pysteps.io.exporters\n.. automodule:: pysteps.io.readers\n" }, { "path": "doc/source/pysteps_reference/motion.rst", "content": "==============\npysteps.motion\n==============\n\nImplementations of optical flow methods.\n\n\n.. automodule:: pysteps.motion.interface\n.. automodule:: pysteps.motion.constant\n.. automodule:: pysteps.motion.darts\n.. automodule:: pysteps.motion.lucaskanade\n.. automodule:: pysteps.motion.proesmans\n.. automodule:: pysteps.motion.vet\n" }, { "path": "doc/source/pysteps_reference/noise.rst", "content": "=============\npysteps.noise\n=============\n\nImplementation of deterministic and ensemble nowcasting methods.\n\n\n.. automodule:: pysteps.noise.interface\n.. automodule:: pysteps.noise.fftgenerators\n.. automodule:: pysteps.noise.motion\n.. automodule:: pysteps.noise.utils\n" }, { "path": "doc/source/pysteps_reference/nowcasts.rst", "content": "================\npysteps.nowcasts\n================\n\nImplementation of deterministic and ensemble nowcasting methods.\n\n\n.. automodule:: pysteps.nowcasts.interface\n.. automodule:: pysteps.nowcasts.anvil\n.. automodule:: pysteps.nowcasts.extrapolation\n.. automodule:: pysteps.nowcasts.linda\n.. automodule:: pysteps.nowcasts.lagrangian_probability\n.. automodule:: pysteps.nowcasts.sprog\n.. automodule:: pysteps.nowcasts.sseps\n.. automodule:: pysteps.nowcasts.steps\n.. automodule:: pysteps.nowcasts.utils\n" }, { "path": "doc/source/pysteps_reference/postprocessing.rst", "content": "======================\npysteps.postprocessing\n======================\n\nMethods for post-processing of forecasts.\n\n\n.. automodule:: pysteps.postprocessing.ensemblestats\n.. automodule:: pysteps.postprocessing.probmatching\n\n" }, { "path": "doc/source/pysteps_reference/pysteps.rst", "content": "=======\npysteps\n=======\n\nPystep top module utils\n\n.. autosummary::\n :toctree: ../generated/\n\n pysteps.load_config_file\n" }, { "path": "doc/source/pysteps_reference/timeseries.rst", "content": "==================\npysteps.timeseries\n==================\n\nMethods and models for time series analysis.\n\n\n.. automodule:: pysteps.timeseries.autoregression\n.. automodule:: pysteps.timeseries.correlation\n\n" }, { "path": "doc/source/pysteps_reference/tracking.rst", "content": "================\npysteps.tracking\n================\n\nImplementations of feature tracking methods.\n\n\n.. automodule:: pysteps.tracking.interface\n.. automodule:: pysteps.tracking.lucaskanade\n.. automodule:: pysteps.tracking.tdating\n" }, { "path": "doc/source/pysteps_reference/utils.rst", "content": "=============\npysteps.utils\n=============\n\nImplementation of miscellaneous utility functions.\n\n\n.. automodule:: pysteps.utils.interface\n.. automodule:: pysteps.utils.arrays\n.. automodule:: pysteps.utils.cleansing\n.. automodule:: pysteps.utils.conversion\n.. automodule:: pysteps.utils.dimension\n.. automodule:: pysteps.utils.fft\n.. automodule:: pysteps.utils.images\n.. automodule:: pysteps.utils.interpolate\n.. automodule:: pysteps.utils.pca\n.. automodule:: pysteps.utils.reprojection\n.. automodule:: pysteps.utils.spectral\n.. automodule:: pysteps.utils.tapering\n.. automodule:: pysteps.utils.transformation\n" }, { "path": "doc/source/pysteps_reference/verification.rst", "content": "====================\npysteps.verification\n====================\n\nMethods for verification of deterministic, probabilistic and ensemble forecasts.\n\n.. automodule:: pysteps.verification.interface\n.. automodule:: pysteps.verification.detcatscores\n.. automodule:: pysteps.verification.detcontscores\n.. automodule:: pysteps.verification.ensscores\n.. automodule:: pysteps.verification.lifetime\n.. automodule:: pysteps.verification.plots\n.. automodule:: pysteps.verification.probscores\n.. automodule:: pysteps.verification.salscores\n.. automodule:: pysteps.verification.spatialscores\n" }, { "path": "doc/source/pysteps_reference/visualization.rst", "content": "=====================\npysteps.visualization\n=====================\n\nMethods for plotting precipitation and motion fields.\n\n.. automodule:: pysteps.visualization.animations\n.. automodule:: pysteps.visualization.basemaps\n.. automodule:: pysteps.visualization.motionfields\n.. automodule:: pysteps.visualization.precipfields\n.. automodule:: pysteps.visualization.spectral\n.. automodule:: pysteps.visualization.thunderstorms\n.. automodule:: pysteps.visualization.utils\n" }, { "path": "doc/source/references.bib", "content": "\n@TECHREPORT{BPS2004,\n AUTHOR = \"N. E. Bowler and C. E. Pierce and A. W. Seed\",\n TITLE = \"{STEPS}: A probabilistic precipitation forecasting scheme which merges an extrapolation nowcast with downscaled {NWP}\",\n INSTITUTION = \"UK Met Office\",\n TYPE = \"Forecasting Research Technical Report\",\n NUMBER = 433,\n ADDRESS = \"Wallingford, United Kingdom\",\n YEAR = 2004,\n}\n\n@ARTICLE{BPS2006,\n AUTHOR = \"N. E. Bowler and C. E. Pierce and A. W. Seed\",\n TITLE = \"{STEPS}: A probabilistic precipitation forecasting scheme which merges an extrapolation nowcast with downscaled {NWP}\",\n JOURNAL = \"Quarterly Journal of the Royal Meteorological Society\",\n VOLUME = 132,\n NUMBER = 620,\n PAGES = \"2127--2155\",\n YEAR = 2006,\n DOI = \"10.1256/qj.04.100\"\n}\n\n@ARTICLE{BS2007,\n AUTHOR = \"J. Br{\\\"o}cker and L. A. Smith\",\n TITLE = \"Increasing the Reliability of Reliability Diagrams\",\n JOURNAL = \"Weather and Forecasting\",\n VOLUME = 22,\n NUMBER = 3,\n PAGES = \"651--661\",\n YEAR = 2007,\n DOI = \"10.1175/WAF993.1\"\n}\n\n@BOOK{CP2002,\n AUTHOR = \"A. Clothier and G. Pegram\",\n TITLE = \"Space-time modelling of rainfall using the string of beads model: integration of radar and raingauge data\",\n SERIES = \"WRC Report No. 1010/1/02\",\n PUBLISHER = \"Water Research Commission\",\n ADDRESS = \"Durban, South Africa\",\n YEAR = 2002\n}\n\n@ARTICLE{CRS2004,\n AUTHOR = \"B. Casati and G. Ross and D. B. Stephenson\",\n TITLE = \"A New Intensity-Scale Approach for the Verification of Spatial Precipitation Forecasts\",\n VOLUME = 11,\n NUMBER = 2,\n JOURNAL = \"Meteorological Applications\",\n PAGES = \"141-–154\",\n YEAR = 2004,\n DOI = \"10.1017/S1350482704001239\"\n}\n\n@ARTICLE{DOnofrio2014,\n TITLE = \"Stochastic rainfall downscaling of climate models\",\n AUTHOR = \"D'Onofrio, D and Palazzi, E and von Hardenberg, J and Provenzale, A and Calmanti, S\",\n JOURNAL = \"J. Hydrometeorol.\",\n PUVLISHER = \"American Meteorological Society\",\n VOLUME = 15,\n NUMBER = 2,\n PAGES = \"830--843\",\n YEAR = 2014,\n}\n\n@ARTICLE{EWWM2013,\n AUTHOR = \"E. Ebert and L. Wilson and A. Weigel and M. Mittermaier and P. Nurmi and P. Gill and M. Göber and S. Joslyn and B. Brown and T. Fowler and A. Watkins\",\n TITLE = \"Progress and challenges in forecast verification\",\n JOURNAL = \"Meteorological Applications\",\n VOLUME = 20,\n NUMBER = 2,\n PAGES = \"130--139\",\n YEAR = 2013,\n DOI = \"10.1002/met.1392\"\n}\n\n@ARTICLE{Feldmann2021,\n AUTHOR = \"M. Feldmann and U. Germann and M. Gabella and A. Berne\",\n TITLE = \"A Characterisation of Alpine Mesocyclone Occurrence\",\n JOURNAL = \"Weather and Climate Dynamics Discussions\",\n PAGES = \"1--26\",\n URL = \"https://wcd.copernicus.org/preprints/wcd-2021-53/\",\n DOI = \"10.5194/wcd-2021-53\",\n YEAR = 2021\n}\n\n@ARTICLE{FSNBG2019,\n AUTHOR = \"Foresti, L. and Sideris, I.V. and Nerini, D. and Beusch, L. and Germann, U.\",\n TITLE = \"Using a 10-Year Radar Archive for Nowcasting Precipitation Growth and Decay: A Probabilistic Machine Learning Approach\",\n JOURNAL = \"Weather and Forecasting\",\n VOLUME = 34,\n PAGES = \"1547--1569\",\n YEAR = 2019,\n DOI = \"10.1175/WAF-D-18-0206.1\"\n}\n\n@ARTICLE{FNPC2020,\n AUTHOR = \"Franch, G. and Nerini, D. and Pendesini, M. and Coviello, L. and Jurman, G. and Furlanello, C.\",\n TITLE = \"Precipitation Nowcasting with Orographic Enhanced Stacked Generalization: Improving Deep Learning Predictions on Extreme Events\",\n JOURNAL = \"Atmosphere\",\n VOLUME = 11,\n NUMBER = 3,\n PAGES = \"267\",\n YEAR = 2020,\n DOI = \"10.3390/atmos11030267\"\n}\n\n@ARTICLE{FW2005,\n AUTHOR = \"N. I. Fox and C. K. Wikle\",\n TITLE = \"A Bayesian Quantitative Precipitation Nowcast Scheme\",\n JOURNAL = \"Weather and Forecasting\",\n VOLUME = 20,\n NUMBER = 3,\n PAGES = \"264--275\",\n YEAR = 2005\n}\n\n@ARTICLE{GZ2002,\n AUTHOR = \"U. Germann and I. Zawadzki\",\n TITLE = \"Scale-Dependence of the Predictability of Precipitation from Continental Radar Images. {P}art {I}: Description of the Methodology\",\n JOURNAL = \"Monthly Weather Review\",\n VOLUME = 130,\n NUMBER = 12,\n PAGES = \"2859--2873\",\n YEAR = 2002,\n DOI = \"10.1175/1520-0493(2002)130<2859:SDOTPO>2.0.CO;2\"\n}\n\n@ARTICLE{GZ2004,\n AUTHOR = \"U. Germann and I. Zawadzki\",\n TITLE = \"Scale-Dependence of the Predictability of Precipitation from Continental Radar Images. {P}art {II}: Probability Forecasts\",\n JOURNAL = \"Journal of Applied Meteorology\",\n VOLUME = 43,\n NUMBER = 1,\n PAGES = \"74--89\",\n YEAR = 2004,\n DOI = \"10.1175/1520-0450(2004)043<0074:SDOTPO>2.0.CO;2\"\n}\n\n@ARTICLE{Her2000,\n AUTHOR = \"H. Hersbach\",\n TITLE = \"Decomposition of the Continuous Ranked Probability Score for Ensemble Prediction Systems\",\n JOURNAL = \"Weather and Forecasting\",\n VOLUME = 15,\n NUMBER = 5,\n PAGES = \"559--570\",\n YEAR = 2000,\n DOI = \"10.1175/1520-0434(2000)015<0559:DOTCRP>2.0.CO;2\"\n}\n\n@article{Hwang2015,\n AUTHOR = \"Hwang, Yunsung and Clark, Adam J and Lakshmanan, Valliappa and Koch, Steven E\",\n TITLE = \"Improved nowcasts by blending extrapolation and model forecasts\",\n JOURNAL = \"Weather and Forecasting\",\n VOLUME = 30,\n NUMBER = 5,\n PAGES = \"1201--1217\",\n YEAR = 2015,\n DOI = \"10.1175/WAF-D-15-0057.1\"\n}\n\n@ARTICLE{LZ1995,\n AUTHOR = \"S. Laroche and I. Zawadzki\",\n TITLE = \"Retrievals of Horizontal Winds from Single-Doppler Clear-Air Data by Methods of Cross Correlation and Variational Analysis\",\n JOURNAL = \"Journal of Atmospheric and Oceanic Technology\",\n VOLUME = 12,\n NUMBER = 4,\n PAGES = \"721--738\",\n YEAR = 1995,\n DOI = \"10.1175/1520-0426(1995)012<0721:ROHWFS>2.0.CO;2\",\n}\n\n@ARTICLE{NBSG2017,\n AUTHOR = \"D. Nerini and N. Besic and I. Sideris and U. Germann and L. Foresti\",\n TITLE = \"A non-stationary stochastic ensemble generator for radar rainfall fields based on the short-space {F}ourier transform\",\n JOURNAL = \"Hydrology and Earth System Sciences\",\n VOLUME = 21,\n NUMBER = 6,\n YEAR = 2017,\n PAGES = \"2777--2797\",\n DOI = \"10.5194/hess-21-2777-2017\"\n}\n\n@ARTICLE{PCH2018,\n AUTHOR = \"S. Pulkkinen and V. Chandrasekar and A.-M. Harri\",\n TITLE = \"Nowcasting of Precipitation in the High-Resolution {D}allas-{F}ort {W}orth ({DFW}) Urban Radar Remote Sensing Network\",\n JOURNAL = \"IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing\",\n VOLUME = 11,\n NUMBER = 8,\n PAGES = \"2773--2787\",\n YEAR = 2018,\n DOI = \"10.1109/JSTARS.2018.2840491\"\n}\n\n@ARTICLE{PCH2019a,\n AUTHOR = \"S. Pulkkinen and V. Chandrasekar and A.-M. Harri\",\n TITLE = \"Fully Spectral Method for Radar-Based Precipitation Nowcasting\",\n JOURNAL = \"IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing\",\n VOLUME = 12,\n NUMBER = 5,\n PAGES = \"1369-1382\",\n YEAR = 2018\n}\n\n@ARTICLE{PCH2019b,\n AUTHOR = \"S. Pulkkinen and V. Chandrasekar and A.-M. Harri\",\n TITLE = \"Stochastic Spectral Method for Radar-Based Probabilistic Precipitation Nowcasting\",\n JOURNAL = \"Journal of Atmospheric and Oceanic Technology\",\n VOLUME = 36,\n NUMBER = 6,\n PAGES = \"971--985\",\n YEAR = 2019\n}\n\n@ARTICLE{PCLH2020,\n AUTHOR = \"S. Pulkkinen and V. Chandrasekar and A. von Lerber and A.-M. Harri\",\n TITLE = \"Nowcasting of Convective Rainfall Using Volumetric Radar Observations\",\n JOURNAL = \"IEEE Transactions on Geoscience and Remote Sensing\",\n DOI = \"10.1109/TGRS.2020.2984594\",\n PAGES = \"1--15\",\n YEAR = 2020\n}\n\n@ARTICLE{PCN2021,\n AUTHOR = \"S. Pulkkinen and V. Chandrasekar and T. Niemi\",\n TITLE = \"Lagrangian Integro-Difference Equation Model for Precipitation Nowcasting\",\n JOURNAL = \"Journal of Atmospheric and Oceanic Technology\",\n NOTE = \"submitted\",\n YEAR = 2021\n}\n\n@INCOLLECTION{PGPO1994,\n AUTHOR = \"M. Proesmans and L. van Gool and E. Pauwels and A. Oosterlinck\",\n TITLE = \"Determination of optical flow and its discontinuities using non-linear diffusion\",\n BOOKTITLE = \"Computer Vision — ECCV '94\",\n VOLUME = 801,\n SERIES = \"Lecture Notes in Computer Science\",\n EDITOR = \"J.-O. Eklundh\",\n PUBLISHER = \"Springer Berlin Heidelberg\",\n PAGES = \"294--304\",\n YEAR = 1994\n}\n\n@ARTICLE{RC2011,\n AUTHOR = \"E. Ruzanski and V. Chandrasekar\",\n JOURNAL = \"IEEE Transactions on Geoscience and Remote Sensing\",\n TITLE = \"Scale Filtering for Improved Nowcasting Performance in a High-Resolution {X}-Band Radar Network\",\n VOLUME = 49,\n NUMBER = 6,\n PAGES=\"2296--2307\",\n MONTH = \"June\",\n YEAR=2011\n}\n\n@ARTICLE{Ravuri2021,\n AUTHOR = \"Ravuri, Suman and Lenc, Karel and Willson, Matthew and Kangin, Dmitry and Lam, Remi and Mirowski, Piotr and Fitzsimons, Megan and Athanassiadou, Maria and Kashem, Sheleem and Madge, Sam and Prudden, Rachel and Mandhane, Amol and Clark, Aidan and Brock, Andrew and Simonyan, Karen and Hadsell, Raia and Robinson, Niall and Clancy, Ellen and Arribas, Alberto and Mohamed, Shakir\",\n JOURNAL = \"Nature\",\n TITLE = \"Skilful precipitation nowcasting using deep generative models of radar\",\n VOLUME = 597,\n NUMBER = 7878,\n PAGES = \"672--677\",\n YEAR = 2011,\n DOI = \"10.1038/s41586-021-03854-z\",\n}\n\n@ARTICLE{RCW2011,\n AUTHOR = \"E. Ruzanski and V. Chandrasekar and Y. Wang\",\n TITLE = \"The {CASA} Nowcasting System\",\n JOURNAL = \"Journal of Atmospheric and Oceanic Technology\",\n VOLUME = 28,\n NUMBER = 5,\n PAGES = \"640--655\",\n YEAR = 2011,\n DOI = \"10.1175/2011JTECHA1496.1\"\n}\n\n@ARTICLE{RL2008,\n AUTHOR = \"N. M. Roberts and H. W. Lean\",\n TITLE = \"Scale-Selective Verification of Rainfall Accumulations from High-Resolution Forecasts of Convective Events\",\n JOURNAL = \"Monthly Weather Review\",\n VOLUME = 136,\n NUMBER = 1,\n PAGES = \"78--97\",\n YEAR = 2008,\n DOI = \"10.1175/2007MWR2123.1\"\n}\n\n@ARTICLE{Rebora2006,\n AUTHOR = \"N. Rebora and L. Ferraris and J. von Hardenberg and A. Provenzale\",\n TITLE = \"RainFARM: Rainfall Downscaling by a Filtered Autoregressive Model\",\n JOURNAL = \"Journal of Hydrometeorology\",\n VOLUME = 7,\n NUMBER = 4,\n PAGES = \"724-738\",\n YEAR = 2006,\n DOI = \"10.1175/JHM517.1\"\n}\n\n@ARTICLE{Seed2003,\n AUTHOR = \"A. W. Seed\",\n TITLE = \"A Dynamic and Spatial Scaling Approach to Advection Forecasting\",\n JOURNAL = \"Journal of Applied Meteorology\",\n VOLUME = 42,\n NUMBER = 3,\n PAGES = \"381-388\",\n YEAR = 2003,\n DOI = \"10.1175/1520-0450(2003)042<0381:ADASSA>2.0.CO;2\"\n}\n\n@ARTICLE{SPN2013,\n AUTHOR = \"A. W. Seed and C. E. Pierce and K. Norman\",\n TITLE = \"Formulation and evaluation of a scale decomposition-based stochastic precipitation nowcast scheme\",\n JOURNAL = \"Water Resources Research\",\n VOLUME = 49,\n NUMBER = 10,\n PAGES = \"6624--6641\",\n YEAR = 2013,\n DOI = \"10.1002/wrcr.20536\"\n}\n\n@Article{Terzago2018,\n AUTHOR = \"Terzago, S. and Palazzi, E. and von Hardenberg, J.\",\n TITLE = \"Stochastic downscaling of precipitation in complex orography: a simple method to reproduce a realistic fine-scale climatology\",\n JOURNAL = \"Natural Hazards and Earth System Sciences\",\n VOLUME = 18,\n YEAR = 2018,\n NUMBER = 11,\n PAGES = \"2825--2840\",\n DOI = \"10.5194/nhess-18-2825-2018\"\n}\n\n@ARTICLE{TRT2004,\n AUTHOR = \"A. M. Hering and C. Morel and G. Galli and P. Ambrosetti and M. Boscacci\",\n TITLE = \"Nowcasting thunderstorms in the Alpine Region using a radar based adaptive thresholding scheme\",\n JOURNAL = \"Proceedings of ERAD Conference 2004\",\n NUMBER = \"January\",\n PAGES = \"206--211\",\n YEAR = 2004\n}\n\n@ARTICLE{WHZ2009,\n AUTHOR = \"Heini Wernli and Christiane Hofmann and Matthias Zimmer\",\n TITLE = \"Spatial Forecast Verification Methods Intercomparison Project: Application of the SAL Technique\",\n JOURNAL = \"Weather and Forecasting\",\n NUMBER = \"6\",\n VOLUME = \"24\",\n PAGES = \"1472 - 14847\",\n YEAR = 2009\n}\n\n@ARTICLE{WPHF2008,\n AUTHOR = \"Heini Wernli and Marcus Paulat and Martin Hagen and Christoph Frei\",\n TITLE = \"SAL—A Novel Quality Measure for the Verification of Quantitative Precipitation Forecasts\",\n JOURNAL = \"Monthly Weather Review\",\n NUMBER = \"11\",\n VOLUME = \"136\",\n PAGES = \"4470 - 4487\",\n YEAR = 2008\n}\n\n@ARTICLE{XWF2005,\n AUTHOR = \"K. Xu and C. K Wikle and N. I. Fox\",\n TITLE = \"A Kernel-Based Spatio-Temporal Dynamical Model for Nowcasting Weather Radar Reflectivities\",\n JOURNAL = \"Journal of the American Statistical Association\",\n VOLUME = 100,\n NUMBER = 472,\n PAGES = \"1133--1144\",\n YEAR = 2005\n}\n\n@ARTICLE{ZR2009,\n AUTHOR = \"P. Zacharov and D. Rezacova\",\n TITLE = \"Using the fractions skill score to assess the relationship between an ensemble {QPF} spread and skill\",\n JOURNAL = \"Atmospheric Research\",\n VOLUME = 94,\n NUMBER = 4,\n PAGES = \"684--693\",\n YEAR = 2009,\n DOI = \"10.1016/j.atmosres.2009.03.004\"\n}\n\n@ARTICLE{Imhoff2023,\n AUTHOR = \"R.O. Imhoff and L. De Cruz and W. Dewettinck and C.C. Brauer and R. Uijlenhoet and K-J. van Heeringen and C. Velasco-Forero and D. Nerini and M. Van Ginderachter and A.H. Weerts\",\n TITLE = \"Scale-dependent blending of ensemble rainfall nowcasts and {NWP} in the open-source pysteps library\",\n JOURNAL = \"Quarterly Journal of the Royal Meteorological Society\",\n VOLUME = 149,\n NUMBER = 753,\n PAGES = \"1--30\",\n YEAR = 2023,\n DOI = \"10.1002/qj.4461\"\n}\n\n@ARTICLE{Nerini2019MWR,\n\ttitle = {A {Reduced}-{Space} {Ensemble} {Kalman} {Filter} {Approach} for {Flow}-{Dependent} {Integration} of {Radar} {Extrapolation} {Nowcasts} and {NWP} {Precipitation} {Ensembles}},\n\tvolume = {147},\n\tdoi = {10.1175/MWR-D-18-0258.1},\n\tnumber = {3},\n\tjournal = {Monthly Weather Review},\n\tauthor = {D. Nerini and L. Foresti and D. Leuenberger and S. Robert and U. Germann},\n\tyear = {2019},\n\tpages = {987--1006},\n}\n" }, { "path": "doc/source/user_guide/example_data.rst", "content": ".. _example_data:\n\nInstalling the example data\n===========================\n\nThe examples scripts in the user guide, as well as the build-in tests,\nuse the example radar data available in a separate repository:\n`pysteps-data `_.\n\nThe easiest way to install the example data is by using the\n:func:`~pysteps.datasets.download_pysteps_data` and\n:func:`~pysteps.datasets.create_default_pystepsrc` functions from\nthe :mod:`pysteps.datasets` module.\n\nInstallation using the datasets module\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nBelow is a snippet code that can be used to install can configure `pystepsrc` file to\npoint to that example data.\n\n\nIn the example below, the example data is placed in the user's home folder under the\n**pysteps_data** directory. It also creates a default configuration file that points to\nthe downloaded data and places it in the $HOME/.pysteps (Unix and Mac OS X) or\n$USERPROFILE/pysteps (Windows). This is one of the default locations where pysteps\nlooks for the configuration file (see :ref:`pysteps_lookup` for\nmore information).\n\n.. code-block:: python\n\n import os\n\n # Import the helper functions\n from pysteps.datasets import download_pysteps_data, create_default_pystepsrc\n\n # In this example we will place it in the user's home folder on the\n # `pysteps_data` folder.\n home_dir = os.path.expanduser(\"~\")\n pysteps_data_dir_path = os.path.join(home_dir, \"pysteps_data\")\n\n # Download the pysteps data.\n download_pysteps_data(pysteps_data_dir_path, force=True)\n\n # Create a default configuration file that points to the downloaded data.\n # By default it will place the configuration file in the\n # $HOME/.pysteps (unix and Mac OS X) or $USERPROFILE/pysteps (windows).\n config_file_path = create_default_pystepsrc(pysteps_data_dir_path)\n\nNote that for these changes to take effect you need to restart the python interpreter or\nuse the :func:`pysteps.load_config_file` function as follows::\n\n # Load the new configuration file and replace the default configuration\n import pysteps\n pysteps.load_config_file(config_file_path, verbose=True)\n\n\nTo customize the default configuration file see the :ref:`pystepsrc` section.\n\n\nManual installation\n~~~~~~~~~~~~~~~~~~~\n\nAnother alternative is to download the data manually into your computer and configure the\n:ref:`pystepsrc ` file to point to that example data.\n\nFirst, download the data from the repository by\n`clicking here `_.\n\nUnzip the data into a folder of your preference. Once the data is unzipped, the\ndirectory structure looks like this::\n\n\n pysteps-data\n |\n ├── radar\n ├── KNMI\n ├── OPERA\n ├── bom\n ├── dwd\n ├── fmi\n ├── mch\n\nThe next step is updating the *pystepsrc* file to point to these directories,\nas described in the :ref:`pystepsrc` section.\n\n\n\n\n" }, { "path": "doc/source/user_guide/install_pysteps.rst", "content": ".. _install_pysteps:\n\nInstalling pysteps\n==================\n\nDependencies\n------------\n\nThe pysteps package needs the following dependencies\n\n* `python >=3.11, <3.14 `_ (lower or higher versions may work but are not tested).\n* `jsonschema `_\n* `matplotlib `_\n* `netCDF4 `_\n* `numpy `_\n* `opencv `_\n* `pillow `_\n* `pyproj `_\n* `scipy `_\n\n\nAdditionally, the following packages can be installed for better computational\nefficiency:\n\n* `dask `_ and\n `toolz `_ (for code parallelization)\n* `pyfftw `_ (for faster FFT computation)\n\n\nOther optional dependencies include:\n\n* `cartopy >=0.18 `_ (for geo-referenced\n visualization)\n* `h5py `_ (for importing HDF5 data)\n* `pygrib `_ (for importing MRMS data)\n* `gdal `_ (for importing GeoTIFF data)\n* `pywavelets `_\n (for intensity-scale verification)\n* `pandas `_ and\n `scikit-image >=0.19 `_ (for advanced feature detection methods)\n* `rasterio `_ (for the reprojection module)\n* `scikit-learn >=1.7 `_ (for PCA-based blending methods)\n\n\n**Important**: If you only want to use pysteps, you can continue reading below.\nBut, if you want to contribute to pysteps or edit the package, you need to install\npysteps in development mode: :ref:`Contributing to pysteps `.\n\nInstall with conda/mamba (recommended)\n--------------------------------------\n\n`Conda `_ is an open-source package management system and environment\nmanagement system that runs on Windows, macOS, and Linux.\n`Mamba `_ is a drop-in replacement for conda offering\nbetter performances and more reliable environment\nsolutions. Mamba quickly installs, runs, and updates packages and their dependencies.\nIt also allows you to easily create, save, load, or switch between different\nenvironments on your local computer.\n\nSince version 1.0, pysteps is available on `conda-forge `_,\na community-driven package repository for conda packages.\n\nTo install pysteps with mamba in a new environment, run in a terminal::\n\n mamba create -n pysteps python=3.11\n mamba activate pysteps\n\nThis will create and activate the new python environment called 'pysteps' using python 3.11.\nThe next step is to add the conda-forge channel where the pysteps package is located::\n\n conda config --env --prepend channels conda-forge\n\nLet's set this channel as the priority one::\n\n conda config --env --set channel_priority strict\n\nThe latter step is not strictly necessary but is recommended since\nthe conda-forge and the default conda channels are not 100% compatible.\n\nFinally, to install pysteps and all its dependencies run::\n\n mamba install pysteps\n\nInstall pysteps on Apple Silicon Macs\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nOn conda-forge, pysteps is currently compiled for Mac computers with Intel processors (osx-64).\nHowever, thanks to `Rosetta 2 `_ it is\npossible to install the same package on a Mac computers with an Apple Silicon processor\n(arm-64).\n\nFirst, make sure that Rosetta 2 is installed::\n\n softwareupdate --install-rosetta\n\nUse mamba to create a new environment called 'pysteps' for intel packages with python 3.11::\n\n CONDA_SUBDIR=osx-64 mamba create -n pysteps python=3.11\n mamba activate pysteps\n\nMake sure that conda/mamba commands in this environment use intel packages::\n\n conda config --env --set subdir osx-64\n\nVerify that the correct platform is being used::\n\n python -c \"import platform;print(platform.machine())\" # Should print \"x86_64\"\n\nFinally, run the same pysteps install instructions as given above::\n\n conda config --env --prepend channels conda-forge\n conda config --env --set channel_priority strict\n mamba install pysteps\n\nWe can now verify that pysteps loads correctly::\n\n python -c \"import pysteps\"\n\nNote that the first time that pysteps is imported will typically take longer, as Rosetta 2\nneeds to translate the binary code for the Apple Silicon processor.\n\nInstall from source\n-------------------\n\nThe recommended way to install pysteps from the source is using ``pip``\nto adhere to the `PEP517 standards `_.\nUsing ``pip`` instead of ``setup.py`` guarantees that all the package dependencies\nare properly handled during the installation process.\n\nOSX users: gcc compiler\n~~~~~~~~~~~~~~~~~~~~~~~\n\npySTEPS uses Cython extensions that need to be compiled with multi-threading\nsupport enabled. The default Apple Clang compiler does not support OpenMP.\nHence, using the default compiler would have disabled multi-threading and may raise\nthe following error during the installation::\n\n clang: error: unsupported option '-fopenmp'\n error: command 'gcc' failed with exit status 1\n\nTo solve this issue, obtain the latest gcc version with\nHomebrew_ that has multi-threading enabled::\n\n brew install gcc@13\n\n.. _Homebrew: https://brew.sh/\n\nTo make sure that the installer uses the homebrew's gcc, export the\nfollowing environmental variables in the terminal\n(supposing that gcc version 13 was installed)::\n\n export CC=gcc-13\n export CXX=g++-13\n\nFirst, check that the homebrew's gcc is detected::\n\n which gcc-13\n\nThis should point to the homebrew's gcc installation.\n\nUnder certain circumstances, Homebrew_ does not add the symbolic links for the\ngcc executables under /usr/local/bin.\nIf that is the case, specify the CC and CCX variables using the full path to\nthe homebrew installation. For example::\n\n export CC=/usr/local/Cellar/gcc/13.2.0/bin/gcc-13\n export CXX=/usr/local/Cellar/gcc/13.2.0/bin/g++-13\n\nThen, you can continue with the normal installation procedure described next.\n\nInstallation using pip\n~~~~~~~~~~~~~~~~~~~~~~\n\nThe latest pysteps version in the repository can be installed using pip by\nsimply running in a terminal::\n\n pip install git+https://github.com/pySTEPS/pysteps\n\nOr, from a local copy of the repo::\n\n git clone https://github.com/pySTEPS/pysteps\n cd pysteps\n pip install .\n\nThe above commands install the latest version of the **master** branch,\nwhich is continuously under development.\n\n.. warning::\n If you are installing pysteps from the sources using pip, the Python interpreter must be launched outside of the pysteps root directory.\n Importing pysteps from a working directory that contains the pysteps source code will raise a ``ModuleNotFoundError``. \n This error is caused by the root pysteps folder being recognized as the pysteps package, also known as \n `the double import trap `_.\n\nSetting up the user-defined configuration file\n----------------------------------------------\n\nThe pysteps package allows the users to customize the default settings\nand configuration.\nThe configuration parameters used by default are loaded from a user-defined\n`JSON `_ file and then stored in the **pysteps.rcparams**, a dictionary-like object\nthat can be accessed as attributes or as items.\n\n.. toctree::\n :maxdepth: 1\n\n Set-up the user-defined configuration file \n Example pystepsrc file \n\n.. _import_pysteps:\n\nFinal test: import pysteps in Python\n------------------------------------\n\nActivate the pysteps environment::\n\n conda activate pysteps\n\nLaunch Python and import pysteps::\n\n python\n >>> import pysteps\n" }, { "path": "doc/source/user_guide/machine_learning_pysteps.rst", "content": ".. _machine_learning_pysteps:\n\nBenchmarking machine learning models with pysteps\n=================================================\nHow to correctly compare the accuracy of machine learning against traditional nowcasting methods available in pysteps?\n\nBefore starting the comparison, you need to ask yourself what is the objective of nowcasting:\n\n#. Do you only want to minimize prediction errors?\n#. Do you also want to represent the prediction uncertainty?\n\nTo achieve objective 1, it is sufficient to produce a single deterministic nowcast that filters out the unpredictable small-scale precipitation features.\nHowever, this will create a nowcast that will become increasingly smooth over time.\n\nTo achieve objective 2, you need to produce a probabilistic or an ensemble nowcast (several ensemble members or realizations).\n\nIn weather forecasting (and nowcasting), we usually want to achieve both goals because it is impossible to predict the evolution of a chaotic system with 100% accuracy, especially space-time precipitation fields and thunderstorms!\n\nMachine learning and pysteps offer several methods to produce both deterministic and probabilistic nowcasts.\nTherefore, if you want to compare machine learning-based nowcasts to simpler extrapolation-based models, you need to select the right method and verification measure.\n\n1. Deterministic nowcasting\n--------------------------------------------\n\nDeterministic nowcasts can be divided into:\n\na. Variance-preserving nowcasts, such as extrapolation nowcasts by Eulerian and Lagrangian persistence.\nb. Error-minimization nowcasts, such as machine learning, Fourier-filtered and ensemble mean nowcasts.\n\n**Very important**: these two types of deterministic nowcasts are not directly comparable because they have a different variance!\nThis is best explained by the decomposition of the mean squared error (MSE):\n\n:math:`MSE = bias^2 + Var`\n\nAll deterministic machine learning algorithms that minimize the MSE (or a related measure) will also inevitably minimize the variance of nowcast fields.\nThis is a natural attempt to filter out the unpredictable evolution of precipitation features, which would otherwise increase the variance (and the MSE).\nThe same principle holds for convolutional and/or deep neural network architectures, which also produce smooth nowcasts.\n\nTherefore, it is better to avoid directly comparing an error-minimization machine learning nowcast to a variance-preserving radar extrapolation, as produced by the module :py:mod:`pysteps.nowcasts.extrapolation`. Instead, you should use compare with the mean of a sufficiently large ensemble.\n\nA deterministic equivalent of the ensemble mean can be approximated using the modules :py:mod:`pysteps.nowcasts.sprog` or :py:mod:`pysteps.nowcasts.anvil`.\nAnother possibility, but more computationally demanding, is to average many ensemble members generated by the modules :py:mod:`pysteps.nowcasts.steps` or :py:mod:`pysteps.nowcasts.linda`.\n\nStill, even by using the pysteps ensemble mean, it is not given that its variance will be the same as the one of machine learning predictions.\nPossible solutions to this:\n\n#. use a normalized MSE (NMSE) or another score accounting for differences in the variance between prediction and observation.\n#. decompose the field with a Fourier (or wavelet) transform to compare features at the same spatial scales.\n\nA good deterministic comparison of a deep convolutional machine learning neural network nowcast and pysteps is given in :cite:`FNPC2020`.\n\n2. Probabilistic nowcasting\n--------------------------------------------\n\nProbabilistic machine learning regression methods can be roughly categorized into:\n\na. Quantile-based methods, such as quantile regression, quantile random forests, and quantile neural networks.\nb. Ensemble-based methods, such as generative adversarial networks (GANs) and variational auto-encoders (VAEs).\n\nQuantile-based machine learning nowcasts are interesting, but can only estimate the probability of exceedance at a given point (see e.g. :cite:`FSNBG2019`).\n\nTo estimate areal exceedance probabilities, for example above catchments, or to propagate the nowcast uncertainty into hydrological models, the full ensemble still needs to be generated, e.g. with generative machine learning models.\n\nGenerative machine learning methods are similar to the pysteps ensemble members. Both are designed to produce an ensemble of possible realizations that preserve the variance of observed radar fields.\n\nA proper probabilistic verification of generative machine learning models against pysteps is an interesting research direction which was recently undertake in the work of :cite:`Ravuri2021`.\n\nSummary\n-------\nThe table below is an attempt to classify machine learning and pysteps nowcasting methods according to the four main prediction types:\n\n#. Deterministic (variance-preserving), like one control NWP forecast\n#. Deterministic (error-minimization), like an ensemble mean NWP forecast\n#. Probabilistic (quantile-based), like a probabilistic NWP forecast (without members)\n#. Probabilistic (ensemble-based), like the members of an ensemble NWP forecast\n\nThe comparison of methods from different types should only be done carefully and with good reasons.\n\n.. list-table::\n :widths: 30 20 20 20\n :header-rows: 1\n\n * - Nowcast type\n - Machine learning\n - pysteps\n - Verification\n * - Deterministic (variance-preserving)\n - SRGAN, Others?\n - :py:mod:`pysteps.nowcasts.extrapolation` (any optical flow method)\n - MSE, RMSE, MAE, ETS, etc\n * - Deterministic (error-minimization)\n - Classical ANNs, (deep) CNNs, random forests, AdaBoost, etc\n - :py:mod:`pysteps.nowcasts.sprog`, :py:mod:`pysteps.nowcasts.anvil` or ensemble mean of :py:mod:`pysteps.nowcasts.steps`/:py:mod:`~pysteps.nowcasts.linda`\n - MSE, RMSE, MAE, ETS, etc or better normalized scores, etc\n * - Probabilistic (quantile-based)\n - Quantile ANN, quantile random forests, quantile regression\n - :py:mod:`pysteps.nowcasts.lagrangian_probability` or probabilities derived from :py:mod:`pysteps.nowcasts.steps`/:py:mod:`~pysteps.nowcasts.linda`\n - Reliability diagram (predicted vs observed quantile), probability integral transform (PIT) histogram\n * - Probabilistic (ensemble-based)\n - GANs (:cite:`Ravuri2021`), VAEs, etc\n - Ensemble and probabilities derived from :py:mod:`pysteps.nowcasts.steps`/:py:mod:`~pysteps.nowcasts.linda`\n - Probabilistic verification: reliability diagrams, continuous ranked probability scores (CRPS), etc.\n Ensemble verification: rank histograms, spread-error relationships, etc\n" }, { "path": "doc/source/user_guide/pystepsrc_example.rst", "content": ".. _pystepsrc_example:\n\nExample of pystepsrc file\n=========================\n\nBelow you can find the default pystepsrc file.\nThe lines starting with \"//\" are comments and they are ignored.\n\n.. code::\n\n // pysteps configuration\n {\n // \"silent_import\" : whether to suppress the initial pysteps message\n \"silent_import\": false,\n \"outputs\": {\n // path_outputs : path where to save results (figures, forecasts, etc)\n \"path_outputs\": \"./\"\n },\n \"plot\": {\n // \"motion_plot\" : \"streamplot\" or \"quiver\"\n \"motion_plot\": \"quiver\",\n // \"colorscale\" : \"BOM-RF3\", \"pysteps\" or \"STEPS-BE\"\n \"colorscale\": \"pysteps\"\n },\n \"data_sources\": {\n \"bom\": {\n \"root_path\": \"./radar/bom\",\n \"path_fmt\": \"prcp-cscn/2/%Y/%m/%d\",\n \"fn_pattern\": \"2_%Y%m%d_%H%M00.prcp-cscn\",\n \"fn_ext\": \"nc\",\n \"importer\": \"bom_rf3\",\n \"timestep\": 6,\n \"importer_kwargs\": {\n \"gzipped\": true\n }\n },\n \"fmi\": {\n \"root_path\": \"./radar/fmi\",\n \"path_fmt\": \"%Y%m%d\",\n \"fn_pattern\": \"%Y%m%d%H%M_fmi.radar.composite.lowest_FIN_SUOMI1\",\n \"fn_ext\": \"pgm.gz\",\n \"importer\": \"fmi_pgm\",\n \"timestep\": 5,\n \"importer_kwargs\": {\n \"gzipped\": true\n }\n },\n \"mch\": {\n \"root_path\": \"./radar/mch\",\n \"path_fmt\": \"%Y%m%d\",\n \"fn_pattern\": \"AQC%y%j%H%M?_00005.801\",\n \"fn_ext\": \"gif\",\n \"importer\": \"mch_gif\",\n \"timestep\": 5,\n \"importer_kwargs\": {\n \"product\": \"AQC\",\n \"unit\": \"mm\",\n \"accutime\": 5\n }\n },\n \"opera\": {\n \"root_path\": \"./radar/OPERA\",\n \"path_fmt\": \"%Y%m%d\",\n \"fn_pattern\": \"T_PAAH21_C_EUOC_%Y%m%d%H%M%S\",\n \"fn_ext\": \"hdf\",\n \"importer\": \"opera_hdf5\",\n \"timestep\": 15,\n \"importer_kwargs\": {}\n },\n \"knmi\": {\n \"root_path\": \"./radar/KNMI\",\n \"path_fmt\": \"%Y/%m\",\n \"fn_pattern\": \"RAD_NL25_RAP_5min_%Y%m%d%H%M\",\n \"fn_ext\": \"h5\",\n \"importer\": \"knmi_hdf5\",\n \"timestep\": 5,\n \"importer_kwargs\": {\n \"accutime\": 5,\n \"qty\": \"ACRR\",\n \"pixelsize\": 1000.0\n }\n },\n \"saf\": {\n \"root_path\": \"./saf\",\n \"path_fmt\": \"%Y%m%d/CRR\",\n \"fn_pattern\": \"S_NWC_CRR_MSG4_Europe-VISIR_%Y%m%dT%H%M00Z\",\n \"fn_ext\": \"nc\",\n \"importer\": \"saf_crri\",\n \"timestep\": 15,\n \"importer_kwargs\": {\n \"gzipped\": true\n }\n }\n }\n }\n" }, { "path": "doc/source/user_guide/set_pystepsrc.rst", "content": ".. _pystepsrc:\n\nThe pysteps configuration file (pystepsrc)\n==========================================\n\n.. _JSON: https://en.wikipedia.org/wiki/JSON\n\nThe pysteps package allows the users to customize the default settings\nand configuration.\nThe configuration parameters used by default are loaded from a user-defined\nJSON_ file and then stored in `pysteps.rcparams`, a dictionary-like object\nthat can be accessed as attributes or as items.\nFor example, the default parameters can be obtained using any of the following ways::\n\n import pysteps\n\n # Retrieve the colorscale for plots\n colorscale = pysteps.rcparams['plot']['colorscale']\n colorscale = pysteps.rcparams.plot.colorscale\n\n # Retrieve the the root directory of the fmi data\n pysteps.rcparams['data_sources']['fmi']['root_path']\n pysteps.rcparams.data_sources.fmi.root_path\n\nA less wordy alternative::\n\n from pysteps import rcparams\n colorscale = rcparams['plot']['colorscale']\n colorscale = rcparams.plot.colorscale\n\n fmi_root_path = rcparams['data_sources']['fmi']['root_path']\n fmi_root_path = rcparams.data_sources.fmi.root_path\n\n.. _pysteps_lookup:\n\nConfiguration file lookup\n~~~~~~~~~~~~~~~~~~~~~~~~~\n\nWhen the pysteps package imported, it looks for **pystepsrc** file in the\nfollowing order:\n\n- **$PWD/pystepsrc** : Looks for the file in the current directory\n- **$PYSTEPSRC** : If the system variable $PYSTEPSRC is defined and it\n points to a file, it is used.\n- **$PYSTEPSRC/pystepsrc** : If $PYSTEPSRC points to a directory, it looks for the\n pystepsrc file inside that directory.\n- **$HOME/.pysteps/pystepsrc** (Unix and Mac OS X) : If the system variable $HOME is defined, it looks\n for the configuration file in this path.\n- **%USERPROFILE%\\\\pysteps\\\\pystepsrc** (Windows only): It looks for the configuration file\n in the pysteps directory located user's home directory (indicated by the %USERPROFILE%\n system variable).\n- Lastly, it looks inside the library in *pysteps\\\\pystepsrc* for a\n system-defined copy.\n\nThe recommended method to setup the configuration files is to edit a copy\nof the default **pystepsrc** file that is distributed with the package\nand place that copy inside the user home folder.\nSee the instructions below.\n\n\nSetting up the user-defined configuration file\n----------------------------------------------\n\n\nLinux and OSX users\n~~~~~~~~~~~~~~~~~~~\n\nFor Linux and OSX users, the recommended way to customize the pysteps\nconfiguration is placing the pystepsrc parameters file in the users home folder\n${HOME} in the following path: **${HOME}/.pysteps/pystepsrc**\n\nTo steps to setup up the configuration file in the home directory first, we\nneed to create the directory if it does not exist. In a terminal, run::\n\n $ mkdir -p ${HOME}/.pysteps\n\nThe next step is to find the location of the library's default pystepsrc file.\nWhen we import pysteps in a python interpreter, the configuration file loaded\nis shown::\n\n import pysteps\n \"Pysteps configuration file found at: /path/to/pysteps/library/pystepsrc\"\n\nThen we copy the library's default configuration file to that directory::\n\n $ cp /path/to/pysteps/library/pystepsrc ${HOME}/.pysteps/pystepsrc\n\nEdit the file with the text editor of your preference and change the default\nconfigurations with your preferences.\n\nFinally, check that the correct configuration file is loaded by the library::\n\n import pysteps\n \"Pysteps configuration file found at: /home/user_name/.pysteps/pystepsrc\"\n\n\nWindows\n~~~~~~~\n\nFor windows users, the recommended way to customize the pysteps\nconfiguration is placing the pystepsrc parameters file in the users' folder\n(defined in the %USERPROFILE% environment variable) in the following path:\n**%USERPROFILE%\\\\pysteps\\\\pystepsrc**\n\nTo setup up the configuration file in the home directory first, we\nneed to create the directory if it does not exist. In a **windows terminal**, run::\n\n $ mkdir %USERPROFILE%\\pysteps\n\n**Important**\n\nIt was reported that the %USERPROFILE% variable may be interpreted as an string\nliteral when the anaconda terminal is used.\nThis will result in a '%USERPROFILE%' folder being created in the current working directory\ninstead of the desired pysteps folder in the user's home.\nIf that is the case, use the explicit path to your home folder instead of `%USERPROFILE%`.\nFor example::\n\n $ mkdir C:\\Users\\your_username\\pysteps\n\nThe next step is to find the location of the library's default pystepsrc file.\nWhen we import pysteps in a python interpreter, the configuration file loaded\nis shown::\n\n import pysteps\n \"Pysteps configuration file found at: C:\\path\\to\\pysteps\\library\\pystepsrc\"\n\nThen we copy the library's default configuration file to that directory::\n\n $ copy C:\\path\\to\\pysteps\\library\\pystepsrc %USERPROFILE%\\pysteps\\pystepsrc\n\nEdit the file with the text editor of your preference and change the default\nconfigurations with your preferences.\n\nFinally, check that the correct configuration file is loaded by the library::\n\n import pysteps\n \"Pysteps configuration file found at: C:\\User\\Profile\\.pysteps\\pystepsrc\"\n\n\n\nMore\n----\n\n.. toctree::\n :maxdepth: 1\n\n Example pystepsrc file \n" }, { "path": "doc/source/zz_bibliography.rst", "content": ".. _bibliography:\n\n============\nBibliography\n============\n\n\n.. bibliography::\n :all:\n" }, { "path": "environment.yml", "content": "name: pysteps\nchannels:\n- conda-forge\n- defaults\ndependencies:\n - python>=3.10\n - jsmin\n - jsonschema\n - matplotlib\n - netCDF4\n - numpy\n - opencv\n - pillow\n - pyproj\n - scipy\n" }, { "path": "environment_dev.yml", "content": "# pysteps development environment\nname: pysteps_dev\nchannels:\n - conda-forge\n - defaults\ndependencies:\n - python>=3.10\n - pip\n - jsmin\n - jsonschema\n - matplotlib\n - netCDF4\n - numpy\n - opencv\n - pillow\n - pyproj\n - scipy\n - pytest\n - pywavelets\n - cython\n - dask\n - pyfftw\n - h5py\n - PyWavelets\n - pygrib\n - black\n - pytest-cov\n - codecov\n - pre_commit\n - cartopy>=0.18\n - scikit-image\n - scikit-learn\n - pandas\n - rasterio\n" }, { "path": "examples/LK_buffer_mask.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nHandling of no-data in Lucas-Kanade\r\n===================================\r\n\r\nAreas of missing data in radar images are typically caused by visibility limits\r\nsuch as beam blockage and the radar coverage itself. These artifacts can mislead\r\nthe echo tracking algorithms. For instance, precipitation leaving the domain\r\nmight be erroneously detected as having nearly stationary velocity.\r\n\r\nThis example shows how the Lucas-Kanade algorithm can be tuned to avoid the\r\nerroneous interpretation of velocities near the maximum range of the radars by\r\nbuffering the no-data mask in the radar image in order to exclude all vectors\r\ndetected nearby no-data areas.\r\n\"\"\"\r\n\r\nfrom datetime import datetime\r\nfrom matplotlib import cm, colors\r\n\r\nimport matplotlib.pyplot as plt\r\nimport numpy as np\r\n\r\nfrom pysteps import io, motion, nowcasts, rcparams, verification\r\nfrom pysteps.utils import conversion, transformation\r\nfrom pysteps.visualization import plot_precip_field, quiver\r\n\r\n################################################################################\r\n# Read the radar input images\r\n# ---------------------------\r\n#\r\n# First, we will import the sequence of radar composites.\r\n# You need the pysteps-data archive downloaded and the pystepsrc file\r\n# configured with the data_source paths pointing to data folders.\r\n\r\n# Selected case\r\ndate = datetime.strptime(\"201607112100\", \"%Y%m%d%H%M\")\r\ndata_source = rcparams.data_sources[\"mch\"]\r\n\r\n###############################################################################\r\n# Load the data from the archive\r\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\r\n\r\nroot_path = data_source[\"root_path\"]\r\npath_fmt = data_source[\"path_fmt\"]\r\nfn_pattern = data_source[\"fn_pattern\"]\r\nfn_ext = data_source[\"fn_ext\"]\r\nimporter_name = data_source[\"importer\"]\r\nimporter_kwargs = data_source[\"importer_kwargs\"]\r\ntimestep = data_source[\"timestep\"]\r\n\r\n# Find the two input files from the archive\r\nfns = io.archive.find_by_date(\r\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_prev_files=1\r\n)\r\n\r\n# Read the radar composites\r\nimporter = io.get_method(importer_name, \"importer\")\r\nR, quality, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\r\n\r\ndel quality # Not used\r\n\r\n###############################################################################\r\n# Preprocess the data\r\n# ~~~~~~~~~~~~~~~~~~~\r\n\r\n# Convert to mm/h\r\nR, metadata = conversion.to_rainrate(R, metadata)\r\n\r\n# Keep the reference frame in mm/h and its mask (for plotting purposes)\r\nref_mm = R[0, :, :].copy()\r\nmask = np.ones(ref_mm.shape)\r\nmask[~np.isnan(ref_mm)] = np.nan\r\n\r\n# Log-transform the data [dBR]\r\nR, metadata = transformation.dB_transform(R, metadata, threshold=0.1, zerovalue=-15.0)\r\n\r\n# Keep the reference frame in dBR (for plotting purposes)\r\nref_dbr = R[0].copy()\r\nref_dbr[ref_dbr < -10] = np.nan\r\n\r\n# Plot the reference field\r\nplot_precip_field(ref_mm, title=\"Reference field\")\r\ncircle = plt.Circle((620, 400), 100, color=\"b\", clip_on=False, fill=False)\r\nplt.gca().add_artist(circle)\r\nplt.show()\r\n\r\n###############################################################################\r\n# Notice the \"half-in, half-out\" precipitation area within the blue circle.\r\n# As we are going to show next, the tracking algorithm can erroneously interpret\r\n# precipitation leaving the domain as stationary motion.\r\n#\r\n# Also note that the radar image includes NaNs in areas of missing data.\r\n# These are used by the optical flow algorithm to define the radar mask.\r\n#\r\n# Sparse Lucas-Kanade\r\n# -------------------\r\n#\r\n# By setting the optional argument ``dense=False`` in ``xy, uv = dense_lucaskanade(...)``,\r\n# the LK algorithm returns the motion vectors detected by the Lucas-Kanade scheme\r\n# without interpolating them on the grid.\r\n# This allows us to better identify the presence of wrongly detected\r\n# stationary motion in areas where precipitation is leaving the domain (look\r\n# for the red dots within the blue circle in the figure below).\r\n\r\n# Get Lucas-Kanade optical flow method\r\ndense_lucaskanade = motion.get_method(\"LK\")\r\n\r\n# Mask invalid values\r\nR = np.ma.masked_invalid(R)\r\n\r\n# Use no buffering of the radar mask\r\nfd_kwargs1 = {\"buffer_mask\": 0}\r\nxy, uv = dense_lucaskanade(R, dense=False, fd_kwargs=fd_kwargs1)\r\nplt.imshow(ref_dbr, cmap=plt.get_cmap(\"Greys\"))\r\nplt.imshow(mask, cmap=colors.ListedColormap([\"black\"]), alpha=0.5)\r\nplt.quiver(\r\n xy[:, 0],\r\n xy[:, 1],\r\n uv[:, 0],\r\n uv[:, 1],\r\n color=\"red\",\r\n angles=\"xy\",\r\n scale_units=\"xy\",\r\n scale=0.2,\r\n)\r\ncircle = plt.Circle((620, 245), 100, color=\"b\", clip_on=False, fill=False)\r\nplt.gca().add_artist(circle)\r\nplt.title(\"buffer_mask = 0\")\r\nplt.show()\r\n\r\n################################################################################\r\n# The LK algorithm cannot distinguish missing values from no precipitation, that is,\r\n# no-data are the same as no-echoes. As a result, the fixed boundaries produced\r\n# by precipitation in contact with no-data areas are interpreted as stationary motion.\r\n# One way to mitigate this effect of the boundaries is to introduce a slight buffer\r\n# of the no-data mask so that the algorithm will ignore all the portions of the\r\n# radar domain that are nearby no-data areas.\r\n# This buffer can be set by the keyword argument ``buffer_mask`` within the\r\n# feature detection optional arguments ``fd_kwargs``.\r\n# Note that by default ``dense_lucaskanade`` uses a 5-pixel buffer.\r\n\r\n# with buffer\r\nbuffer = 10\r\nfd_kwargs2 = {\"buffer_mask\": buffer}\r\nxy, uv = dense_lucaskanade(R, dense=False, fd_kwargs=fd_kwargs2)\r\nplt.imshow(ref_dbr, cmap=plt.get_cmap(\"Greys\"))\r\nplt.imshow(mask, cmap=colors.ListedColormap([\"black\"]), alpha=0.5)\r\nplt.quiver(\r\n xy[:, 0],\r\n xy[:, 1],\r\n uv[:, 0],\r\n uv[:, 1],\r\n color=\"red\",\r\n angles=\"xy\",\r\n scale_units=\"xy\",\r\n scale=0.2,\r\n)\r\ncircle = plt.Circle((620, 245), 100, color=\"b\", clip_on=False, fill=False)\r\nplt.gca().add_artist(circle)\r\nplt.title(\"buffer_mask = %i\" % buffer)\r\nplt.show()\r\n\r\n################################################################################\r\n# Dense Lucas-Kanade\r\n# ------------------\r\n#\r\n# The above displacement vectors produced by the Lucas-Kanade method are now\r\n# interpolated to produce a full field of motion (i.e., ``dense=True``).\r\n# By comparing the velocity of the motion fields, we can easily notice\r\n# the negative bias that is introduced by the the erroneous interpretation of\r\n# velocities near the maximum range of the radars.\r\n\r\nUV1 = dense_lucaskanade(R, dense=True, fd_kwargs=fd_kwargs1)\r\nUV2 = dense_lucaskanade(R, dense=True, fd_kwargs=fd_kwargs2)\r\n\r\nV1 = np.sqrt(UV1[0] ** 2 + UV1[1] ** 2)\r\nV2 = np.sqrt(UV2[0] ** 2 + UV2[1] ** 2)\r\n\r\nplt.imshow((V1 - V2) / V2, cmap=cm.RdBu_r, vmin=-0.5, vmax=0.5)\r\nplt.colorbar(fraction=0.04, pad=0.04)\r\nplt.title(\"Relative difference in motion speed\")\r\nplt.show()\r\n\r\n################################################################################\r\n# Notice how the presence of erroneous velocity vectors produces a significantly\r\n# slower motion field near the right edge of the domain.\r\n#\r\n# Forecast skill\r\n# --------------\r\n#\r\n# We are now going to evaluate the benefit of buffering the radar mask by computing\r\n# the forecast skill in terms of the Spearman correlation coefficient.\r\n# The extrapolation forecasts are computed using the dense UV motion fields\r\n# estimated above.\r\n\r\n# Get the advection routine and extrapolate the last radar frame by 12 time steps\r\n# (i.e., 1 hour lead time)\r\nextrapolate = nowcasts.get_method(\"extrapolation\")\r\nR[~np.isfinite(R)] = metadata[\"zerovalue\"]\r\nR_f1 = extrapolate(R[-1], UV1, 12)\r\nR_f2 = extrapolate(R[-1], UV2, 12)\r\n\r\n# Back-transform to rain rate\r\nR_f1 = transformation.dB_transform(R_f1, threshold=-10.0, inverse=True)[0]\r\nR_f2 = transformation.dB_transform(R_f2, threshold=-10.0, inverse=True)[0]\r\n\r\n# Find the veriyfing observations in the archive\r\nfns = io.archive.find_by_date(\r\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_next_files=12\r\n)\r\n\r\n# Read and convert the radar composites\r\nR_o, _, metadata_o = io.read_timeseries(fns, importer, **importer_kwargs)\r\nR_o, metadata_o = conversion.to_rainrate(R_o, metadata_o)\r\n\r\n# Compute Spearman correlation\r\nskill = verification.get_method(\"corr_s\")\r\nscore_1 = []\r\nscore_2 = []\r\nfor i in range(12):\r\n score_1.append(skill(R_f1[i, :, :], R_o[i + 1, :, :])[\"corr_s\"])\r\n score_2.append(skill(R_f2[i, :, :], R_o[i + 1, :, :])[\"corr_s\"])\r\n\r\nx = (np.arange(12) + 1) * 5 # [min]\r\nplt.plot(x, score_1, label=\"buffer_mask = 0\")\r\nplt.plot(x, score_2, label=\"buffer_mask = %i\" % buffer)\r\nplt.legend()\r\nplt.xlabel(\"Lead time [min]\")\r\nplt.ylabel(\"Corr. coeff. []\")\r\nplt.title(\"Spearman correlation\")\r\n\r\nplt.tight_layout()\r\nplt.show()\r\n\r\n################################################################################\r\n# As expected, the corrected motion field produces better forecast skill already\r\n# within the first hour into the nowcast.\r\n\r\n# sphinx_gallery_thumbnail_number = 2\r\n" }, { "path": "examples/README.txt", "content": ".. _example_gallery:\n\nExample gallery\n===============\n\nBelow is a collection of example scripts and tutorials to illustrate the usage \nof pysteps.\n\nThese scripts require the pysteps example data.\nSee the installation instructions in the :ref:`example_data` section." }, { "path": "examples/advection_correction.py", "content": "\"\"\"\r\nAdvection correction\r\n====================\r\n\r\nThis tutorial shows how to use the optical flow routines of pysteps to implement\r\nthe advection correction procedure described in Anagnostou and Krajewski (1999).\r\n\r\nAdvection correction is a temporal interpolation procedure that is often used\r\nwhen estimating rainfall accumulations to correct for the shift of rainfall patterns\r\nbetween consecutive radar rainfall maps. This shift becomes particularly\r\nsignificant for long radar scanning cycles and in presence of fast moving\r\nprecipitation features.\r\n\r\n.. note:: The code for the advection correction using pysteps was originally\r\n written by `Daniel Wolfensberger `_.\r\n\r\n\"\"\"\r\n\r\nfrom datetime import datetime\r\nimport matplotlib.pyplot as plt\r\nimport numpy as np\r\n\r\nfrom pysteps import io, motion, rcparams\r\nfrom pysteps.utils import conversion, dimension\r\nfrom pysteps.visualization import plot_precip_field\r\nfrom scipy.ndimage import map_coordinates\r\n\r\n################################################################################\r\n# Read the radar input images\r\n# ---------------------------\r\n#\r\n# First, we import a sequence of 36 images of 5-minute radar composites\r\n# that we will use to produce a 3-hour rainfall accumulation map.\r\n# We will keep only one frame every 10 minutes, to simulate a longer scanning\r\n# cycle and thus better highlight the need for advection correction.\r\n#\r\n# You need the pysteps-data archive downloaded and the pystepsrc file\r\n# configured with the data_source paths pointing to data folders.\r\n\r\n# Selected case\r\ndate = datetime.strptime(\"201607112100\", \"%Y%m%d%H%M\")\r\ndata_source = rcparams.data_sources[\"mch\"]\r\n\r\n###############################################################################\r\n# Load the data from the archive\r\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\r\n\r\nroot_path = data_source[\"root_path\"]\r\npath_fmt = data_source[\"path_fmt\"]\r\nfn_pattern = data_source[\"fn_pattern\"]\r\nfn_ext = data_source[\"fn_ext\"]\r\nimporter_name = data_source[\"importer\"]\r\nimporter_kwargs = data_source[\"importer_kwargs\"]\r\ntimestep = data_source[\"timestep\"]\r\n\r\n# Find the input files from the archive\r\nfns = io.archive.find_by_date(\r\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_next_files=35\r\n)\r\n\r\n# Read the radar composites\r\nimporter = io.get_method(importer_name, \"importer\")\r\nR, __, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\r\n\r\n# Convert to mm/h\r\nR, metadata = conversion.to_rainrate(R, metadata)\r\n\r\n# Upscale to 2 km (simply to reduce the memory demand)\r\nR, metadata = dimension.aggregate_fields_space(R, metadata, 2000)\r\n\r\n# Keep only one frame every 10 minutes (i.e., every 2 timesteps)\r\n# (to highlight the need for advection correction)\r\nR = R[::2]\r\n\r\n################################################################################\r\n# Advection correction\r\n# --------------------\r\n#\r\n# Now we need to implement the advection correction for a pair of successive\r\n# radar images. The procedure is based on the algorithm described in Anagnostou\r\n# and Krajewski (Appendix A, 1999).\r\n#\r\n# To evaluate the advection occurred between two successive radar images, we are\r\n# going to use the Lucas-Kanade optical flow routine available in pysteps.\r\n\r\n\r\ndef advection_correction(R, T=5, t=1):\r\n \"\"\"\r\n R = np.array([qpe_previous, qpe_current])\r\n T = time between two observations (5 min)\r\n t = interpolation timestep (1 min)\r\n \"\"\"\r\n\r\n # Evaluate advection\r\n oflow_method = motion.get_method(\"LK\")\r\n fd_kwargs = {\"buffer_mask\": 10} # avoid edge effects\r\n V = oflow_method(np.log(R), fd_kwargs=fd_kwargs)\r\n\r\n # Perform temporal interpolation\r\n Rd = np.zeros((R[0].shape))\r\n x, y = np.meshgrid(\r\n np.arange(R[0].shape[1], dtype=float), np.arange(R[0].shape[0], dtype=float)\r\n )\r\n for i in range(t, T + t, t):\r\n pos1 = (y - i / T * V[1], x - i / T * V[0])\r\n R1 = map_coordinates(R[0], pos1, order=1)\r\n\r\n pos2 = (y + (T - i) / T * V[1], x + (T - i) / T * V[0])\r\n R2 = map_coordinates(R[1], pos2, order=1)\r\n\r\n Rd += (T - i) * R1 + i * R2\r\n\r\n return t / T**2 * Rd\r\n\r\n\r\n###############################################################################\r\n# Finally, we apply the advection correction to the whole sequence of radar\r\n# images and produce the rainfall accumulation map.\r\n\r\nR_ac = R[0].copy()\r\nfor i in range(R.shape[0] - 1):\r\n R_ac += advection_correction(R[i : (i + 2)], T=10, t=1)\r\nR_ac /= R.shape[0]\r\n\r\n###############################################################################\r\n# Results\r\n# -------\r\n#\r\n# We compare the two accumulation maps. The first map on the left is\r\n# computed without advection correction and we can therefore see that the shift\r\n# between successive images 10 minutes apart produces irregular accumulations.\r\n# Conversely, the rainfall accumulation of the right is produced using advection\r\n# correction to account for this spatial shift. The final result is a smoother\r\n# rainfall accumulation map.\r\n\r\nplt.figure(figsize=(9, 4))\r\nplt.subplot(121)\r\nplot_precip_field(R.mean(axis=0), title=\"3-h rainfall accumulation\")\r\nplt.subplot(122)\r\nplot_precip_field(R_ac, title=\"Same with advection correction\")\r\nplt.tight_layout()\r\nplt.show()\r\n\r\n################################################################################\r\n# Reference\r\n# ~~~~~~~~~\r\n#\r\n# Anagnostou, E. N., and W. F. Krajewski. 1999. \"Real-Time Radar Rainfall\r\n# Estimation. Part I: Algorithm Formulation.\" Journal of Atmospheric and\r\n# Oceanic Technology 16: 189–97.\r\n# https://doi.org/10.1175/1520-0426(1999)016<0189:RTRREP>2.0.CO;2\r\n" }, { "path": "examples/anvil_nowcast.py", "content": "# coding: utf-8\n\n\"\"\"\nANVIL nowcast\n=============\n\nThis example demonstrates how to use ANVIL and the advantages compared to\nextrapolation nowcast and S-PROG.\n\nLoad the libraries.\n\"\"\"\n\nfrom datetime import datetime, timedelta\nimport warnings\n\nwarnings.simplefilter(\"ignore\")\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom pysteps import motion, io, rcparams, utils\nfrom pysteps.nowcasts import anvil, extrapolation, sprog\nfrom pysteps.utils import transformation\nfrom pysteps.visualization import plot_precip_field\n\n###############################################################################\n# Read the input data\n# -------------------\n#\n# ANVIL was originally developed to use vertically integrated liquid (VIL) as\n# the input data, but the model allows using any two-dimensional input fields.\n# Here we use a composite of rain rates.\n\ndate = datetime.strptime(\"201505151620\", \"%Y%m%d%H%M\")\n\n# Read the data source information from rcparams\ndata_source = rcparams.data_sources[\"mch\"]\n\nroot_path = data_source[\"root_path\"]\npath_fmt = data_source[\"path_fmt\"]\nfn_pattern = data_source[\"fn_pattern\"]\nfn_ext = data_source[\"fn_ext\"]\nimporter_name = data_source[\"importer\"]\nimporter_kwargs = data_source[\"importer_kwargs\"]\n\n# Find the input files in the archive. Use history length of 5 timesteps\nfilenames = io.archive.find_by_date(\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_prev_files=5\n)\n\n# Read the input time series\nimporter = io.get_method(importer_name, \"importer\")\nrainrate_field, quality, metadata = io.read_timeseries(\n filenames, importer, **importer_kwargs\n)\n\n# Convert to rain rate (mm/h)\nrainrate_field, metadata = utils.to_rainrate(rainrate_field, metadata)\n\n################################################################################\n# Compute the advection field\n# ---------------------------\n#\n# Apply the Lucas-Kanade method with the parameters given in Pulkkinen et al.\n# (2020) to compute the advection field.\n\nfd_kwargs = {}\nfd_kwargs[\"max_corners\"] = 1000\nfd_kwargs[\"quality_level\"] = 0.01\nfd_kwargs[\"min_distance\"] = 2\nfd_kwargs[\"block_size\"] = 8\n\nlk_kwargs = {}\nlk_kwargs[\"winsize\"] = (15, 15)\n\noflow_kwargs = {}\noflow_kwargs[\"fd_kwargs\"] = fd_kwargs\noflow_kwargs[\"lk_kwargs\"] = lk_kwargs\noflow_kwargs[\"decl_scale\"] = 10\n\noflow = motion.get_method(\"lucaskanade\")\n\n# transform the input data to logarithmic scale\nrainrate_field_log, _ = utils.transformation.dB_transform(\n rainrate_field, metadata=metadata\n)\nvelocity = oflow(rainrate_field_log, **oflow_kwargs)\n\n###############################################################################\n# Compute the nowcasts and threshold rain rates below 0.5 mm/h\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nforecast_extrap = extrapolation.forecast(\n rainrate_field[-1], velocity, 3, extrap_kwargs={\"allow_nonfinite_values\": True}\n)\nforecast_extrap[forecast_extrap < 0.5] = 0.0\n\n# log-transform the data and the threshold value to dBR units for S-PROG\nrainrate_field_db, _ = transformation.dB_transform(\n rainrate_field, metadata, threshold=0.1, zerovalue=-15.0\n)\nrainrate_thr, _ = transformation.dB_transform(\n np.array([0.5]), metadata, threshold=0.1, zerovalue=-15.0\n)\nforecast_sprog = sprog.forecast(\n rainrate_field_db[-3:], velocity, 3, n_cascade_levels=6, precip_thr=rainrate_thr[0]\n)\nforecast_sprog, _ = transformation.dB_transform(\n forecast_sprog, threshold=-10.0, inverse=True\n)\nforecast_sprog[forecast_sprog < 0.5] = 0.0\n\nforecast_anvil = anvil.forecast(\n rainrate_field[-4:], velocity, 3, ar_window_radius=25, ar_order=2\n)\nforecast_anvil[forecast_anvil < 0.5] = 0.0\n\n###############################################################################\n# Read the reference observation field and threshold rain rates below 0.5 mm/h\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nfilenames = io.archive.find_by_date(\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_next_files=3\n)\n\nrefobs_field, _, metadata = io.read_timeseries(filenames, importer, **importer_kwargs)\n\nrefobs_field, metadata = utils.to_rainrate(refobs_field[-1], metadata)\nrefobs_field[refobs_field < 0.5] = 0.0\n\n\n###############################################################################\n# Plot the extrapolation, S-PROG and ANVIL nowcasts.\n# --------------------------------------------------\n#\n# For comparison, the observed rain rate fields are also plotted. Growth and\n# decay areas are marked with red and blue circles, respectively.\ndef plot_growth_decay_circles(ax):\n circle = plt.Circle(\n (360, 300), 25, color=\"b\", clip_on=False, fill=False, zorder=1e9\n )\n ax.add_artist(circle)\n circle = plt.Circle(\n (420, 350), 30, color=\"b\", clip_on=False, fill=False, zorder=1e9\n )\n ax.add_artist(circle)\n circle = plt.Circle(\n (405, 380), 30, color=\"b\", clip_on=False, fill=False, zorder=1e9\n )\n ax.add_artist(circle)\n circle = plt.Circle(\n (420, 500), 25, color=\"b\", clip_on=False, fill=False, zorder=1e9\n )\n ax.add_artist(circle)\n circle = plt.Circle(\n (480, 535), 30, color=\"b\", clip_on=False, fill=False, zorder=1e9\n )\n ax.add_artist(circle)\n circle = plt.Circle(\n (330, 470), 35, color=\"b\", clip_on=False, fill=False, zorder=1e9\n )\n ax.add_artist(circle)\n circle = plt.Circle(\n (505, 205), 30, color=\"b\", clip_on=False, fill=False, zorder=1e9\n )\n ax.add_artist(circle)\n circle = plt.Circle(\n (440, 180), 30, color=\"r\", clip_on=False, fill=False, zorder=1e9\n )\n ax.add_artist(circle)\n circle = plt.Circle(\n (590, 240), 30, color=\"r\", clip_on=False, fill=False, zorder=1e9\n )\n ax.add_artist(circle)\n circle = plt.Circle(\n (585, 160), 15, color=\"r\", clip_on=False, fill=False, zorder=1e9\n )\n ax.add_artist(circle)\n\n\nfig = plt.figure(figsize=(10, 13))\n\nax = fig.add_subplot(321)\nrainrate_field[-1][rainrate_field[-1] < 0.5] = 0.0\nplot_precip_field(rainrate_field[-1])\nplot_growth_decay_circles(ax)\nax.set_title(\"Obs. %s\" % str(date))\n\nax = fig.add_subplot(322)\nplot_precip_field(refobs_field)\nplot_growth_decay_circles(ax)\nax.set_title(\"Obs. %s\" % str(date + timedelta(minutes=15)))\n\nax = fig.add_subplot(323)\nplot_precip_field(forecast_extrap[-1])\nplot_growth_decay_circles(ax)\nax.set_title(\"Extrapolation +15 minutes\")\n\nax = fig.add_subplot(324)\nplot_precip_field(forecast_sprog[-1])\nplot_growth_decay_circles(ax)\nax.set_title(\"S-PROG (with post-processing)\\n +15 minutes\")\n\nax = fig.add_subplot(325)\nplot_precip_field(forecast_anvil[-1])\nplot_growth_decay_circles(ax)\nax.set_title(\"ANVIL +15 minutes\")\n\nplt.show()\n\n###############################################################################\n# Remarks\n# -------\n#\n# The extrapolation nowcast is static, i.e. it does not predict any growth or\n# decay. While S-PROG is to some extent able to predict growth and decay, this\n# this comes with loss of small-scale features. In addition, statistical\n# post-processing needs to be applied to correct the bias and incorrect wet-area\n# ratio introduced by the autoregressive process. ANVIL is able to do both:\n# predict growth and decay and preserve the small-scale structure in a way that\n# post-processing is not necessary.\n" }, { "path": "examples/data_transformations.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\nData transformations\r\n====================\r\n\r\nThe statistics of intermittent precipitation rates are particularly non-Gaussian\r\nand display an asymmetric distribution bounded at zero.\r\nSuch properties restrict the usage of well-established statistical methods that\r\nassume symmetric or Gaussian data.\r\n\r\nA common workaround is to introduce a suitable data transformation to approximate\r\na normal distribution.\r\n\r\nIn this example, we test the data transformation methods available in pysteps\r\nin order to obtain a more symmetric distribution of the precipitation data\r\n(excluding the zeros).\r\nThe currently available transformations include the Box-Cox, dB, square-root and\r\nnormal quantile transforms.\r\n\r\n\"\"\"\r\n\r\nfrom datetime import datetime\r\nimport matplotlib.pyplot as plt\r\nimport numpy as np\r\nfrom pysteps import io, rcparams\r\nfrom pysteps.utils import conversion, transformation\r\nfrom scipy.stats import skew\r\n\r\n###############################################################################\r\n# Read the radar input images\r\n# ---------------------------\r\n#\r\n# First, we will import the sequence of radar composites.\r\n# You need the pysteps-data archive downloaded and the pystepsrc file\r\n# configured with the data_source paths pointing to data folders.\r\n\r\n# Selected case\r\ndate = datetime.strptime(\"201609281600\", \"%Y%m%d%H%M\")\r\ndata_source = rcparams.data_sources[\"fmi\"]\r\n\r\n\r\n###############################################################################\r\n# Load the data from the archive\r\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\r\n\r\nroot_path = data_source[\"root_path\"]\r\npath_fmt = data_source[\"path_fmt\"]\r\nfn_pattern = data_source[\"fn_pattern\"]\r\nfn_ext = data_source[\"fn_ext\"]\r\nimporter_name = data_source[\"importer\"]\r\nimporter_kwargs = data_source[\"importer_kwargs\"]\r\ntimestep = data_source[\"timestep\"]\r\n\r\n# Get 1 hour of observations in the data archive\r\nfns = io.archive.find_by_date(\r\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_next_files=11\r\n)\r\n\r\n# Read the radar composites\r\nimporter = io.get_method(importer_name, \"importer\")\r\nZ, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\r\n\r\n# Keep only positive rainfall values\r\nZ = Z[Z > metadata[\"zerovalue\"]].flatten()\r\n\r\n# Convert to rain rate\r\nR, metadata = conversion.to_rainrate(Z, metadata)\r\n\r\n###############################################################################\r\n# Test data transformations\r\n# -------------------------\r\n\r\n\r\n# Define method to visualize the data distribution with boxplots and plot the\r\n# corresponding skewness\r\ndef plot_distribution(data, labels, skw):\r\n N = len(data)\r\n fig, ax1 = plt.subplots()\r\n ax2 = ax1.twinx()\r\n\r\n ax2.plot(np.arange(N + 2), np.zeros(N + 2), \":r\")\r\n ax1.boxplot(data, labels=labels, sym=\"\", medianprops={\"color\": \"k\"})\r\n\r\n ymax = []\r\n for i in range(N):\r\n y = skw[i]\r\n x = i + 1\r\n ax2.plot(x, y, \"*r\", ms=10, markeredgecolor=\"k\")\r\n ymax.append(np.max(data[i]))\r\n\r\n # ylims\r\n ylims = np.percentile(ymax, 50)\r\n ax1.set_ylim((-1 * ylims, ylims))\r\n ylims = np.max(np.abs(skw))\r\n ax2.set_ylim((-1.1 * ylims, 1.1 * ylims))\r\n\r\n # labels\r\n ax1.set_ylabel(r\"Standardized values [$\\sigma$]\")\r\n ax2.set_ylabel(r\"Skewness []\", color=\"r\")\r\n ax2.tick_params(axis=\"y\", labelcolor=\"r\")\r\n\r\n\r\n###############################################################################\r\n# Box-Cox transform\r\n# ~~~~~~~~~~~~~~~~~\r\n# The Box-Cox transform is a well-known power transformation introduced by\r\n# `Box and Cox (1964)`_. In its one-parameter version, the Box-Cox transform\r\n# takes the form T(x) = ln(x) for lambda = 0, or T(x) = (x**lambda - 1)/lambda\r\n# otherwise.\r\n#\r\n# To find a suitable lambda, we will experiment with a range of values\r\n# and select the one that produces the most symmetric distribution, i.e., the\r\n# lambda associated with a value of skewness closest to zero.\r\n# To visually compare the results, the transformed data are standardized.\r\n#\r\n# .. _`Box and Cox (1964)`: https://doi.org/10.1111/j.2517-6161.1964.tb00553.x\r\n\r\ndata = []\r\nlabels = []\r\nskw = []\r\n\r\n# Test a range of values for the transformation parameter Lambda\r\nLambdas = np.linspace(-0.4, 0.4, 11)\r\nfor i, Lambda in enumerate(Lambdas):\r\n R_, _ = transformation.boxcox_transform(R, metadata, Lambda)\r\n R_ = (R_ - np.mean(R_)) / np.std(R_)\r\n data.append(R_)\r\n labels.append(\"{0:.2f}\".format(Lambda))\r\n skw.append(skew(R_)) # skewness\r\n\r\n# Plot the transformed data distribution as a function of lambda\r\nplot_distribution(data, labels, skw)\r\nplt.title(\"Box-Cox transform\")\r\nplt.tight_layout()\r\nplt.show()\r\n\r\n# Best lambda\r\nidx_best = np.argmin(np.abs(skw))\r\nLambda = Lambdas[idx_best]\r\n\r\nprint(\"Best parameter lambda: %.2f\\n(skewness = %.2f)\" % (Lambda, skw[idx_best]))\r\n\r\n###############################################################################\r\n# Compare data transformations\r\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~\r\n\r\ndata = []\r\nlabels = []\r\nskw = []\r\n\r\n###############################################################################\r\n# Rain rates\r\n# ~~~~~~~~~~\r\n# First, let's have a look at the original rain rate values.\r\n\r\ndata.append((R - np.mean(R)) / np.std(R))\r\nlabels.append(\"R\")\r\nskw.append(skew(R))\r\n\r\n###############################################################################\r\n# dB transform\r\n# ~~~~~~~~~~~~\r\n# We transform the rainfall data into dB units: 10*log(R)\r\n\r\nR_, _ = transformation.dB_transform(R, metadata)\r\ndata.append((R_ - np.mean(R_)) / np.std(R_))\r\nlabels.append(\"dB\")\r\nskw.append(skew(R_))\r\n\r\n###############################################################################\r\n# Square-root transform\r\n# ~~~~~~~~~~~~~~~~~~~~~\r\n# Transform the data using the square-root: sqrt(R)\r\n\r\nR_, _ = transformation.sqrt_transform(R, metadata)\r\ndata.append((R_ - np.mean(R_)) / np.std(R_))\r\nlabels.append(\"sqrt\")\r\nskw.append(skew(R_))\r\n\r\n###############################################################################\r\n# Box-Cox transform\r\n# ~~~~~~~~~~~~~~~~~\r\n# We now apply the Box-Cox transform using the best parameter lambda found above.\r\n\r\nR_, _ = transformation.boxcox_transform(R, metadata, Lambda)\r\ndata.append((R_ - np.mean(R_)) / np.std(R_))\r\nlabels.append(\"Box-Cox\\n($\\lambda=$%.2f)\" % Lambda)\r\nskw.append(skew(R_))\r\n\r\n###############################################################################\r\n# Normal quantile transform\r\n# ~~~~~~~~~~~~~~~~~~~~~~~~~\r\n# At last, we apply the empirical normal quantile (NQ) transform as described in\r\n# `Bogner et al (2012)`_.\r\n#\r\n# .. _`Bogner et al (2012)`: http://dx.doi.org/10.5194/hess-16-1085-2012\r\n\r\nR_, _ = transformation.NQ_transform(R, metadata)\r\ndata.append((R_ - np.mean(R_)) / np.std(R_))\r\nlabels.append(\"NQ\")\r\nskw.append(skew(R_))\r\n\r\n###############################################################################\r\n# By plotting all the results, we can notice first of all the strongly asymmetric\r\n# distribution of the original data (R) and that all transformations manage to\r\n# reduce its skewness. Among these, the Box-Cox transform (using the best parameter\r\n# lambda) and the normal quantile (NQ) transform provide the best correction.\r\n# Despite not producing a perfectly symmetric distribution, the square-root (sqrt)\r\n# transform has the strong advantage of being defined for zeros, too, while all\r\n# other transformations need an arbitrary rule for non-positive values.\r\n\r\nplot_distribution(data, labels, skw)\r\nplt.title(\"Data transforms\")\r\nplt.tight_layout()\r\nplt.show()\r\n" }, { "path": "examples/ens_kalman_filter_blended_forecast.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nEnsemble-based Blending\n=======================\n\nThis tutorial demonstrates how to construct a blended rainfall forecast by combining\nan ensemble nowcast with an ensemble Numerical Weather Prediction (NWP) forecast.\nThe method follows the Reduced-Space Ensemble Kalman Filter approach described in\n:cite:`Nerini2019MWR`.\n\nThe procedure starts from the most recent radar observations. In the **prediction step**,\na stochastic radar extrapolation technique generates short-term forecasts. In the\n**correction step**, these forecasts are updated using information from the latest\nensemble NWP run. To make the matrix operations tractable, the Bayesian update is carried\nout in the subspace defined by the leading principal components—hence the term *reduced\nspace*.\n\nThe datasets used in this tutorial are provided by the German Weather Service (DWD).\n\"\"\"\n\nimport os\nfrom datetime import datetime, timedelta\n\nimport numpy as np\nfrom matplotlib import pyplot as plt\n\nimport pysteps\nfrom pysteps import io, rcparams, blending\nfrom pysteps.utils import aggregate_fields_space\nfrom pysteps.visualization import plot_precip_field\nimport pysteps_nwp_importers\n\n################################################################################\n# Read the radar images and the NWP forecast\n# ------------------------------------------\n#\n# First, we import a sequence of 4 images of 5-minute radar composites\n# and the corresponding NWP rainfall forecast that was available at that time.\n#\n# You need the pysteps-data archive downloaded and the pystepsrc file\n# configured with the data_source paths pointing to data folders.\n# Additionally, the pysteps-nwp-importers plugin needs to be installed, see\n# https://github.com/pySTEPS/pysteps-nwp-importers.\n\n# Selected case\ndate_radar = datetime.strptime(\"202506041645\", \"%Y%m%d%H%M\")\n# The last NWP forecast was issued at 16:00 - the blending tool will be able\n# to find the correct lead times itself.\ndate_nwp = datetime.strptime(\"202506041600\", \"%Y%m%d%H%M\")\nradar_data_source = rcparams.data_sources[\"dwd\"]\nnwp_data_source = rcparams.data_sources[\"dwd_nwp\"]\n\n\n###############################################################################\n# Load the data from the archive\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nroot_path = radar_data_source[\"root_path\"]\npath_fmt = radar_data_source[\"path_fmt\"]\nfn_pattern = radar_data_source[\"fn_pattern\"]\nfn_ext = radar_data_source[\"fn_ext\"]\nimporter_name = radar_data_source[\"importer\"]\nimporter_kwargs = radar_data_source[\"importer_kwargs\"]\ntimestep_radar = radar_data_source[\"timestep\"]\n\n# Find the radar files in the archive\nfns = io.find_by_date(\n date_radar,\n root_path,\n path_fmt,\n fn_pattern,\n fn_ext,\n timestep_radar,\n num_prev_files=2,\n)\n\n# Read the radar composites (which are already in mm/h)\nimporter = io.get_method(importer_name, \"importer\")\nradar_precip, _, radar_metadata = io.read_timeseries(fns, importer, **importer_kwargs)\n\n# Import the NWP data\nfilename = os.path.join(\n nwp_data_source[\"root_path\"],\n datetime.strftime(date_nwp, nwp_data_source[\"path_fmt\"]),\n datetime.strftime(date_nwp, nwp_data_source[\"fn_pattern\"])\n + \".\"\n + nwp_data_source[\"fn_ext\"],\n)\nnwp_importer = io.get_method(\"dwd_nwp\", \"importer\")\nkwargs = nwp_data_source[\"importer_kwargs\"]\n# Resolve grid_file_path relative to PYSTEPS_DATA_PATH\nkwargs[\"grid_file_path\"] = os.path.join(\n os.environ[\"PYSTEPS_DATA_PATH\"], kwargs[\"grid_file_path\"]\n)\nnwp_precip, _, nwp_metadata = nwp_importer(filename, **kwargs)\n# We lower the number of ens members to 10 to reduce the memory needs in the\n# example here. However, it is advised to have a minimum of 20 members for the\n# Reduced-Space Ensemble Kalman filter approach\nnwp_precip = nwp_precip[:, 0:10, :].astype(\"single\")\n\n\n################################################################################\n# Pre-processing steps\n# --------------------\n\n# Set the zerovalue and precipitation thresholds (these are fixed from DWD)\nprec_thr = 0.049\nzerovalue = 0.027\n\n# Transform the zerovalue and precipitation thresholds to dBR\nlog_thr_prec = 10.0 * np.log10(prec_thr)\nlog_zerovalue = 10.0 * np.log10(zerovalue)\n\n# Reproject the DWD ICON NWP data onto a regular grid\nnwp_metadata[\"clon\"] = nwp_precip[\"longitude\"].values\nnwp_metadata[\"clat\"] = nwp_precip[\"latitude\"].values\n# We change the time step from the DWD NWP data to 15 min (it is actually 5 min)\n# to have a longer forecast horizon available for this example, as pysteps_data\n# only contains 1 hour of DWD forecast data (to minimize storage).\nnwp_metadata[\"accutime\"] = 15.0\nnwp_precip = (\n nwp_precip.values.astype(\"single\") * 3.0\n) # (to account for the change in time step from 5 to 15 min)\n\n# Reproject ID2 data onto a regular grid\nnwp_precip_rprj, nwp_metadata_rprj = (\n pysteps_nwp_importers.importer_dwd_nwp.unstructured2regular(\n nwp_precip, nwp_metadata, radar_metadata\n )\n)\nnwp_precip = None\n\n# Upscale both the radar and NWP data to a twice as coarse resolution to lower\n# the memory needs (for this example)\nradar_precip, radar_metadata = aggregate_fields_space(\n radar_precip, radar_metadata, radar_metadata[\"xpixelsize\"] * 4\n)\nnwp_precip_rprj, nwp_metadata_rprj = aggregate_fields_space(\n nwp_precip_rprj.astype(\"single\"),\n nwp_metadata_rprj,\n nwp_metadata_rprj[\"xpixelsize\"] * 4,\n)\n\n# Make sure the units are in mm/h\nconverter = pysteps.utils.get_method(\"mm/h\")\nradar_precip, radar_metadata = converter(\n radar_precip, radar_metadata\n) # The radar data should already be in mm/h\nnwp_precip_rprj, nwp_metadata_rprj = converter(nwp_precip_rprj, nwp_metadata_rprj)\n\n# Threshold the data\nradar_precip[radar_precip < prec_thr] = 0.0\nnwp_precip_rprj[nwp_precip_rprj < prec_thr] = 0.0\n\n# Plot the radar rainfall field and the first time step and first ensemble member\n# of the NWP forecast.\ndate_str = datetime.strftime(date_radar, \"%Y-%m-%d %H:%M\")\nplt.figure(figsize=(10, 5))\nplt.subplot(121)\nplot_precip_field(\n radar_precip[-1, :, :],\n geodata=radar_metadata,\n title=f\"Radar observation at {date_str}\",\n colorscale=\"STEPS-NL\",\n)\nplt.subplot(122)\nplot_precip_field(\n nwp_precip_rprj[0, 0, :, :],\n geodata=nwp_metadata_rprj,\n title=f\"NWP forecast at {date_str}\",\n colorscale=\"STEPS-NL\",\n)\nplt.tight_layout()\nplt.show()\n\n# transform the data to dB\ntransformer = pysteps.utils.get_method(\"dB\")\nradar_precip, radar_metadata = transformer(\n radar_precip, radar_metadata, threshold=prec_thr, zerovalue=log_zerovalue\n)\nnwp_precip_rprj, nwp_metadata_rprj = transformer(\n nwp_precip_rprj, nwp_metadata_rprj, threshold=prec_thr, zerovalue=log_zerovalue\n)\n\n\n###############################################################################\n# Determine the velocity fields\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n#\n# In contrast to the STEPS blending method, no motion field for the NWP fields\n# is needed in the ensemble kalman filter blending approach.\n\n# Estimate the motion vector field\noflow_method = pysteps.motion.get_method(\"lucaskanade\")\nvelocity_radar = oflow_method(radar_precip)\n\n\n################################################################################\n# The blended forecast\n# ~~~~~~~~~~~~~~~~~~~~\n\n# Set the timestamps for radar_precip and nwp_precip_rprj\ntimestamps_radar = np.array(\n sorted(\n [\n date_radar - timedelta(minutes=i * timestep_radar)\n for i in range(len(radar_precip))\n ]\n )\n)\ntimestamps_nwp = np.array(\n sorted(\n [\n date_nwp + timedelta(minutes=i * int(nwp_metadata_rprj[\"accutime\"]))\n for i in range(nwp_precip_rprj.shape[0])\n ]\n )\n)\n\n# Set the combination kwargs\ncombination_kwargs = dict(\n n_tapering=0, # Tapering parameter: controls how many diagonals of the covariance matrix are kept (0 = no tapering)\n non_precip_mask=True, # Specifies whether the computation should be truncated on grid boxes where at least a minimum number of ens. members forecast precipitation.\n n_ens_prec=1, # Minimum number of ens. members that forecast precip for the above-mentioned mask.\n lien_criterion=True, # Specifies wheter the Lien criterion should be applied.\n n_lien=5, # Minimum number of ensemble members that forecast precipitation for the Lien criterion (equals half the ens. members here)\n prob_matching=\"iterative\", # The type of probability matching used.\n inflation_factor_bg=3.0, # Inflation factor of the background (NWC) covariance matrix. (this value indicates a faster convergence towards the NWP ensemble)\n inflation_factor_obs=1.0, # Inflation factor of the observation (NWP) covariance matrix.\n offset_bg=0.0, # Offset of the background (NWC) covariance matrix.\n offset_obs=0.0, # Offset of the observation (NWP) covariance matrix.\n nwp_hres_eff=14.0, # Effective horizontal resolution of the utilized NWP model (in km here).\n sampling_prob_source=\"ensemble\", # Computation method of the sampling probability for the probability matching. 'ensemble' computes this probability as the ratio between the ensemble differences.\n use_accum_sampling_prob=False, # Specifies whether the current sampling probability should be used for the probability matching or a probability integrated over the previous forecast time.\n)\n\n\n# Call the PCA EnKF method\nblending_method = blending.get_method(\"pca_enkf\")\nprecip_forecast = blending_method(\n obs_precip=radar_precip, # Radar data in dBR\n obs_timestamps=timestamps_radar, # Radar timestamps\n nwp_precip=nwp_precip_rprj, # NWP in dBR\n nwp_timestamps=timestamps_nwp, # NWP timestamps\n velocity=velocity_radar, # Velocity vector field\n forecast_horizon=120, # Forecast length (horizon) in minutes - only a short forecast horizon due to the limited dataset length stored here.\n issuetime=date_radar, # Forecast issue time as datetime object\n n_ens_members=10, # No. of ensemble members\n precip_mask_dilation=1, # Dilation of precipitation mask in grid boxes\n n_cascade_levels=6, # No. of cascade levels\n precip_thr=log_thr_prec, # Precip threshold\n norain_thr=0.0005, # Minimum of 0.5% precip needed, otherwise 'zero rainfall'\n num_workers=4, # No. of parallel threads\n noise_stddev_adj=\"auto\", # Standard deviation adjustment\n noise_method=\"ssft\", # SSFT as noise method\n enable_combination=True, # Enable combination\n noise_kwargs={\"win_size\": (512, 512), \"win_fun\": \"hann\", \"overlap\": 0.5},\n extrap_kwargs={\"interp_order\": 3, \"map_coordinates_mode\": \"nearest\"},\n combination_kwargs=combination_kwargs,\n filter_kwargs={\"include_mean\": True},\n)\n\n# Transform the data back into mm/h\nprecip_forecast, _ = converter(precip_forecast, radar_metadata)\nradar_precip, _ = converter(radar_precip, radar_metadata)\nnwp_precip, _ = converter(nwp_precip_rprj, nwp_metadata_rprj)\n\n\n################################################################################\n# Visualize the output\n# ~~~~~~~~~~~~~~~~~~~~\n#\n# The NWP rainfall forecast has a much lower weight than the radar-based\n# extrapolation # forecast at the issue time of the forecast (+0 min). Therefore,\n# the first time steps consist mostly of the extrapolation. However, near the end\n# of the forecast (+180 min), the NWP share in the blended forecast has become\n# the more dominant contribution to the forecast and thus the forecast starts\n# to resemble the NWP forecast.\n\nfig = plt.figure(figsize=(5, 12))\n\nleadtimes_min = [15, 30, 45, 60, 90, 120]\nn_leadtimes = len(leadtimes_min)\nfor n, leadtime in enumerate(leadtimes_min):\n # Nowcast with blending into NWP\n plt.subplot(n_leadtimes, 2, n * 2 + 1)\n plot_precip_field(\n precip_forecast[0, int(leadtime / timestep_radar) - 1, :, :],\n geodata=radar_metadata,\n title=f\"Blended +{leadtime} min\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n\n # Raw NWP forecast\n plt.subplot(n_leadtimes, 2, n * 2 + 2)\n plot_precip_field(\n nwp_precip[int(leadtime / int(nwp_metadata_rprj[\"accutime\"])) - 1, 0, :, :],\n geodata=nwp_metadata_rprj,\n title=f\"NWP +{leadtime} min\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n\n\n################################################################################\n# References\n# ~~~~~~~~~~\n#\n\n# Nerini, D., Foresti, L., Leuenberger, D., Robert, S., Germann, U. 2019. \"A\n# Reduced-Space Ensemble Kalman Filter Approach for Flow-Dependent Integration\n# of Radar Extrapolation Nowcasts and NWP Precipitation Ensembles.\" Monthly\n# Weather Review 147(3): 987-1006. https://doi.org/10.1175/MWR-D-18-0258.1.\n" }, { "path": "examples/linda_nowcasts.py", "content": "#!/bin/env python\n\"\"\"\nLINDA nowcasts\n==============\n\nThis example shows how to compute and plot a deterministic and ensemble LINDA\nnowcasts using Swiss radar data.\n\n\"\"\"\n\nfrom datetime import datetime\nimport warnings\n\nwarnings.simplefilter(\"ignore\")\n\nimport matplotlib.pyplot as plt\n\nfrom pysteps import io, rcparams\nfrom pysteps.motion.lucaskanade import dense_lucaskanade\nfrom pysteps.nowcasts import linda, sprog, steps\nfrom pysteps.utils import conversion, dimension, transformation\nfrom pysteps.visualization import plot_precip_field\n\n###############################################################################\n# Read the input rain rate fields\n# -------------------------------\n\ndate = datetime.strptime(\"201701311200\", \"%Y%m%d%H%M\")\ndata_source = \"mch\"\n\n# Read the data source information from rcparams\ndatasource_params = rcparams.data_sources[data_source]\n\n# Find the radar files in the archive\nfns = io.find_by_date(\n date,\n datasource_params[\"root_path\"],\n datasource_params[\"path_fmt\"],\n datasource_params[\"fn_pattern\"],\n datasource_params[\"fn_ext\"],\n datasource_params[\"timestep\"],\n num_prev_files=2,\n)\n\n# Read the data from the archive\nimporter = io.get_method(datasource_params[\"importer\"], \"importer\")\nreflectivity, _, metadata = io.read_timeseries(\n fns, importer, **datasource_params[\"importer_kwargs\"]\n)\n\n# Convert reflectivity to rain rate\nrainrate, metadata = conversion.to_rainrate(reflectivity, metadata)\n\n# Upscale data to 2 km to reduce computation time\nrainrate, metadata = dimension.aggregate_fields_space(rainrate, metadata, 2000)\n\n# Plot the most recent rain rate field\nplt.figure()\nplot_precip_field(rainrate[-1, :, :])\nplt.show()\n\n###############################################################################\n# Estimate the advection field\n# ----------------------------\n\n# The advection field is estimated using the Lucas-Kanade optical flow\nadvection = dense_lucaskanade(rainrate, verbose=True)\n\n###############################################################################\n# Deterministic nowcast\n# ---------------------\n\n# Compute 30-minute LINDA nowcast with 8 parallel workers\n# Restrict the number of features to 15 to reduce computation time\nnowcast_linda = linda.forecast(\n rainrate,\n advection,\n 6,\n max_num_features=15,\n add_perturbations=False,\n num_workers=8,\n measure_time=True,\n)[0]\n\n# Compute S-PROG nowcast for comparison\nrainrate_db, _ = transformation.dB_transform(\n rainrate, metadata, threshold=0.1, zerovalue=-15.0\n)\nnowcast_sprog = sprog.forecast(\n rainrate_db[-3:, :, :],\n advection,\n 6,\n n_cascade_levels=6,\n precip_thr=-10.0,\n)\n\n# Convert reflectivity nowcast to rain rate\nnowcast_sprog = transformation.dB_transform(\n nowcast_sprog, threshold=-10.0, inverse=True\n)[0]\n\n# Plot the nowcasts\nfig = plt.figure(figsize=(9, 4))\nax = fig.add_subplot(1, 2, 1)\nplot_precip_field(\n nowcast_linda[-1, :, :],\n title=\"LINDA (+ 30 min)\",\n)\n\nax = fig.add_subplot(1, 2, 2)\nplot_precip_field(\n nowcast_sprog[-1, :, :],\n title=\"S-PROG (+ 30 min)\",\n)\n\nplt.show()\n\n###############################################################################\n# The above figure shows that the filtering scheme implemented in LINDA preserves\n# small-scale and band-shaped features better than S-PROG. This is because the\n# former uses a localized elliptical convolution kernel instead of the\n# cascade-based autoregressive process, where the parameters are estimated over\n# the whole domain.\n\n###############################################################################\n# Probabilistic nowcast\n# ---------------------\n\n# Compute 30-minute LINDA nowcast ensemble with 40 members and 8 parallel workers\nnowcast_linda = linda.forecast(\n rainrate,\n advection,\n 6,\n max_num_features=15,\n add_perturbations=True,\n vel_pert_method=None,\n n_ens_members=40,\n num_workers=8,\n measure_time=True,\n)[0]\n\n# Compute 40-member STEPS nowcast for comparison\nnowcast_steps = steps.forecast(\n rainrate_db[-3:, :, :],\n advection,\n 6,\n 40,\n n_cascade_levels=6,\n precip_thr=-10.0,\n mask_method=\"incremental\",\n kmperpixel=2.0,\n timestep=datasource_params[\"timestep\"],\n vel_pert_method=None,\n)\n\n# Convert reflectivity nowcast to rain rate\nnowcast_steps = transformation.dB_transform(\n nowcast_steps, threshold=-10.0, inverse=True\n)[0]\n\n# Plot two ensemble members of both nowcasts\nfig = plt.figure()\nfor i in range(2):\n ax = fig.add_subplot(2, 2, i + 1)\n ax = plot_precip_field(\n nowcast_linda[i, -1, :, :], geodata=metadata, colorbar=False, axis=\"off\"\n )\n ax.set_title(f\"LINDA Member {i+1}\")\n\nfor i in range(2):\n ax = fig.add_subplot(2, 2, 3 + i)\n ax = plot_precip_field(\n nowcast_steps[i, -1, :, :], geodata=metadata, colorbar=False, axis=\"off\"\n )\n ax.set_title(f\"STEPS Member {i+1}\")\n\n###############################################################################\n# The above figure shows the main difference between LINDA and STEPS. In\n# addition to the convolution kernel, another improvement in LINDA is a\n# localized perturbation generator using the short-space Fourier transform\n# (SSFT) and a spatially variable marginal distribution. As a result, the\n# LINDA ensemble members preserve the anisotropic and small-scale structures\n# considerably better than STEPS.\n\nplt.tight_layout()\nplt.show()\n" }, { "path": "examples/my_first_nowcast.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"L_dntwSQBnbK\"\n },\n \"source\": [\n \"[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/pySTEPS/pysteps/blob/master/examples/my_first_nowcast.ipynb)\\n\",\n \"\\n\",\n \"# My first precipitation nowcast\\n\",\n \"\\n\",\n \"In this example, we will use pysteps to compute and plot an extrapolation nowcast using the NSSL's Multi-Radar/Multi-Sensor System\\n\",\n \"([MRMS](https://www.nssl.noaa.gov/projects/mrms/)) rain rate product.\\n\",\n \"\\n\",\n \"The MRMS precipitation product is available every 2 minutes, over the contiguous US. \\n\",\n \"Each precipitation composite has 3500 x 7000 grid points, separated 1 km from each other.\\n\",\n \"\\n\",\n \"## Set-up Colab environment\\n\",\n \"\\n\",\n \"**Important**: In colab, execute this section one cell at a time. Trying to excecute all the cells at once may results in cells being skipped and some dependencies not being installed.\\n\",\n \"\\n\",\n \"First, let's set up our working environment. Note that these steps are only needed to work with google colab. \\n\",\n \"\\n\",\n \"To install pysteps locally, you can follow [these instructions](https://pysteps.readthedocs.io/en/latest/user_guide/install_pysteps.html).\\n\",\n \"\\n\",\n \"First, let's install the latest Pysteps version from the Python Package Index (PyPI) using pip. This will also install the minimal dependencies needed to run pysteps. \\n\",\n \"\\n\",\n \"#### Install optional dependencies\\n\",\n \"\\n\",\n \"Now, let's install the optional dependendies that will allow us to plot and read the example data.\\n\",\n \"- pygrib: to read the MRMS data grib format\\n\",\n \"- pyproj: needed by pygrib\\n\",\n \"\\n\",\n \"**NOTE:** Do not import pysteps in this notebook until the following optional dependencies are loaded. Otherwise, pysteps will assume that they are not installed and some of its functionalities won't work.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"mFx4hq_DBtp-\"\n },\n \"outputs\": [],\n \"source\": [\n \"# These libraries are needed for the pygrib library in Colab.\\n\",\n \"# Note that is needed if you install pygrib using pip.\\n\",\n \"# If you use conda, the libraries will be installed automatically.\\n\",\n \"! apt-get install libeccodes-dev libproj-dev\\n\",\n \"\\n\",\n \"# Install the python packages\\n\",\n \"! pip install pyproj\\n\",\n \"! pip install pygrib\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"6BF2paxnTuGB\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Uninstall existing shapely\\n\",\n \"# We will re-install shapely in the next step by ignoring the binary\\n\",\n \"# wheels to make it compatible with other modules that depend on\\n\",\n \"# GEOS, such as Cartopy (used here).\\n\",\n \"!pip uninstall --yes shapely\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"7x8Hx_4hE_BU\"\n },\n \"outputs\": [],\n \"source\": [\n \"# To install cartopy in Colab using pip, we need to install the library\\n\",\n \"# dependencies first.\\n\",\n \"\\n\",\n \"!apt-get install -qq libgdal-dev libgeos-dev\\n\",\n \"!pip install shapely --no-binary shapely\\n\",\n \"!pip install cartopy\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"ybD55ZJhmdYa\"\n },\n \"source\": [\n \"#### Install pysteps\\n\",\n \"\\n\",\n \"Now that all dependencies are installed, we can install pysteps.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"VA7zp3nRmhfF\"\n },\n \"outputs\": [],\n \"source\": [\n \"# ! pip install git+https://github.com/pySTEPS/pysteps\\n\",\n \"! pip install pysteps\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"-AkfR6JSBujn\"\n },\n \"source\": [\n \"## Getting the example data\\n\",\n \"\\n\",\n \"Now that we have the environment ready, let's install the example data and configure the pysteps's default parameters by following [this tutorial](https://pysteps.readthedocs.io/en/latest/user_guide/example_data.html).\\n\",\n \"\\n\",\n \"First, we will use the [pysteps.datasets.download_pysteps_data()](https://pysteps.readthedocs.io/en/latest/generated/pysteps.datasets.download_pysteps_data.html) function to download the data.\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"vri-R_ZVGihj\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Import the helper functions\\n\",\n \"from pysteps.datasets import download_pysteps_data, create_default_pystepsrc\\n\",\n \"\\n\",\n \"# Download the pysteps data in the \\\"pysteps_data\\\"\\n\",\n \"download_pysteps_data(\\\"pysteps_data\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"wdKfjliCKXhx\"\n },\n \"source\": [\n \"Next, we need to create a default configuration file that points to the downloaded data. \\n\",\n \"By default, pysteps will place the configuration file in `$HOME/.pysteps` (unix and Mac OS X) or `$USERPROFILE/pysteps` (windows).\\n\",\n \"To quickly create a configuration file, we will use the [pysteps.datasets.create_default_pystepsrc()](https://pysteps.readthedocs.io/en/latest/generated/pysteps.datasets.create_default_pystepsrc.html#pysteps.datasets.create_default_pystepsrc) helper function.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"pGdKHa36H5JX\"\n },\n \"outputs\": [],\n \"source\": [\n \"# If the configuration file is placed in one of the default locations\\n\",\n \"# (https://pysteps.readthedocs.io/en/latest/user_guide/set_pystepsrc.html#configuration-file-lookup)\\n\",\n \"# it will be loaded automatically when pysteps is imported.\\n\",\n \"config_file_path = create_default_pystepsrc(\\\"pysteps_data\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"DAFUJgR5K1CS\"\n },\n \"source\": [\n \"Since pysteps was already initialized in this notebook, we need to load the new configuration file and update the default configuration.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"tMIbQLPAK42h\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Import pysteps and load the new configuration file\\n\",\n \"import pysteps\\n\",\n \"\\n\",\n \"_ = pysteps.load_config_file(config_file_path, verbose=True)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"SzSqp1DFJ0M9\"\n },\n \"source\": [\n \"Let's see what the default parameters look like (these are stored in the\\n\",\n \"[pystepsrc file](https://pysteps.readthedocs.io/en/latest/user_guide/set_pystepsrc.html)). We will be using them to load the MRMS data set.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"6Gr65nH4BnbP\"\n },\n \"outputs\": [],\n \"source\": [\n \"# The default parameters are stored in pysteps.rcparams.\\n\",\n \"from pprint import pprint\\n\",\n \"\\n\",\n \"pprint(pysteps.rcparams.data_sources[\\\"mrms\\\"])\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"9M_buv7WBnbf\"\n },\n \"source\": [\n \"This should have printed the following lines:\\n\",\n \"\\n\",\n \"- `fn_ext`: 'grib2' -- The file extension\\n\",\n \"- `fn_pattern`: 'PrecipRate_00.00_%Y%m%d-%H%M%S' -- The file naming convention of the MRMS data.\\n\",\n \"- `importer`: 'mrms_grib' -- The name of the importer for the MRMS data.\\n\",\n \"- `importer_kwargs`: {} -- Extra options provided to the importer. None in this example.\\n\",\n \"- `path_fmt`: '%Y/%m/%d' -- The folder structure in which the files are stored. Here, year/month/day/filename.\\n\",\n \"- `root_path`: '/content/pysteps_data/mrms' -- The root path of the MRMS-data.\\n\",\n \"- `timestep`: 2 -- The temporal interval of the (radar) rainfall data\\n\",\n \"\\n\",\n \"Note that the default `timestep` parameter is 2 minutes, which corresponds to the time interval at which the MRMS product is available.\\n\",\n \"\\n\",\n \"## Load the MRMS example data\\n\",\n \"\\n\",\n \"Now that we have installed the example data, let's import the example MRMS dataset using the [load_dataset()](https://pysteps.readthedocs.io/en/latest/generated/pysteps.datasets.load_dataset.html) helper function from the `pysteps.datasets` module.\\n\",\n \"\\n\",\n \"We import 1 hour and 10 minutes of data, which corresponds to a sequence of 35 frames of 2-D precipitation composites.\\n\",\n \"Note that importing the data takes approximately 30 seconds.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"-8Q4e58VBnbl\"\n },\n \"outputs\": [],\n \"source\": [\n \"from pysteps.datasets import load_dataset\\n\",\n \"\\n\",\n \"# We'll import the time module to measure the time the importer needed\\n\",\n \"import time\\n\",\n \"\\n\",\n \"start_time = time.time()\\n\",\n \"\\n\",\n \"# Import the data\\n\",\n \"precipitation, metadata, timestep = load_dataset(\\n\",\n \" \\\"mrms\\\", frames=35\\n\",\n \") # precipitation in mm/h\\n\",\n \"\\n\",\n \"end_time = time.time()\\n\",\n \"\\n\",\n \"print(\\\"Precipitation data imported\\\")\\n\",\n \"print(\\\"Importing the data took \\\", (end_time - start_time), \\\" seconds\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"btiTxYYMBnby\"\n },\n \"source\": [\n \"Let's have a look at the values returned by the `load_dataset()` function. \\n\",\n \"\\n\",\n \"- `precipitation`: A numpy array with (time, latitude, longitude) dimensions.\\n\",\n \"- `metadata`: A dictionary with additional information (pixel sizes, map projections, etc.).\\n\",\n \"- `timestep`: Time separation between each sample (in minutes)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"WqUHbJ_qBnb3\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Let's inspect the shape of the imported data array\\n\",\n \"precipitation.shape\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"xa8woT0ABncD\"\n },\n \"source\": [\n \"Note that the shape of the precipitation is 4 times smaller than the raw MRMS data (3500 x 7000).\\n\",\n \"The `load_dataset()` function uses the default parameters from `importers` to read the data. By default, the MRMS importer upscales the data 4x. That is, from ~1km resolution to ~4km. It also uses single precision to reduce the memory requirements.\\n\",\n \"Thanks to the upscaling, the memory footprint of this example dataset is ~200Mb instead of the 3.1Gb of the raw (3500 x 7000) data. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"22O2YXrfBncG\"\n },\n \"outputs\": [],\n \"source\": [\n \"timestep # In minutes\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"J8_4hwcXBncT\"\n },\n \"outputs\": [],\n \"source\": [\n \"pprint(metadata)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"uQREORtJBnch\"\n },\n \"source\": [\n \"# Time to make a nowcast\\n\",\n \"\\n\",\n \"So far, we have 1 hour and 10 minutes of precipitation images, separated 2 minutes from each other.\\n\",\n \"But, how do we use that data to run a precipitation forecast? \\n\",\n \"\\n\",\n \"A simple way is by extrapolating the precipitation field, assuming it will continue to move as observed in the recent past, and without changes in intensity. This is commonly known as *Lagrangian persistence*.\\n\",\n \"\\n\",\n \"The first step to run our nowcast based on Lagrangian persistence, is the estimation of the motion field from a sequence of past precipitation observations.\\n\",\n \"We use the Lucas-Kanade (LK) optical flow method implemented in pysteps.\\n\",\n \"This method follows a local tracking approach that relies on the OpenCV package.\\n\",\n \"Local features are tracked in a sequence of two or more radar images.\\n\",\n \"The scheme includes a final interpolation step to produce a smooth field of motion vectors.\\n\",\n \"Other optical flow methods are also available in pysteps. \\n\",\n \"Check the full list [here](https://pysteps.readthedocs.io/en/latest/pysteps_reference/motion.html).\\n\",\n \"\\n\",\n \"Now let's use the first 5 precipitation images (10 min) to estimate the motion field of the radar pattern and the remaining 30 images (1h) to evaluate the quality of our forecast.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"jcb2Sf6xBnck\"\n },\n \"outputs\": [],\n \"source\": [\n \"# precipitation[0:5] -> Used to find motion (past data). Let's call it training precip.\\n\",\n \"train_precip = precipitation[0:5]\\n\",\n \"\\n\",\n \"# precipitation[5:] -> Used to evaluate forecasts (future data, not available in \\\"real\\\" forecast situation)\\n\",\n \"# Let's call it observed precipitation because we will use it to compare our forecast with the actual observations.\\n\",\n \"observed_precip = precipitation[3:]\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"xt1TbB0RBncu\"\n },\n \"source\": [\n \"Let's see what this 'training' precipitation event looks like using the [pysteps.visualization.plot_precip_field](https://pysteps.readthedocs.io/en/latest/generated/pysteps.visualization.precipfields.plot_precip_field.html) function.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"bmNYLo1jBncw\"\n },\n \"outputs\": [],\n \"source\": [\n \"from matplotlib import pyplot as plt\\n\",\n \"from pysteps.visualization import plot_precip_field\\n\",\n \"\\n\",\n \"# Set a figure size that looks nice ;)\\n\",\n \"plt.figure(figsize=(9, 5), dpi=100)\\n\",\n \"\\n\",\n \"# Plot the last rainfall field in the \\\"training\\\" data.\\n\",\n \"# train_precip[-1] -> Last available composite for nowcasting.\\n\",\n \"plot_precip_field(train_precip[-1], geodata=metadata, axis=\\\"off\\\")\\n\",\n \"plt.show() # (This line is actually not needed if you are using jupyter notebooks)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"NVRfJm11Bnc7\"\n },\n \"source\": [\n \"Did you note the **shaded grey** regions? Those are the regions were no valid observations where available to estimate the precipitation (e.g., due to ground clutter, no radar coverage, or radar beam blockage).\\n\",\n \"Those regions need to be handled with care when we run our nowcast.\\n\",\n \"\\n\",\n \"### Data exploration\\n\",\n \"\\n\",\n \"Before we produce a forecast, let's explore the precipitation data. In particular, let's see how the distribution of the rain rate values looks.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"WER6RttPBnc9\"\n },\n \"outputs\": [],\n \"source\": [\n \"import numpy as np\\n\",\n \"\\n\",\n \"# Let's define some plotting default parameters for the next plots\\n\",\n \"# Note: This is not strictly needed.\\n\",\n \"plt.rc(\\\"figure\\\", figsize=(4, 4))\\n\",\n \"plt.rc(\\\"figure\\\", dpi=100)\\n\",\n \"plt.rc(\\\"font\\\", size=14) # controls default text sizes\\n\",\n \"plt.rc(\\\"axes\\\", titlesize=14) # fontsize of the axes title\\n\",\n \"plt.rc(\\\"axes\\\", labelsize=14) # fontsize of the x and y labels\\n\",\n \"plt.rc(\\\"xtick\\\", labelsize=14) # fontsize of the tick labels\\n\",\n \"plt.rc(\\\"ytick\\\", labelsize=14) # fontsize of the tick labels\\n\",\n \"\\n\",\n \"# Let's use the last available composite for nowcasting from the \\\"training\\\" data (train_precip[-1])\\n\",\n \"# Also, we will discard any invalid value.\\n\",\n \"valid_precip_values = train_precip[-1][~np.isnan(train_precip[-1])]\\n\",\n \"\\n\",\n \"# Plot the histogram\\n\",\n \"bins = np.concatenate(([-0.01, 0.01], np.linspace(1, 40, 39)))\\n\",\n \"plt.hist(valid_precip_values, bins=bins, log=True, edgecolor=\\\"black\\\")\\n\",\n \"plt.autoscale(tight=True, axis=\\\"x\\\")\\n\",\n \"plt.xlabel(\\\"Rainfall intensity [mm/h]\\\")\\n\",\n \"plt.ylabel(\\\"Counts\\\")\\n\",\n \"plt.title(\\\"Precipitation rain rate histogram in mm/h units\\\")\\n\",\n \"plt.show()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"O6TvIXS3BndH\"\n },\n \"source\": [\n \"The histogram shows that rain rate values have a non-Gaussian and asymmetric distribution that is bounded at zero. Also, the probability of occurrence decays extremely fast with increasing rain rate values (note the logarithmic y-axis).\\n\",\n \"\\n\",\n \"\\n\",\n \"For better performance of the motion estimation algorithms, we can convert the rain rate values (in mm/h) to a more log-normal distribution of rain rates by applying the following logarithmic transformation:\\n\",\n \"\\n\",\n \"\\\\begin{equation}\\n\",\n \"R\\\\rightarrow\\n\",\n \"\\\\begin{cases}\\n\",\n \" 10\\\\log_{10}R, & \\\\text{if } R\\\\geq 0.1\\\\text{mm h$^{-1}$} \\\\\\\\\\n\",\n \" -15, & \\\\text{otherwise}\\n\",\n \"\\\\end{cases}\\n\",\n \"\\\\end{equation}\\n\",\n \"\\n\",\n \"The transformed precipitation corresponds to logarithmic rain rates in units of dBR. The value of −15 dBR is equivalent to assigning a rain rate of approximately 0.03 mm h$^{−1}$ to the zeros. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"hgA4PeapBndK\"\n },\n \"outputs\": [],\n \"source\": [\n \"from pysteps.utils import transformation\\n\",\n \"\\n\",\n \"# Log-transform the data to dBR.\\n\",\n \"# The threshold of 0.1 mm/h sets the fill value to -15 dBR.\\n\",\n \"train_precip_dbr, metadata_dbr = transformation.dB_transform(\\n\",\n \" train_precip, metadata, threshold=0.1, zerovalue=-15.0\\n\",\n \")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"Nx3VESBlBndU\"\n },\n \"source\": [\n \"Let's inspect the resulting **transformed precipitation** distribution.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"rYS5hBIGBndX\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Only use the valid data!\\n\",\n \"valid_precip_dbr = train_precip_dbr[-1][~np.isnan(train_precip_dbr[-1])]\\n\",\n \"\\n\",\n \"plt.figure(figsize=(4, 4), dpi=100)\\n\",\n \"\\n\",\n \"# Plot the histogram\\n\",\n \"counts, bins, _ = plt.hist(valid_precip_dbr, bins=40, log=True, edgecolor=\\\"black\\\")\\n\",\n \"plt.autoscale(tight=True, axis=\\\"x\\\")\\n\",\n \"plt.xlabel(\\\"Rainfall intensity [dB]\\\")\\n\",\n \"plt.ylabel(\\\"Counts\\\")\\n\",\n \"plt.title(\\\"Precipitation rain rate histogram in dB units\\\")\\n\",\n \"\\n\",\n \"# Let's add a lognormal distribution that fits that data to the plot.\\n\",\n \"import scipy\\n\",\n \"\\n\",\n \"bin_center = (bins[1:] + bins[:-1]) * 0.5\\n\",\n \"bin_width = np.diff(bins)\\n\",\n \"\\n\",\n \"# We will only use one composite to fit the function to speed up things.\\n\",\n \"# First, remove the no precip areas.\\\"\\n\",\n \"precip_to_fit = valid_precip_dbr[valid_precip_dbr > -15]\\n\",\n \"\\n\",\n \"fit_params = scipy.stats.lognorm.fit(precip_to_fit)\\n\",\n \"\\n\",\n \"fitted_pdf = scipy.stats.lognorm.pdf(bin_center, *fit_params)\\n\",\n \"\\n\",\n \"# Multiply pdf by the bin width and the total number of grid points: pdf -> total counts per bin.\\n\",\n \"fitted_pdf = fitted_pdf * bin_width * precip_to_fit.size\\n\",\n \"\\n\",\n \"# Plot the log-normal fit\\n\",\n \"plt.plot(bin_center, fitted_pdf, label=\\\"Fitted log-normal\\\")\\n\",\n \"plt.legend()\\n\",\n \"plt.show()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"ZocO5zqUBndg\"\n },\n \"source\": [\n \"That looks more like a log-normal distribution. Note the large peak at -15dB. That peak corresponds to \\\"zero\\\" (below threshold) precipitation. The jump with no data in between -15 and -10 dB is caused by the precision of the data, which we had set to 1 decimal. Hence, the lowest precipitation intensities (above zero) are 0.1 mm/h (= -10 dB).\\n\",\n \"\\n\",\n \"## Compute the nowcast\\n\",\n \"\\n\",\n \"These are the minimal steps to compute a short-term forecast using Lagrangian extrapolation of the precipitation patterns:\\n\",\n \" \\n\",\n \" 1. Estimate the precipitation motion field.\\n\",\n \" 1. Use the motion field to advect the most recent radar rainfall field and produce an extrapolation forecast.\\n\",\n \"\\n\",\n \"### Estimate the motion field\\n\",\n \"\\n\",\n \"Now we can estimate the motion field. Here we use a local feature-tracking approach (Lucas-Kanade).\\n\",\n \"However, check the other methods available in the [pysteps.motion](https://pysteps.readthedocs.io/en/latest/pysteps_reference/motion.html) module.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"mnACmX_0Bndi\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Estimate the motion field with Lucas-Kanade\\n\",\n \"from pysteps import motion\\n\",\n \"from pysteps.visualization import plot_precip_field, quiver\\n\",\n \"\\n\",\n \"# Import the Lucas-Kanade optical flow algorithm\\n\",\n \"oflow_method = motion.get_method(\\\"LK\\\")\\n\",\n \"\\n\",\n \"# Estimate the motion field from the training data (in dBR)\\n\",\n \"motion_field = oflow_method(train_precip_dbr)\\n\",\n \"\\n\",\n \"## Plot the motion field.\\n\",\n \"# Use a figure size that looks nice ;)\\n\",\n \"plt.figure(figsize=(9, 5), dpi=100)\\n\",\n \"plt.title(\\\"Estimated motion field with the Lukas-Kanade algorithm\\\")\\n\",\n \"\\n\",\n \"# Plot the last rainfall field in the \\\"training\\\" data.\\n\",\n \"# Remember to use the mm/h precipitation data since plot_precip_field assumes\\n\",\n \"# mm/h by default. You can change this behavior using the \\\"units\\\" keyword.\\n\",\n \"plot_precip_field(train_precip[-1], geodata=metadata, axis=\\\"off\\\")\\n\",\n \"\\n\",\n \"# Plot the motion field vectors\\n\",\n \"quiver(motion_field, geodata=metadata, step=40)\\n\",\n \"\\n\",\n \"plt.show()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"YObddRFCBnd1\"\n },\n \"source\": [\n \"### Extrapolate the observations\\n\",\n \"\\n\",\n \"We have all ingredients to make an extrapolation nowcast now. \\n\",\n \"The final step is to advect the most recent radar rainfall field along the estimated motion field, producing an extrapolation forecast.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"erSLAzvNBnd3\"\n },\n \"outputs\": [],\n \"source\": [\n \"from pysteps import nowcasts\\n\",\n \"\\n\",\n \"start = time.time()\\n\",\n \"\\n\",\n \"# Extrapolate the last radar observation\\n\",\n \"extrapolate = nowcasts.get_method(\\\"extrapolation\\\")\\n\",\n \"\\n\",\n \"# You can use the precipitation observations directly in mm/h for this step.\\n\",\n \"last_observation = train_precip[-1]\\n\",\n \"\\n\",\n \"last_observation[~np.isfinite(last_observation)] = metadata[\\\"zerovalue\\\"]\\n\",\n \"\\n\",\n \"# We set the number of leadtimes (the length of the forecast horizon) to the\\n\",\n \"# length of the observed/verification preipitation data. In this way, we'll get\\n\",\n \"# a forecast that covers these time intervals.\\n\",\n \"n_leadtimes = observed_precip.shape[0]\\n\",\n \"\\n\",\n \"# Advect the most recent radar rainfall field and make the nowcast.\\n\",\n \"precip_forecast = extrapolate(train_precip[-1], motion_field, n_leadtimes)\\n\",\n \"\\n\",\n \"# This shows the shape of the resulting array with [time intervals, rows, cols]\\n\",\n \"print(\\\"The shape of the resulting array is: \\\", precip_forecast.shape)\\n\",\n \"\\n\",\n \"end = time.time()\\n\",\n \"print(\\\"Advecting the radar rainfall fields took \\\", (end - start), \\\" seconds\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"csy5s-yRBneB\"\n },\n \"source\": [\n \"Let's inspect the last forecast time (hence this is the forecast rainfall an hour ahead).\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"MUiS5-HPBneD\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Plot precipitation at the end of the forecast period.\\n\",\n \"plt.figure(figsize=(9, 5), dpi=100)\\n\",\n \"plot_precip_field(precip_forecast[-1], geodata=metadata, axis=\\\"off\\\")\\n\",\n \"plt.show()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"colab_type\": \"text\",\n \"id\": \"mQEseXvhBneI\"\n },\n \"source\": [\n \"## Evaluate the forecast quality\\n\",\n \"\\n\",\n \"Many verification methods are already present in pysteps (see a complete list [here](https://pysteps.readthedocs.io/en/latest/pysteps_reference/verification.html)). We just have to import them. \\n\",\n \"\\n\",\n \"Here, we will evaluate our forecast using the Fractions Skill Score (FSS). \\n\",\n \"This metric provides an intuitive assessment of the dependency of forecast skill on spatial scale and intensity. This makes the FSS an ideal skill score for high-resolution precipitation forecasts.\\n\",\n \"\\n\",\n \"More precisely, the FSS is a neighborhood spatial verification method that directly compares the fractional coverage of events in windows surrounding the observations and forecasts.\\n\",\n \"The FSS varies from 0 (total mismatch) to 1 (perfect forecast).\\n\",\n \"For most situations, an FSS value of > 0.5 serves as a good indicator of a useful forecast ([Roberts and Lean, 2008](https://journals.ametsoc.org/doi/full/10.1175/2007MWR2123.1) and [Skok and Roberts, 2016](https://rmets.onlinelibrary.wiley.com/doi/full/10.1002/qj.2849)). \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"colab\": {},\n \"colab_type\": \"code\",\n \"id\": \"No3qBjqSBneK\"\n },\n \"outputs\": [],\n \"source\": [\n \"from pysteps import verification\\n\",\n \"\\n\",\n \"fss = verification.get_method(\\\"FSS\\\")\\n\",\n \"\\n\",\n \"# Compute fractions skill score (FSS) for all lead times for different scales using a 1 mm/h detection threshold.\\n\",\n \"scales = [\\n\",\n \" 2,\\n\",\n \" 4,\\n\",\n \" 8,\\n\",\n \" 16,\\n\",\n \" 32,\\n\",\n \" 64,\\n\",\n \"] # In grid points.\\n\",\n \"\\n\",\n \"scales_in_km = np.array(scales) * 4\\n\",\n \"\\n\",\n \"# Set the threshold\\n\",\n \"thr = 1.0 # in mm/h\\n\",\n \"\\n\",\n \"score = []\\n\",\n \"\\n\",\n \"# Calculate the FSS for every lead time and all predefined scales.\\n\",\n \"for i in range(n_leadtimes):\\n\",\n \" score_ = []\\n\",\n \" for scale in scales:\\n\",\n \" score_.append(\\n\",\n \" fss(precip_forecast[i, :, :], observed_precip[i, :, :], thr, scale)\\n\",\n \" )\\n\",\n \" score.append(score_)\\n\",\n \"\\n\",\n \"# Now plot it\\n\",\n \"plt.figure()\\n\",\n \"x = np.arange(1, n_leadtimes + 1) * timestep\\n\",\n \"plt.plot(x, score, lw=2.0)\\n\",\n \"plt.xlabel(\\\"Lead time [min]\\\")\\n\",\n \"plt.ylabel(\\\"FSS ( > 1.0 mm/h ) \\\")\\n\",\n \"plt.title(\\\"Fractions Skill Score\\\")\\n\",\n \"plt.legend(\\n\",\n \" scales_in_km,\\n\",\n \" title=\\\"Scale [km]\\\",\\n\",\n \" loc=\\\"center left\\\",\\n\",\n \" bbox_to_anchor=(1.01, 0.5),\\n\",\n \" bbox_transform=plt.gca().transAxes,\\n\",\n \")\\n\",\n \"plt.autoscale(axis=\\\"x\\\", tight=True)\\n\",\n \"plt.show()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"As you can see, the FSS decreases with increasing lead time.\\n\",\n \"This is expected, as the forecasting quality slowly decreases when we forecast further ahead.\\n\",\n \"Upscaling the forecast, however, clearly leads to higher skill (up to longer ahead) compared to the forecast on the highest resolutions.\\n\",\n \"\\n\",\n \"## Concluding remarks\\n\",\n \"Congratulations, you have successfully made your first nowcast using the pysteps library!\\n\",\n \"This was a simple extrapolation-based nowcast and a lot more advanced options are possible too, see [the pysteps examples gallery](https://pysteps.readthedocs.io/en/latest/auto_examples/index.html) for some nice examples.\"\n ]\n }\n ],\n \"metadata\": {\n \"colab\": {\n \"collapsed_sections\": [],\n \"name\": \"my_first_nowcast.ipynb\",\n \"provenance\": []\n },\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.8.2\"\n },\n \"pycharm\": {\n \"stem_cell\": {\n \"cell_type\": \"raw\",\n \"metadata\": {\n \"collapsed\": false\n },\n \"source\": []\n }\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}" }, { "path": "examples/optical_flow_methods_convergence.py", "content": "# coding: utf-8\n\n\"\"\"\nOptical flow methods convergence\n================================\n\nIn this example we test the convergence of the optical flow methods available in\npysteps using idealized motion fields.\n\nTo test the convergence, using an example precipitation field we will:\n\n- Read precipitation field from a file\n- Morph the precipitation field using a given motion field (linear or rotor) to\n generate a sequence of moving precipitation patterns.\n- Using the available optical flow methods, retrieve the motion field from the\n precipitation time sequence (synthetic precipitation observations).\n\nLet's first load the libraries that we will use.\n\"\"\"\n\nfrom datetime import datetime\nimport time\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom matplotlib.pyplot import get_cmap\nfrom scipy.ndimage import uniform_filter\n\nimport pysteps as stp\nfrom pysteps import motion, io, rcparams\nfrom pysteps.motion.vet import morph\nfrom pysteps.visualization import plot_precip_field, quiver\n\n################################################################################\n# Load the reference precipitation data\n# -------------------------------------\n#\n# First, we will import a radar composite from the archive.\n# You need the pysteps-data archive downloaded and the pystepsrc file\n# configured with the data_source paths pointing to data folders.\n\n\n# Selected case\ndate = datetime.strptime(\"201505151630\", \"%Y%m%d%H%M\")\ndata_source = rcparams.data_sources[\"mch\"]\n\n###############################################################################\n# Load the data from the archive\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nroot_path = data_source[\"root_path\"]\npath_fmt = data_source[\"path_fmt\"]\nfn_pattern = data_source[\"fn_pattern\"]\nfn_ext = data_source[\"fn_ext\"]\nimporter_name = data_source[\"importer\"]\nimporter_kwargs = data_source[\"importer_kwargs\"]\n\n# Find the reference field in the archive\nfns = io.archive.find_by_date(\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_prev_files=0\n)\n\n# Read the reference radar composite\nimporter = io.get_method(importer_name, \"importer\")\nreference_field, quality, metadata = io.read_timeseries(\n fns, importer, **importer_kwargs\n)\n\ndel quality # Not used\n\nreference_field = np.squeeze(reference_field) # Remove time dimension\n\n###############################################################################\n# Preprocess the data\n# ~~~~~~~~~~~~~~~~~~~\n\n# Convert to mm/h\nreference_field, metadata = stp.utils.to_rainrate(reference_field, metadata)\n\n# Mask invalid values\nreference_field = np.ma.masked_invalid(reference_field)\n\n# Plot the reference precipitation\nplot_precip_field(reference_field, title=\"Reference field\")\nplt.show()\n\n# Log-transform the data [dBR]\nreference_field, metadata = stp.utils.dB_transform(\n reference_field, metadata, threshold=0.1, zerovalue=-15.0\n)\n\nprint(\"Precip. pattern shape: \" + str(reference_field.shape))\n\n# This suppress nan conversion warnings in plot functions\nreference_field.data[reference_field.mask] = np.nan\n\n\n################################################################################\n# Synthetic precipitation observations\n# ------------------------------------\n#\n# Now we need to create a series of precipitation fields by applying the ideal\n# motion field to the reference precipitation field \"n\" times.\n#\n# To evaluate the accuracy of the computed_motion vectors, we will use\n# a relative RMSE measure.\n# Relative MSE = <(expected_motion - computed_motion)^2> / \n\n# Relative RMSE = Rel_RMSE = sqrt(Relative MSE)\n#\n# - Rel_RMSE = 0%: no error\n# - Rel_RMSE = 100%: The retrieved motion field has an average error equal in\n# magnitude to the motion field.\n#\n# Relative RMSE is computed over a region surrounding the precipitation\n# field, were there is enough information to retrieve the motion field.\n# The \"precipitation region\" includes the precipitation pattern plus a margin of\n# approximately 20 grid points.\n\n\n################################################################################\n# Let's create a function to construct different motion fields.\ndef create_motion_field(input_precip, motion_type):\n \"\"\"\n Create idealized motion fields to be applied to the reference image.\n\n Parameters\n ----------\n\n input_precip: numpy array (lat, lon)\n\n motion_type: str\n The supported motion fields are:\n\n - linear_x: (u=2, v=0)\n - linear_y: (u=0, v=2)\n - rotor: rotor field\n\n Returns\n -------\n ideal_motion : numpy array (u, v)\n \"\"\"\n\n # Create an imaginary grid on the image and create a motion field to be\n # applied to the image.\n ny, nx = input_precip.shape\n\n x_pos = np.arange(nx)\n y_pos = np.arange(ny)\n x, y = np.meshgrid(x_pos, y_pos, indexing=\"ij\")\n\n ideal_motion = np.zeros((2, nx, ny))\n\n if motion_type == \"linear_x\":\n ideal_motion[0, :] = 2 # Motion along x\n elif motion_type == \"linear_y\":\n ideal_motion[1, :] = 2 # Motion along y\n elif motion_type == \"rotor\":\n x_mean = x.mean()\n y_mean = y.mean()\n norm = np.sqrt(x * x + y * y)\n mask = norm != 0\n ideal_motion[0, mask] = 2 * (y - y_mean)[mask] / norm[mask]\n ideal_motion[1, mask] = -2 * (x - x_mean)[mask] / norm[mask]\n else:\n raise ValueError(\"motion_type not supported.\")\n\n # We need to swap the axes because the optical flow methods expect\n # (lat, lon) or (y,x) indexing convention.\n ideal_motion = ideal_motion.swapaxes(1, 2)\n return ideal_motion\n\n\n################################################################################\n# Let's create another function that construct the temporal series of\n# precipitation observations.\ndef create_observations(input_precip, motion_type, num_times=9):\n \"\"\"\n Create synthetic precipitation observations by displacing the input field\n using an ideal motion field.\n\n Parameters\n ----------\n\n input_precip: numpy array (lat, lon)\n Input precipitation field.\n\n motion_type: str\n The supported motion fields are:\n\n - linear_x: (u=2, v=0)\n - linear_y: (u=0, v=2)\n - rotor: rotor field\n\n num_times: int, optional\n Length of the observations sequence.\n\n\n Returns\n -------\n synthetic_observations: numpy array\n Sequence of observations\n \"\"\"\n\n ideal_motion = create_motion_field(input_precip, motion_type)\n\n # The morph function expects (lon, lat) or (x, y) dimensions.\n # Hence, we need to swap the lat,lon axes.\n\n # NOTE: The motion field passed to the morph function can't have any NaNs.\n # Otherwise, it can result in a segmentation fault.\n morphed_field, mask = morph(\n input_precip.swapaxes(0, 1), ideal_motion.swapaxes(1, 2)\n )\n\n mask = np.array(mask, dtype=bool)\n\n synthetic_observations = np.ma.MaskedArray(morphed_field, mask=mask)\n synthetic_observations = synthetic_observations[np.newaxis, :]\n\n for t in range(1, num_times):\n morphed_field, mask = morph(\n synthetic_observations[t - 1], ideal_motion.swapaxes(1, 2)\n )\n mask = np.array(mask, dtype=bool)\n\n morphed_field = np.ma.MaskedArray(\n morphed_field[np.newaxis, :], mask=mask[np.newaxis, :]\n )\n\n synthetic_observations = np.ma.concatenate(\n [synthetic_observations, morphed_field], axis=0\n )\n\n # Swap back to (lat, lon)\n synthetic_observations = synthetic_observations.swapaxes(1, 2)\n\n synthetic_observations = np.ma.masked_invalid(synthetic_observations)\n\n synthetic_observations.data[np.ma.getmaskarray(synthetic_observations)] = 0\n\n return ideal_motion, synthetic_observations\n\n\ndef plot_optflow_method_convergence(input_precip, optflow_method_name, motion_type):\n \"\"\"\n Test the convergence to the actual solution of the optical flow method used.\n\n Parameters\n ----------\n\n input_precip: numpy array (lat, lon)\n Input precipitation field.\n\n optflow_method_name: str\n Optical flow method name\n\n motion_type: str\n The supported motion fields are:\n\n - linear_x: (u=2, v=0)\n - linear_y: (u=0, v=2)\n - rotor: rotor field\n \"\"\"\n\n if optflow_method_name.lower() != \"darts\":\n num_times = 2\n else:\n num_times = 9\n\n ideal_motion, precip_obs = create_observations(\n input_precip, motion_type, num_times=num_times\n )\n\n oflow_method = motion.get_method(optflow_method_name)\n\n elapsed_time = time.perf_counter()\n\n computed_motion = oflow_method(precip_obs, verbose=False)\n\n print(\n f\"{optflow_method_name} computation time: \"\n f\"{(time.perf_counter() - elapsed_time):.1f} [s]\"\n )\n\n precip_obs, _ = stp.utils.dB_transform(precip_obs, inverse=True)\n\n precip_data = precip_obs.max(axis=0)\n precip_data.data[precip_data.mask] = 0\n\n precip_mask = (uniform_filter(precip_data, size=20) > 0.1) & ~precip_obs.mask.any(\n axis=0\n )\n\n cmap = get_cmap(\"jet\").copy()\n cmap.set_under(\"grey\", alpha=0.25)\n cmap.set_over(\"none\")\n\n # Compare retrieved motion field with the ideal one\n plt.figure(figsize=(9, 4))\n plt.subplot(1, 2, 1)\n ax = plot_precip_field(precip_obs[0], title=\"Reference motion\")\n quiver(ideal_motion, step=25, ax=ax)\n\n plt.subplot(1, 2, 2)\n ax = plot_precip_field(precip_obs[0], title=\"Retrieved motion\")\n quiver(computed_motion, step=25, ax=ax)\n\n # To evaluate the accuracy of the computed_motion vectors, we will use\n # a relative RMSE measure.\n # Relative MSE = < (expected_motion - computed_motion)^2 > / \n # Relative RMSE = sqrt(Relative MSE)\n\n mse = ((ideal_motion - computed_motion)[:, precip_mask] ** 2).mean()\n\n rel_mse = mse / (ideal_motion[:, precip_mask] ** 2).mean()\n plt.suptitle(\n f\"{optflow_method_name} \" f\"Relative RMSE: {np.sqrt(rel_mse) * 100:.2f}%\"\n )\n plt.show()\n\n\n################################################################################\n# Lucas-Kanade\n# ------------\n#\n# Constant motion x-direction\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"LucasKanade\", \"linear_x\")\n\n################################################################################\n# Constant motion y-direction\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"LucasKanade\", \"linear_y\")\n\n################################################################################\n# Rotational motion\n# ~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"LucasKanade\", \"rotor\")\n\n################################################################################\n# Variational Echo Tracking (VET)\n# -------------------------------\n#\n# Constant motion x-direction\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"VET\", \"linear_x\")\n\n################################################################################\n# Constant motion y-direction\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"VET\", \"linear_y\")\n\n################################################################################\n# Rotational motion\n# ~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"VET\", \"rotor\")\n\n################################################################################\n# DARTS\n# -----\n#\n# Constant motion x-direction\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"DARTS\", \"linear_x\")\n\n################################################################################\n# Constant motion y-direction\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"DARTS\", \"linear_y\")\n\n################################################################################\n# Rotational motion\n# ~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"DARTS\", \"rotor\")\n\n################################################################################\n# Farneback\n# ---------\n#\n# Constant motion x-direction\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"farneback\", \"linear_x\")\n\n################################################################################\n# Constant motion y-direction\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"farneback\", \"linear_y\")\n\n################################################################################\n# Rotational motion\n# ~~~~~~~~~~~~~~~~~\nplot_optflow_method_convergence(reference_field, \"farneback\", \"rotor\")\n\n# sphinx_gallery_thumbnail_number = 5\n" }, { "path": "examples/plot_cascade_decomposition.py", "content": "#!/bin/env python\n\"\"\"\nCascade decomposition\n=====================\n\nThis example script shows how to compute and plot the cascade decompositon of\na single radar precipitation field in pysteps.\n\n\"\"\"\n\nfrom matplotlib import cm, pyplot as plt\nimport numpy as np\nimport os\nfrom pprint import pprint\nfrom pysteps.cascade.bandpass_filters import filter_gaussian\nfrom pysteps import io, rcparams\nfrom pysteps.cascade.decomposition import decomposition_fft\nfrom pysteps.utils import conversion, transformation\nfrom pysteps.visualization import plot_precip_field\n\n###############################################################################\n# Read precipitation field\n# ------------------------\n#\n# First thing, the radar composite is imported and transformed in units\n# of dB.\n\n# Import the example radar composite\nroot_path = rcparams.data_sources[\"fmi\"][\"root_path\"]\nfilename = os.path.join(\n root_path, \"20160928\", \"201609281600_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\"\n)\nR, _, metadata = io.import_fmi_pgm(filename, gzipped=True)\n\n# Convert to rain rate\nR, metadata = conversion.to_rainrate(R, metadata)\n\n# Nicely print the metadata\npprint(metadata)\n\n# Plot the rainfall field\nplot_precip_field(R, geodata=metadata)\nplt.show()\n\n# Log-transform the data\nR, metadata = transformation.dB_transform(R, metadata, threshold=0.1, zerovalue=-15.0)\n\n###############################################################################\n# 2D Fourier spectrum\n# --------------------\n#\n# Compute and plot the 2D Fourier power spectrum of the precipitaton field.\n\n# Set Nans as the fill value\nR[~np.isfinite(R)] = metadata[\"zerovalue\"]\n\n# Compute the Fourier transform of the input field\nF = abs(np.fft.fftshift(np.fft.fft2(R)))\n\n# Plot the power spectrum\nM, N = F.shape\nfig, ax = plt.subplots()\nim = ax.imshow(\n np.log(F**2), vmin=4, vmax=24, cmap=cm.jet, extent=(-N / 2, N / 2, -M / 2, M / 2)\n)\ncb = fig.colorbar(im)\nax.set_xlabel(\"Wavenumber $k_x$\")\nax.set_ylabel(\"Wavenumber $k_y$\")\nax.set_title(\"Log-power spectrum of R\")\nplt.show()\n\n###############################################################################\n# Cascade decomposition\n# ---------------------\n#\n# First, construct a set of Gaussian bandpass filters and plot the corresponding\n# 1D filters.\n\nnum_cascade_levels = 7\n\n# Construct the Gaussian bandpass filters\nfilter = filter_gaussian(R.shape, num_cascade_levels)\n\n# Plot the bandpass filter weights\nL = max(N, M)\nfig, ax = plt.subplots()\nfor k in range(num_cascade_levels):\n ax.semilogx(\n np.linspace(0, L / 2, len(filter[\"weights_1d\"][k, :])),\n filter[\"weights_1d\"][k, :],\n \"k-\",\n base=pow(0.5 * L / 3, 1.0 / (num_cascade_levels - 2)),\n )\nax.set_xlim(1, L / 2)\nax.set_ylim(0, 1)\nxt = np.hstack([[1.0], filter[\"central_wavenumbers\"][1:]])\nax.set_xticks(xt)\nax.set_xticklabels([\"%.2f\" % cf for cf in filter[\"central_wavenumbers\"]])\nax.set_xlabel(\"Radial wavenumber $|\\mathbf{k}|$\")\nax.set_ylabel(\"Normalized weight\")\nax.set_title(\"Bandpass filter weights\")\nplt.show()\n\n###############################################################################\n# Finally, apply the 2D Gaussian filters to decompose the radar rainfall field\n# into a set of cascade levels of decreasing spatial scale and plot them.\n\ndecomp = decomposition_fft(R, filter, compute_stats=True)\n\n# Plot the normalized cascade levels\nfor i in range(num_cascade_levels):\n mu = decomp[\"means\"][i]\n sigma = decomp[\"stds\"][i]\n decomp[\"cascade_levels\"][i] = (decomp[\"cascade_levels\"][i] - mu) / sigma\n\nfig, ax = plt.subplots(nrows=2, ncols=4)\n\nax[0, 0].imshow(R, cmap=cm.RdBu_r, vmin=-5, vmax=5)\nax[0, 1].imshow(decomp[\"cascade_levels\"][0], cmap=cm.RdBu_r, vmin=-3, vmax=3)\nax[0, 2].imshow(decomp[\"cascade_levels\"][1], cmap=cm.RdBu_r, vmin=-3, vmax=3)\nax[0, 3].imshow(decomp[\"cascade_levels\"][2], cmap=cm.RdBu_r, vmin=-3, vmax=3)\nax[1, 0].imshow(decomp[\"cascade_levels\"][3], cmap=cm.RdBu_r, vmin=-3, vmax=3)\nax[1, 1].imshow(decomp[\"cascade_levels\"][4], cmap=cm.RdBu_r, vmin=-3, vmax=3)\nax[1, 2].imshow(decomp[\"cascade_levels\"][5], cmap=cm.RdBu_r, vmin=-3, vmax=3)\nax[1, 3].imshow(decomp[\"cascade_levels\"][6], cmap=cm.RdBu_r, vmin=-3, vmax=3)\n\nax[0, 0].set_title(\"Observed\")\nax[0, 1].set_title(\"Level 1\")\nax[0, 2].set_title(\"Level 2\")\nax[0, 3].set_title(\"Level 3\")\nax[1, 0].set_title(\"Level 4\")\nax[1, 1].set_title(\"Level 5\")\nax[1, 2].set_title(\"Level 6\")\nax[1, 3].set_title(\"Level 7\")\n\nfor i in range(2):\n for j in range(4):\n ax[i, j].set_xticks([])\n ax[i, j].set_yticks([])\nplt.tight_layout()\nplt.show()\n\n# sphinx_gallery_thumbnail_number = 4\n" }, { "path": "examples/plot_custom_precipitation_range.py", "content": "#!/bin/env python\n\"\"\"\nPlot precipitation using custom colormap\n=============\n\nThis tutorial shows how to plot data using a custom colormap with a specific\nrange of precipitation values.\n\n\"\"\"\n\nimport os\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\n\nimport pysteps\nfrom pysteps import io, rcparams\nfrom pysteps.utils import conversion\nfrom pysteps.visualization import plot_precip_field\nfrom pysteps.datasets import download_pysteps_data, create_default_pystepsrc\n\n###############################################################################\n# Download the data if it is not available\n# ----------------------------------------\n#\n# The following code block downloads datasets from the pysteps-data repository\n# if it is not available on the disk. The dataset is used to demonstrate the\n# plotting of precipitation data using a custom colormap.\n\n# Check if the pysteps-data repository is available (it would be pysteps-data in pysteps)\n# Implies that you are running this script from the `pysteps/examples` folder\n\nif not os.path.exists(rcparams.data_sources[\"mrms\"][\"root_path\"]):\n download_pysteps_data(\"pysteps_data\")\n config_file_path = create_default_pystepsrc(\"pysteps_data\")\n print(f\"Configuration file has been created at {config_file_path}\")\n\n\n###############################################################################\n# Read precipitation field\n# ------------------------\n#\n# First thing, load a frame from Multi-Radar Multi-Sensor dataset and convert it\n# to precipitation rate in mm/h.\n\n# Define the dataset and the date for which you want to load data\ndata_source = pysteps.rcparams.data_sources[\"mrms\"]\ndate = datetime(2019, 6, 10, 0, 2, 0) # Example date\n\n# Extract the parameters from the data source\nroot_path = data_source[\"root_path\"]\npath_fmt = data_source[\"path_fmt\"]\nfn_pattern = data_source[\"fn_pattern\"]\nfn_ext = data_source[\"fn_ext\"]\nimporter_name = data_source[\"importer\"]\nimporter_kwargs = data_source[\"importer_kwargs\"]\ntimestep = data_source[\"timestep\"]\n\n# Find the frame in the archive for the specified date\nfns = io.find_by_date(\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_prev_files=1\n)\n\n# Read the frame from the archive\nimporter = io.get_method(importer_name, \"importer\")\nR, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\n\n# Convert the reflectivity data to rain rate\nR, metadata = conversion.to_rainrate(R, metadata)\n\n# Plot the first rainfall field from the loaded data\nplt.figure(figsize=(10, 5), dpi=300)\nplt.axis(\"off\")\nplot_precip_field(R[0, :, :], geodata=metadata, axis=\"off\")\n\nplt.tight_layout()\nplt.show()\n\n###############################################################################\n# Define the custom colormap\n# --------------------------\n#\n# Assume that the default colormap does not represent the precipitation values\n# in the desired range. In this case, you can define a custom colormap that will\n# be used to plot the precipitation data and pass the class instance to the\n# `plot_precip_field` function.\n#\n# It essential for the custom colormap to have the following attributes:\n#\n# - `cmap`: The colormap object.\n# - `norm`: The normalization object.\n# - `clevs`: The color levels for the colormap.\n#\n# `plot_precip_field` can handle each of the classes defined in the `matplotlib.colors`\n# https://matplotlib.org/stable/api/colors_api.html#colormaps\n# There must be as many colors in the colormap as there are levels in the color levels.\n\n\n# Define the custom colormap\n\nfrom matplotlib import colors\n\n\nclass ColormapConfig:\n def __init__(self):\n self.cmap = None\n self.norm = None\n self.clevs = None\n\n self.build_colormap()\n\n def build_colormap(self):\n # Define the colormap boundaries and colors\n # color_list = ['lightgrey', 'lightskyblue', 'blue', 'yellow', 'orange', 'red', 'darkred']\n color_list = [\"blue\", \"navy\", \"yellow\", \"orange\", \"green\", \"brown\", \"red\"]\n\n self.clevs = [0.1, 0.5, 1.5, 2.5, 4, 6, 10] # mm/hr\n\n # Create a ListedColormap object with the defined colors\n self.cmap = colors.ListedColormap(color_list)\n self.cmap.name = \"Custom Colormap\"\n\n # Set the color for values above the maximum level\n self.cmap.set_over(\"darkmagenta\")\n # Set the color for values below the minimum level\n self.cmap.set_under(\"none\")\n # Set the color for missing values\n self.cmap.set_bad(\"gray\", alpha=0.5)\n\n # Create a BoundaryNorm object to normalize the data values to the colormap boundaries\n self.norm = colors.BoundaryNorm(self.clevs, self.cmap.N)\n\n\n# Create an instance of the ColormapConfig class\nconfig = ColormapConfig()\n\n# Plot the precipitation field using the custom colormap\nplt.figure(figsize=(10, 5), dpi=300)\nplt.axis(\"off\")\nplot_precip_field(R[0, :, :], geodata=metadata, axis=\"off\", colormap_config=config)\n\nplt.tight_layout()\nplt.show()\n" }, { "path": "examples/plot_ensemble_verification.py", "content": "#!/bin/env python\n\"\"\"\nEnsemble verification\n=====================\n\nIn this tutorial we perform a verification of a probabilistic extrapolation nowcast\nusing MeteoSwiss radar data.\n\n\"\"\"\n\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom pprint import pprint\nfrom pysteps import io, nowcasts, rcparams, verification\nfrom pysteps.motion.lucaskanade import dense_lucaskanade\nfrom pysteps.postprocessing import ensemblestats\nfrom pysteps.utils import conversion, dimension, transformation\nfrom pysteps.visualization import plot_precip_field\n\n###############################################################################\n# Read precipitation field\n# ------------------------\n#\n# First, we will import the sequence of MeteoSwiss (\"mch\") radar composites.\n# You need the pysteps-data archive downloaded and the pystepsrc file\n# configured with the data_source paths pointing to data folders.\n\n# Selected case\ndate = datetime.strptime(\"201607112100\", \"%Y%m%d%H%M\")\ndata_source = rcparams.data_sources[\"mch\"]\nn_ens_members = 20\nn_leadtimes = 6\nseed = 24\n\n###############################################################################\n# Load the data from the archive\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n#\n# The data are upscaled to 2 km resolution to limit the memory usage and thus\n# be able to afford a larger number of ensemble members.\n\nroot_path = data_source[\"root_path\"]\npath_fmt = data_source[\"path_fmt\"]\nfn_pattern = data_source[\"fn_pattern\"]\nfn_ext = data_source[\"fn_ext\"]\nimporter_name = data_source[\"importer\"]\nimporter_kwargs = data_source[\"importer_kwargs\"]\ntimestep = data_source[\"timestep\"]\n\n# Find the radar files in the archive\nfns = io.find_by_date(\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_prev_files=2\n)\n\n# Read the data from the archive\nimporter = io.get_method(importer_name, \"importer\")\nR, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\n\n# Convert to rain rate\nR, metadata = conversion.to_rainrate(R, metadata)\n\n# Upscale data to 2 km\nR, metadata = dimension.aggregate_fields_space(R, metadata, 2000)\n\n# Plot the rainfall field\nplot_precip_field(R[-1, :, :], geodata=metadata)\nplt.show()\n\n# Log-transform the data to unit of dBR, set the threshold to 0.1 mm/h,\n# set the fill value to -15 dBR\nR, metadata = transformation.dB_transform(R, metadata, threshold=0.1, zerovalue=-15.0)\n\n# Set missing values with the fill value\nR[~np.isfinite(R)] = -15.0\n\n# Nicely print the metadata\npprint(metadata)\n\n###############################################################################\n# Forecast\n# --------\n#\n# We use the STEPS approach to produce a ensemble nowcast of precipitation fields.\n\n# Estimate the motion field\nV = dense_lucaskanade(R)\n\n# Perform the ensemble nowcast with STEPS\nnowcast_method = nowcasts.get_method(\"steps\")\nR_f = nowcast_method(\n R[-3:, :, :],\n V,\n n_leadtimes,\n n_ens_members,\n n_cascade_levels=6,\n precip_thr=-10.0,\n kmperpixel=2.0,\n timestep=timestep,\n decomp_method=\"fft\",\n bandpass_filter_method=\"gaussian\",\n noise_method=\"nonparametric\",\n vel_pert_method=\"bps\",\n mask_method=\"incremental\",\n seed=seed,\n)\n\n# Back-transform to rain rates\nR_f = transformation.dB_transform(R_f, threshold=-10.0, inverse=True)[0]\n\n# Plot some of the realizations\nfig = plt.figure()\nfor i in range(4):\n ax = fig.add_subplot(221 + i)\n ax.set_title(\"Member %02d\" % i)\n plot_precip_field(R_f[i, -1, :, :], geodata=metadata, colorbar=False, axis=\"off\")\nplt.tight_layout()\nplt.show()\n\n###############################################################################\n# Verification\n# ------------\n#\n# Pysteps includes a number of verification metrics to help users to analyze\n# the general characteristics of the nowcasts in terms of consistency and\n# quality (or goodness).\n# Here, we will verify our probabilistic forecasts using the ROC curve,\n# reliability diagrams, and rank histograms, as implemented in the verification\n# module of pysteps.\n\n# Find the files containing the verifying observations\nfns = io.archive.find_by_date(\n date,\n root_path,\n path_fmt,\n fn_pattern,\n fn_ext,\n timestep,\n 0,\n num_next_files=n_leadtimes,\n)\n\n# Read the observations\nR_o, _, metadata_o = io.read_timeseries(fns, importer, **importer_kwargs)\n\n# Convert to mm/h\nR_o, metadata_o = conversion.to_rainrate(R_o, metadata_o)\n\n# Upscale data to 2 km\nR_o, metadata_o = dimension.aggregate_fields_space(R_o, metadata_o, 2000)\n\n# Compute the verification for the last lead time\n\n# compute the exceedance probability of 0.1 mm/h from the ensemble\nP_f = ensemblestats.excprob(R_f[:, -1, :, :], 0.1, ignore_nan=True)\n\n###############################################################################\n# ROC curve\n# ~~~~~~~~~\n\nroc = verification.ROC_curve_init(0.1, n_prob_thrs=10)\nverification.ROC_curve_accum(roc, P_f, R_o[-1, :, :])\nfig, ax = plt.subplots()\nverification.plot_ROC(roc, ax, opt_prob_thr=True)\nax.set_title(\"ROC curve (+%i min)\" % (n_leadtimes * timestep))\nplt.show()\n\n###############################################################################\n# Reliability diagram\n# ~~~~~~~~~~~~~~~~~~~\n\nreldiag = verification.reldiag_init(0.1)\nverification.reldiag_accum(reldiag, P_f, R_o[-1, :, :])\nfig, ax = plt.subplots()\nverification.plot_reldiag(reldiag, ax)\nax.set_title(\"Reliability diagram (+%i min)\" % (n_leadtimes * timestep))\nplt.show()\n\n###############################################################################\n# Rank histogram\n# ~~~~~~~~~~~~~~\n\nrankhist = verification.rankhist_init(R_f.shape[0], 0.1)\nverification.rankhist_accum(rankhist, R_f[:, -1, :, :], R_o[-1, :, :])\nfig, ax = plt.subplots()\nverification.plot_rankhist(rankhist, ax)\nax.set_title(\"Rank histogram (+%i min)\" % (n_leadtimes * timestep))\nplt.show()\n\n# sphinx_gallery_thumbnail_number = 5\n" }, { "path": "examples/plot_extrapolation_nowcast.py", "content": "#!/bin/env python\n\"\"\"\nExtrapolation nowcast\n=====================\n\nThis tutorial shows how to compute and plot an extrapolation nowcast using\nFinnish radar data.\n\n\"\"\"\n\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom pprint import pprint\nfrom pysteps import io, motion, nowcasts, rcparams, verification\nfrom pysteps.utils import conversion, transformation\nfrom pysteps.visualization import plot_precip_field, quiver\n\n###############################################################################\n# Read the radar input images\n# ---------------------------\n#\n# First, we will import the sequence of radar composites.\n# You need the pysteps-data archive downloaded and the pystepsrc file\n# configured with the data_source paths pointing to data folders.\n\n# Selected case\ndate = datetime.strptime(\"201609281600\", \"%Y%m%d%H%M\")\ndata_source = rcparams.data_sources[\"fmi\"]\nn_leadtimes = 12\n\n###############################################################################\n# Load the data from the archive\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nroot_path = data_source[\"root_path\"]\npath_fmt = data_source[\"path_fmt\"]\nfn_pattern = data_source[\"fn_pattern\"]\nfn_ext = data_source[\"fn_ext\"]\nimporter_name = data_source[\"importer\"]\nimporter_kwargs = data_source[\"importer_kwargs\"]\ntimestep = data_source[\"timestep\"]\n\n# Find the input files from the archive\nfns = io.archive.find_by_date(\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_prev_files=2\n)\n\n# Read the radar composites\nimporter = io.get_method(importer_name, \"importer\")\nZ, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\n\n# Convert to rain rate\nR, metadata = conversion.to_rainrate(Z, metadata)\n\n# Plot the rainfall field\nplot_precip_field(R[-1, :, :], geodata=metadata)\nplt.show()\n\n# Store the last frame for plotting it later later\nR_ = R[-1, :, :].copy()\n\n# Log-transform the data to unit of dBR, set the threshold to 0.1 mm/h,\n# set the fill value to -15 dBR\nR, metadata = transformation.dB_transform(R, metadata, threshold=0.1, zerovalue=-15.0)\n\n# Nicely print the metadata\npprint(metadata)\n\n###############################################################################\n# Compute the nowcast\n# -------------------\n#\n# The extrapolation nowcast is based on the estimation of the motion field,\n# which is here performed using a local tracking approach (Lucas-Kanade).\n# The most recent radar rainfall field is then simply advected along this motion\n# field in oder to produce an extrapolation forecast.\n\n# Estimate the motion field with Lucas-Kanade\noflow_method = motion.get_method(\"LK\")\nV = oflow_method(R[-3:, :, :])\n\n# Extrapolate the last radar observation\nextrapolate = nowcasts.get_method(\"extrapolation\")\nR[~np.isfinite(R)] = metadata[\"zerovalue\"]\nR_f = extrapolate(R[-1, :, :], V, n_leadtimes)\n\n# Back-transform to rain rate\nR_f = transformation.dB_transform(R_f, threshold=-10.0, inverse=True)[0]\n\n# Plot the motion field\nplot_precip_field(R_, geodata=metadata)\nquiver(V, geodata=metadata, step=50)\nplt.show()\n\n###############################################################################\n# Verify with FSS\n# ---------------\n#\n# The fractions skill score (FSS) provides an intuitive assessment of the\n# dependency of skill on spatial scale and intensity, which makes it an ideal\n# skill score for high-resolution precipitation forecasts.\n\n# Find observations in the data archive\nfns = io.archive.find_by_date(\n date,\n root_path,\n path_fmt,\n fn_pattern,\n fn_ext,\n timestep,\n num_prev_files=0,\n num_next_files=n_leadtimes,\n)\n# Read the radar composites\nR_o, _, metadata_o = io.read_timeseries(fns, importer, **importer_kwargs)\nR_o, metadata_o = conversion.to_rainrate(R_o, metadata_o, 223.0, 1.53)\n\n# Compute fractions skill score (FSS) for all lead times, a set of scales and 1 mm/h\nfss = verification.get_method(\"FSS\")\nscales = [2, 4, 8, 16, 32, 64, 128, 256, 512]\nthr = 1.0\nscore = []\nfor i in range(n_leadtimes):\n score_ = []\n for scale in scales:\n score_.append(fss(R_f[i, :, :], R_o[i + 1, :, :], thr, scale))\n score.append(score_)\n\nplt.figure()\nx = np.arange(1, n_leadtimes + 1) * timestep\nplt.plot(x, score)\nplt.legend(scales, title=\"Scale [km]\")\nplt.xlabel(\"Lead time [min]\")\nplt.ylabel(\"FSS ( > 1.0 mm/h ) \")\nplt.title(\"Fractions skill score\")\nplt.show()\n\n# sphinx_gallery_thumbnail_number = 3\n" }, { "path": "examples/plot_linear_blending.py", "content": "# -*- coding: utf-8 -*-\n\n\"\"\"\nLinear blending\n===============\n\nThis tutorial shows how to construct a simple linear blending between a STEPS\nensemble nowcast and a Numerical Weather Prediction (NWP) rainfall forecast. The\nused datasets are from the Bureau of Meteorology, Australia.\n\"\"\"\n\nimport os\nfrom datetime import datetime\n\nfrom matplotlib import pyplot as plt\n\nimport pysteps\nfrom pysteps import io, rcparams, nowcasts, blending\nfrom pysteps.utils import conversion\nfrom pysteps.visualization import plot_precip_field\n\n################################################################################\n# Read the radar images and the NWP forecast\n# ------------------------------------------\n#\n# First, we import a sequence of 3 images of 10-minute radar composites\n# and the corresponding NWP rainfall forecast that was available at that time.\n#\n# You need the pysteps-data archive downloaded and the pystepsrc file\n# configured with the data_source paths pointing to data folders.\n# Additionally, the pysteps-nwp-importers plugin needs to be installed, see\n# https://github.com/pySTEPS/pysteps-nwp-importers.\n\n# Selected case\ndate_radar = datetime.strptime(\"202010310400\", \"%Y%m%d%H%M\")\n# The last NWP forecast was issued at 00:00\ndate_nwp = datetime.strptime(\"202010310000\", \"%Y%m%d%H%M\")\nradar_data_source = rcparams.data_sources[\"bom\"]\nnwp_data_source = rcparams.data_sources[\"bom_nwp\"]\n\n###############################################################################\n# Load the data from the archive\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nroot_path = radar_data_source[\"root_path\"]\npath_fmt = \"prcp-c10/66/%Y/%m/%d\"\nfn_pattern = \"66_%Y%m%d_%H%M00.prcp-c10\"\nfn_ext = radar_data_source[\"fn_ext\"]\nimporter_name = radar_data_source[\"importer\"]\nimporter_kwargs = radar_data_source[\"importer_kwargs\"]\ntimestep = 10.0\n\n# Find the radar files in the archive\nfns = io.find_by_date(\n date_radar, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_prev_files=2\n)\n\n# Read the radar composites\nimporter = io.get_method(importer_name, \"importer\")\nradar_precip, _, radar_metadata = io.read_timeseries(fns, importer, **importer_kwargs)\n\n# Import the NWP data\nfilename = os.path.join(\n nwp_data_source[\"root_path\"],\n datetime.strftime(date_nwp, nwp_data_source[\"path_fmt\"]),\n datetime.strftime(date_nwp, nwp_data_source[\"fn_pattern\"])\n + \".\"\n + nwp_data_source[\"fn_ext\"],\n)\n\nnwp_importer = io.get_method(\"bom_nwp\", \"importer\")\nnwp_precip, _, nwp_metadata = nwp_importer(filename)\n\n# Only keep the NWP forecasts from the last radar observation time (2020-10-31 04:00)\n# End of the forecast is 18 time steps (+3 hours) in advance.\nprecip_nwp = nwp_precip[24:43, :, :]\n\n\n################################################################################\n# Pre-processing steps\n# --------------------\n\n# Make sure the units are in mm/h\nconverter = pysteps.utils.get_method(\"mm/h\")\nradar_precip, radar_metadata = converter(radar_precip, radar_metadata)\nprecip_nwp, nwp_metadata = converter(precip_nwp, nwp_metadata)\n\n# Threshold the data\nradar_precip[radar_precip < 0.1] = 0.0\nprecip_nwp[precip_nwp < 0.1] = 0.0\n\n# Plot the radar rainfall field and the first time step of the NWP forecast.\n# For the initial time step (t=0), the NWP rainfall forecast is not that different\n# from the observed radar rainfall, but it misses some of the locations and\n# shapes of the observed rainfall fields. Therefore, the NWP rainfall forecast will\n# initially get a low weight in the blending process.\ndate_str = datetime.strftime(date_radar, \"%Y-%m-%d %H:%M\")\nplt.figure(figsize=(10, 5))\nplt.subplot(121)\nplot_precip_field(\n radar_precip[-1, :, :],\n geodata=radar_metadata,\n title=f\"Radar observation at {date_str}\",\n)\nplt.subplot(122)\nplot_precip_field(\n precip_nwp[0, :, :], geodata=nwp_metadata, title=f\"NWP forecast at {date_str}\"\n)\nplt.tight_layout()\nplt.show()\n\n# Only keep the NWP forecasts from 2020-10-31 04:05 onwards, because the first\n# forecast lead time starts at 04:05.\nprecip_nwp = precip_nwp[1:]\n\n# Transform the radar data to dB - this transformation is useful for the motion\n# field estimation and the subsequent nowcasts. The NWP forecast is not\n# transformed, because the linear blending code sets everything back in mm/h\n# after the nowcast.\ntransformer = pysteps.utils.get_method(\"dB\")\nradar_precip, radar_metadata = transformer(radar_precip, radar_metadata, threshold=0.1)\n\n\n################################################################################\n# Determine the velocity field for the radar rainfall nowcast\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\noflow_method = pysteps.motion.get_method(\"lucaskanade\")\nvelocity_radar = oflow_method(radar_precip)\n\n\n################################################################################\n# The linear blending of nowcast and NWP rainfall forecast\n# --------------------------------------------------------\n\n# Calculate the blended precipitation field\nprecip_blended = blending.linear_blending.forecast(\n precip=radar_precip[-1, :, :],\n precip_metadata=radar_metadata,\n velocity=velocity_radar,\n timesteps=18,\n timestep=10,\n nowcast_method=\"extrapolation\", # simple advection nowcast\n precip_nwp=precip_nwp,\n precip_nwp_metadata=nwp_metadata,\n start_blending=60, # in minutes (this is an arbritrary choice)\n end_blending=120, # in minutes (this is an arbritrary choice)\n)\n\n\n################################################################################\n# The salient blending of nowcast and NWP rainfall forecast\n# ---------------------------------------------------------\n#\n# This method follows the saliency-based blending procedure described in :cite:`Hwang2015`. The\n# blending is based on intensities and forecast times. The blended product preserves pixel\n# intensities with time if they are strong enough based on their ranked salience. Saliency is\n# the property of an object to be outstanding with respect to its surroundings. The ranked salience\n# is calculated by first determining the difference in the normalized intensity of the nowcasts\n# and NWP. Next, the pixel intensities are ranked, in which equally comparable values receive\n# the same ranking number.\n\n# Calculate the salient blended precipitation field\nprecip_salient_blended = blending.linear_blending.forecast(\n precip=radar_precip[-1, :, :],\n precip_metadata=radar_metadata,\n velocity=velocity_radar,\n timesteps=18,\n timestep=10,\n nowcast_method=\"extrapolation\", # simple advection nowcast\n precip_nwp=precip_nwp,\n precip_nwp_metadata=nwp_metadata,\n start_blending=60, # in minutes (this is an arbritrary choice)\n end_blending=120, # in minutes (this is an arbritrary choice)\n saliency=True,\n)\n\n\n################################################################################\n# Visualize the output\n# --------------------\n\n################################################################################\n# Calculate the radar rainfall nowcasts for visualization\n\nnowcast_method_func = nowcasts.get_method(\"extrapolation\")\nprecip_nowcast = nowcast_method_func(\n precip=radar_precip[-1, :, :],\n velocity=velocity_radar,\n timesteps=18,\n)\n\n# Make sure that precip_nowcast are in mm/h\nprecip_nowcast, _ = conversion.to_rainrate(precip_nowcast, metadata=radar_metadata)\n\n################################################################################\n# The linear blending starts at 60 min, so during the first 60 minutes the\n# blended forecast only consists of the extrapolation forecast (consisting of an\n# extrapolation nowcast). Between 60 and 120 min, the NWP forecast gradually gets more\n# weight, whereas the extrapolation forecasts gradually gets less weight. In addition,\n# the saliency-based blending takes also the difference in pixel intensities into account,\n# which are preserved over time if they are strong enough based on their ranked salience.\n# Furthermore, pixels with relative low intensities get a lower weight and stay smaller in\n# the saliency-based blending compared to linear blending. After 120 min, the blended\n# forecast entirely consists of the NWP rainfall forecast.\n\nfig = plt.figure(figsize=(8, 12))\n\nleadtimes_min = [30, 60, 80, 100, 120]\nn_leadtimes = len(leadtimes_min)\nfor n, leadtime in enumerate(leadtimes_min):\n # Extrapolation\n plt.subplot(n_leadtimes, 4, n * 4 + 1)\n plot_precip_field(\n precip_nowcast[int(leadtime / timestep) - 1, :, :],\n geodata=radar_metadata,\n title=f\"Nowcast + {leadtime} min\",\n axis=\"off\",\n colorbar=False,\n )\n\n # Nowcast with blending into NWP\n plt.subplot(n_leadtimes, 4, n * 4 + 2)\n plot_precip_field(\n precip_blended[int(leadtime / timestep) - 1, :, :],\n geodata=radar_metadata,\n title=f\"Linear + {leadtime} min\",\n axis=\"off\",\n colorbar=False,\n )\n\n # Nowcast with salient blending into NWP\n plt.subplot(n_leadtimes, 4, n * 4 + 3)\n plot_precip_field(\n precip_salient_blended[int(leadtime / timestep) - 1, :, :],\n geodata=radar_metadata,\n title=f\"Salient + {leadtime} min\",\n axis=\"off\",\n colorbar=False,\n )\n\n # Raw NWP forecast\n plt.subplot(n_leadtimes, 4, n * 4 + 4)\n plot_precip_field(\n precip_nwp[int(leadtime / timestep) - 1, :, :],\n geodata=nwp_metadata,\n title=f\"NWP + {leadtime} min\",\n axis=\"off\",\n colorbar=False,\n )\n\nplt.tight_layout()\nplt.show()\n\n################################################################################\n# Note that the NaN values of the extrapolation forecast are replaced with NWP data\n# in the blended forecast, even before the blending starts.\n" }, { "path": "examples/plot_noise_generators.py", "content": "#!/bin/env python\r\n\"\"\"\r\nGeneration of stochastic noise\r\n==============================\r\n\r\nThis example script shows how to run the stochastic noise field generators\r\nincluded in pysteps.\r\n\r\nThese noise fields are used as perturbation terms during an extrapolation\r\nnowcast in order to represent the uncertainty in the evolution of the rainfall\r\nfield.\r\n\"\"\"\r\n\r\nfrom matplotlib import cm, pyplot as plt\r\nimport numpy as np\r\nimport os\r\nfrom pprint import pprint\r\nfrom pysteps import io, rcparams\r\nfrom pysteps.noise.fftgenerators import initialize_param_2d_fft_filter\r\nfrom pysteps.noise.fftgenerators import initialize_nonparam_2d_fft_filter\r\nfrom pysteps.noise.fftgenerators import generate_noise_2d_fft_filter\r\nfrom pysteps.utils import conversion, rapsd, transformation\r\nfrom pysteps.visualization import plot_precip_field, plot_spectrum1d\r\n\r\n###############################################################################\r\n# Read precipitation field\r\n# ------------------------\r\n#\r\n# First thing, the radar composite is imported and transformed in units\r\n# of dB.\r\n# This image will be used to train the Fourier filters that are necessary to\r\n# produce the fields of spatially correlated noise.\r\n\r\n# Import the example radar composite\r\nroot_path = rcparams.data_sources[\"mch\"][\"root_path\"]\r\nfilename = os.path.join(root_path, \"20160711\", \"AQC161932100V_00005.801.gif\")\r\nR, _, metadata = io.import_mch_gif(filename, product=\"AQC\", unit=\"mm\", accutime=5.0)\r\n\r\n# Convert to mm/h\r\nR, metadata = conversion.to_rainrate(R, metadata)\r\n\r\n# Nicely print the metadata\r\npprint(metadata)\r\n\r\n# Plot the rainfall field\r\nplot_precip_field(R, geodata=metadata)\r\nplt.show()\r\n\r\n# Log-transform the data\r\nR, metadata = transformation.dB_transform(R, metadata, threshold=0.1, zerovalue=-15.0)\r\n\r\n# Assign the fill value to all the Nans\r\nR[~np.isfinite(R)] = metadata[\"zerovalue\"]\r\n\r\n###############################################################################\r\n# Parametric filter\r\n# -----------------\r\n#\r\n# In the parametric approach, a power-law model is used to approximate the power\r\n# spectral density (PSD) of a given rainfall field.\r\n#\r\n# The parametric model uses a piece-wise linear function with two spectral\r\n# slopes (beta1 and beta2) and one breaking point\r\n\r\n# Fit the parametric PSD to the observation\r\nFp = initialize_param_2d_fft_filter(R)\r\n\r\n# Compute the observed and fitted 1D PSD\r\nL = np.max(Fp[\"input_shape\"])\r\nif L % 2 == 1:\r\n wn = np.arange(0, int(L / 2) + 1)\r\nelse:\r\n wn = np.arange(0, int(L / 2))\r\nR_, freq = rapsd(R, fft_method=np.fft, return_freq=True)\r\nf = np.exp(Fp[\"model\"](np.log(wn), *Fp[\"pars\"]))\r\n\r\n# Extract the scaling break in km, beta1 and beta2\r\nw0 = L / np.exp(Fp[\"pars\"][0])\r\nb1 = Fp[\"pars\"][2]\r\nb2 = Fp[\"pars\"][3]\r\n\r\n# Plot the observed power spectrum and the model\r\nfig, ax = plt.subplots()\r\nplot_scales = [512, 256, 128, 64, 32, 16, 8, 4]\r\nplot_spectrum1d(\r\n freq,\r\n R_,\r\n x_units=\"km\",\r\n y_units=\"dBR\",\r\n color=\"k\",\r\n ax=ax,\r\n label=\"Observed\",\r\n wavelength_ticks=plot_scales,\r\n)\r\nplot_spectrum1d(\r\n freq,\r\n f,\r\n x_units=\"km\",\r\n y_units=\"dBR\",\r\n color=\"r\",\r\n ax=ax,\r\n label=\"Fit\",\r\n wavelength_ticks=plot_scales,\r\n)\r\nplt.legend()\r\nax.set_title(\r\n \"Radially averaged log-power spectrum of R\\n\"\r\n r\"$\\omega_0=%.0f km, \\beta_1=%.1f, \\beta_2=%.1f$\" % (w0, b1, b2)\r\n)\r\nplt.show()\r\n\r\n###############################################################################\r\n# Nonparametric filter\r\n# --------------------\r\n#\r\n# In the nonparametric approach, the Fourier filter is obtained directly\r\n# from the power spectrum of the observed precipitation field R.\r\n\r\nFnp = initialize_nonparam_2d_fft_filter(R)\r\n\r\n###############################################################################\r\n# Noise generator\r\n# ---------------\r\n#\r\n# The parametric and nonparametric filters obtained above can now be used\r\n# to produce N realizations of random fields of prescribed power spectrum,\r\n# hence with the same correlation structure as the initial rainfall field.\r\n\r\nseed = 42\r\nnum_realizations = 3\r\n\r\n# Generate noise\r\nNp = []\r\nNnp = []\r\nfor k in range(num_realizations):\r\n Np.append(generate_noise_2d_fft_filter(Fp, seed=seed + k))\r\n Nnp.append(generate_noise_2d_fft_filter(Fnp, seed=seed + k))\r\n\r\n# Plot the generated noise fields\r\n\r\nfig, ax = plt.subplots(nrows=2, ncols=3)\r\n\r\n# parametric noise\r\nax[0, 0].imshow(Np[0], cmap=cm.RdBu_r, vmin=-3, vmax=3)\r\nax[0, 1].imshow(Np[1], cmap=cm.RdBu_r, vmin=-3, vmax=3)\r\nax[0, 2].imshow(Np[2], cmap=cm.RdBu_r, vmin=-3, vmax=3)\r\n\r\n# nonparametric noise\r\nax[1, 0].imshow(Nnp[0], cmap=cm.RdBu_r, vmin=-3, vmax=3)\r\nax[1, 1].imshow(Nnp[1], cmap=cm.RdBu_r, vmin=-3, vmax=3)\r\nax[1, 2].imshow(Nnp[2], cmap=cm.RdBu_r, vmin=-3, vmax=3)\r\n\r\nfor i in range(2):\r\n for j in range(3):\r\n ax[i, j].set_xticks([])\r\n ax[i, j].set_yticks([])\r\nplt.tight_layout()\r\nplt.show()\r\n\r\n###############################################################################\r\n# The above figure highlights the main limitation of the parametric approach\r\n# (top row), that is, the assumption of an isotropic power law scaling\r\n# relationship, meaning that anisotropic structures such as rainfall bands\r\n# cannot be represented.\r\n#\r\n# Instead, the nonparametric approach (bottom row) allows generating\r\n# perturbation fields with anisotropic structures, but it also requires a\r\n# larger sample size and is sensitive to the quality of the input data, e.g.\r\n# the presence of residual clutter in the radar image.\r\n#\r\n# In addition, both techniques assume spatial stationarity of the covariance\r\n# structure of the field.\r\n\r\n# sphinx_gallery_thumbnail_number = 3\r\n" }, { "path": "examples/plot_optical_flow.py", "content": "\"\"\"\r\nOptical flow\r\n============\r\n\r\nThis tutorial offers a short overview of the optical flow routines available in\r\npysteps and it will cover how to compute and plot the motion field from a\r\nsequence of radar images.\r\n\"\"\"\r\n\r\nfrom datetime import datetime\r\nfrom pprint import pprint\r\nimport matplotlib.pyplot as plt\r\nimport numpy as np\r\n\r\nfrom pysteps import io, motion, rcparams\r\nfrom pysteps.utils import conversion, transformation\r\nfrom pysteps.visualization import plot_precip_field, quiver\r\n\r\n################################################################################\r\n# Read the radar input images\r\n# ---------------------------\r\n#\r\n# First, we will import the sequence of radar composites.\r\n# You need the pysteps-data archive downloaded and the pystepsrc file\r\n# configured with the data_source paths pointing to data folders.\r\n\r\n# Selected case\r\ndate = datetime.strptime(\"201505151630\", \"%Y%m%d%H%M\")\r\ndata_source = rcparams.data_sources[\"mch\"]\r\n\r\n###############################################################################\r\n# Load the data from the archive\r\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\r\n\r\nroot_path = data_source[\"root_path\"]\r\npath_fmt = data_source[\"path_fmt\"]\r\nfn_pattern = data_source[\"fn_pattern\"]\r\nfn_ext = data_source[\"fn_ext\"]\r\nimporter_name = data_source[\"importer\"]\r\nimporter_kwargs = data_source[\"importer_kwargs\"]\r\ntimestep = data_source[\"timestep\"]\r\n\r\n# Find the input files from the archive\r\nfns = io.archive.find_by_date(\r\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep=5, num_prev_files=9\r\n)\r\n\r\n# Read the radar composites\r\nimporter = io.get_method(importer_name, \"importer\")\r\nR, quality, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\r\n\r\ndel quality # Not used\r\n\r\n###############################################################################\r\n# Preprocess the data\r\n# ~~~~~~~~~~~~~~~~~~~\r\n\r\n# Convert to mm/h\r\nR, metadata = conversion.to_rainrate(R, metadata)\r\n\r\n# Store the reference frame\r\nR_ = R[-1, :, :].copy()\r\n\r\n# Log-transform the data [dBR]\r\nR, metadata = transformation.dB_transform(R, metadata, threshold=0.1, zerovalue=-15.0)\r\n\r\n# Nicely print the metadata\r\npprint(metadata)\r\n\r\n################################################################################\r\n# Lucas-Kanade (LK)\r\n# -----------------\r\n#\r\n# The Lucas-Kanade optical flow method implemented in pysteps is a local\r\n# tracking approach that relies on the OpenCV package.\r\n# Local features are tracked in a sequence of two or more radar images. The\r\n# scheme includes a final interpolation step in order to produce a smooth\r\n# field of motion vectors.\r\n\r\noflow_method = motion.get_method(\"LK\")\r\nV1 = oflow_method(R[-3:, :, :])\r\n\r\n# Plot the motion field on top of the reference frame\r\nplot_precip_field(R_, geodata=metadata, title=\"LK\")\r\nquiver(V1, geodata=metadata, step=25)\r\nplt.show()\r\n\r\n################################################################################\r\n# Variational echo tracking (VET)\r\n# -------------------------------\r\n#\r\n# This module implements the VET algorithm presented\r\n# by Laroche and Zawadzki (1995) and used in the McGill Algorithm for\r\n# Prediction by Lagrangian Extrapolation (MAPLE) described in\r\n# Germann and Zawadzki (2002).\r\n# The approach essentially consists of a global optimization routine that seeks\r\n# at minimizing a cost function between the displaced and the reference image.\r\n\r\noflow_method = motion.get_method(\"VET\")\r\nV2 = oflow_method(R[-3:, :, :])\r\n\r\n# Plot the motion field\r\nplot_precip_field(R_, geodata=metadata, title=\"VET\")\r\nquiver(V2, geodata=metadata, step=25)\r\nplt.show()\r\n\r\n################################################################################\r\n# Dynamic and adaptive radar tracking of storms (DARTS)\r\n# -----------------------------------------------------\r\n#\r\n# DARTS uses a spectral approach to optical flow that is based on the discrete\r\n# Fourier transform (DFT) of a temporal sequence of radar fields.\r\n# The level of truncation of the DFT coefficients controls the degree of\r\n# smoothness of the estimated motion field, allowing for an efficient\r\n# motion estimation. DARTS requires a longer sequence of radar fields for\r\n# estimating the motion, here we are going to use all the available 10 fields.\r\n\r\noflow_method = motion.get_method(\"DARTS\")\r\nR[~np.isfinite(R)] = metadata[\"zerovalue\"]\r\nV3 = oflow_method(R) # needs longer training sequence\r\n\r\n# Plot the motion field\r\nplot_precip_field(R_, geodata=metadata, title=\"DARTS\")\r\nquiver(V3, geodata=metadata, step=25)\r\nplt.show()\r\n\r\n################################################################################\r\n# Anisotropic diffusion method (Proesmans et al 1994)\r\n# ---------------------------------------------------\r\n#\r\n# This module implements the anisotropic diffusion method presented in Proesmans\r\n# et al. (1994), a robust optical flow technique which employs the notion of\r\n# inconsistency during the solution of the optical flow equations.\r\n\r\noflow_method = motion.get_method(\"proesmans\")\r\nR[~np.isfinite(R)] = metadata[\"zerovalue\"]\r\nV4 = oflow_method(R[-2:, :, :])\r\n\r\n# Plot the motion field\r\nplot_precip_field(R_, geodata=metadata, title=\"Proesmans\")\r\nquiver(V4, geodata=metadata, step=25)\r\nplt.show()\r\n\r\n################################################################################\r\n# Farnebäck smoothed method\r\n# -------------------------\r\n#\r\n# This module implements the pyramidal decomposition method for motion estimation\r\n# of Farnebäck as implemented in OpenCV, with an option for smoothing and\r\n# renormalization of the motion fields proposed by Driedger et al.:\r\n# https://cmosarchives.ca/Congress_P_A/program_abstracts2022.pdf (p. 392).\r\n\r\noflow_method = motion.get_method(\"farneback\")\r\nR[~np.isfinite(R)] = metadata[\"zerovalue\"]\r\nV5 = oflow_method(R[-2:, :, :], verbose=True)\r\n\r\n# Plot the motion field\r\nplot_precip_field(R_, geodata=metadata, title=\"Farneback\")\r\nquiver(V5, geodata=metadata, step=25)\r\nplt.show()\r\n\r\n# sphinx_gallery_thumbnail_number = 1\r\n" }, { "path": "examples/plot_steps_nowcast.py", "content": "#!/bin/env python\n\"\"\"\nSTEPS nowcast\n=============\n\nThis tutorial shows how to compute and plot an ensemble nowcast using Swiss\nradar data.\n\n\"\"\"\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nfrom datetime import datetime\nfrom pprint import pprint\nfrom pysteps import io, nowcasts, rcparams\nfrom pysteps.motion.lucaskanade import dense_lucaskanade\nfrom pysteps.postprocessing.ensemblestats import excprob\nfrom pysteps.utils import conversion, dimension, transformation\nfrom pysteps.visualization import plot_precip_field\n\n# Set nowcast parameters\nn_ens_members = 20\nn_leadtimes = 6\nseed = 24\n\n###############################################################################\n# Read precipitation field\n# ------------------------\n#\n# First thing, the sequence of Swiss radar composites is imported, converted and\n# transformed into units of dBR.\n\n\ndate = datetime.strptime(\"201701311200\", \"%Y%m%d%H%M\")\ndata_source = \"mch\"\n\n# Load data source config\nroot_path = rcparams.data_sources[data_source][\"root_path\"]\npath_fmt = rcparams.data_sources[data_source][\"path_fmt\"]\nfn_pattern = rcparams.data_sources[data_source][\"fn_pattern\"]\nfn_ext = rcparams.data_sources[data_source][\"fn_ext\"]\nimporter_name = rcparams.data_sources[data_source][\"importer\"]\nimporter_kwargs = rcparams.data_sources[data_source][\"importer_kwargs\"]\ntimestep = rcparams.data_sources[data_source][\"timestep\"]\n\n# Find the radar files in the archive\nfns = io.find_by_date(\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_prev_files=2\n)\n\n# Read the data from the archive\nimporter = io.get_method(importer_name, \"importer\")\nR, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\n\n# Convert to rain rate\nR, metadata = conversion.to_rainrate(R, metadata)\n\n# Upscale data to 2 km to limit memory usage\nR, metadata = dimension.aggregate_fields_space(R, metadata, 2000)\n\n# Plot the rainfall field\nplot_precip_field(R[-1, :, :], geodata=metadata)\nplt.show()\n\n# Log-transform the data to unit of dBR, set the threshold to 0.1 mm/h,\n# set the fill value to -15 dBR\nR, metadata = transformation.dB_transform(R, metadata, threshold=0.1, zerovalue=-15.0)\n\n# Set missing values with the fill value\nR[~np.isfinite(R)] = -15.0\n\n# Nicely print the metadata\npprint(metadata)\n\n###############################################################################\n# Deterministic nowcast with S-PROG\n# ---------------------------------\n#\n# First, the motiong field is estimated using a local tracking approach based\n# on the Lucas-Kanade optical flow.\n# The motion field can then be used to generate a deterministic nowcast with\n# the S-PROG model, which implements a scale filtering appraoch in order to\n# progressively remove the unpredictable spatial scales during the forecast.\n\n# Estimate the motion field\nV = dense_lucaskanade(R)\n\n# The S-PROG nowcast\nnowcast_method = nowcasts.get_method(\"sprog\")\nR_f = nowcast_method(\n R[-3:, :, :],\n V,\n n_leadtimes,\n n_cascade_levels=6,\n precip_thr=-10.0,\n)\n\n# Back-transform to rain rate\nR_f = transformation.dB_transform(R_f, threshold=-10.0, inverse=True)[0]\n\n# Plot the S-PROG forecast\nplot_precip_field(\n R_f[-1, :, :],\n geodata=metadata,\n title=\"S-PROG (+ %i min)\" % (n_leadtimes * timestep),\n)\nplt.show()\n\n###############################################################################\n# As we can see from the figure above, the forecast produced by S-PROG is a\n# smooth field. In other words, the forecast variance is lower than the\n# variance of the original observed field.\n# However, certain applications demand that the forecast retain the same\n# statistical properties of the observations. In such cases, the S-PROG\n# forecasts are of limited use and a stochatic approach might be of more\n# interest.\n\n###############################################################################\n# Stochastic nowcast with STEPS\n# -----------------------------\n#\n# The S-PROG approach is extended to include a stochastic term which represents\n# the variance associated to the unpredictable development of precipitation. This\n# approach is known as STEPS (short-term ensemble prediction system).\n\n# The STEPS nowcast\nnowcast_method = nowcasts.get_method(\"steps\")\nR_f = nowcast_method(\n R[-3:, :, :],\n V,\n n_leadtimes,\n n_ens_members,\n n_cascade_levels=6,\n precip_thr=-10.0,\n kmperpixel=2.0,\n timestep=timestep,\n noise_method=\"nonparametric\",\n vel_pert_method=\"bps\",\n mask_method=\"incremental\",\n seed=seed,\n)\n\n# Back-transform to rain rates\nR_f = transformation.dB_transform(R_f, threshold=-10.0, inverse=True)[0]\n\n\n# Plot the ensemble mean\nR_f_mean = np.mean(R_f[:, -1, :, :], axis=0)\nplot_precip_field(\n R_f_mean,\n geodata=metadata,\n title=\"Ensemble mean (+ %i min)\" % (n_leadtimes * timestep),\n)\nplt.show()\n\n###############################################################################\n# The mean of the ensemble displays similar properties as the S-PROG\n# forecast seen above, although the degree of smoothing also depends on\n# the ensemble size. In this sense, the S-PROG forecast can be seen as\n# the mean of an ensemble of infinite size.\n\n# Plot some of the realizations\nfig = plt.figure()\nfor i in range(4):\n ax = fig.add_subplot(221 + i)\n ax = plot_precip_field(\n R_f[i, -1, :, :], geodata=metadata, colorbar=False, axis=\"off\"\n )\n ax.set_title(\"Member %02d\" % i)\nplt.tight_layout()\nplt.show()\n\n###############################################################################\n# As we can see from these two members of the ensemble, the stochastic forecast\n# mantains the same variance as in the observed rainfall field.\n# STEPS also includes a stochatic perturbation of the motion field in order\n# to quantify the its uncertainty.\n\n###############################################################################\n# Finally, it is possible to derive probabilities from our ensemble forecast.\n\n# Compute exceedence probabilities for a 0.5 mm/h threshold\nP = excprob(R_f[:, -1, :, :], 0.5)\n\n# Plot the field of probabilities\nplot_precip_field(\n P,\n geodata=metadata,\n ptype=\"prob\",\n units=\"mm/h\",\n probthr=0.5,\n title=\"Exceedence probability (+ %i min)\" % (n_leadtimes * timestep),\n)\nplt.show()\n\n# sphinx_gallery_thumbnail_number = 5\n" }, { "path": "examples/probability_forecast.py", "content": "#!/bin/env python\n\"\"\"\nProbability forecasts\n=====================\n\nThis example script shows how to forecast the probability of exceeding an\nintensity threshold.\n\nThe method is based on the local Lagrangian approach described in Germann and\nZawadzki (2004).\n\"\"\"\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nfrom pysteps.nowcasts.lagrangian_probability import forecast\nfrom pysteps.visualization import plot_precip_field\n\n###############################################################################\n# Numerical example\n# -----------------\n#\n# First, we use some dummy data to show the basic principle of this approach.\n# The probability forecast is produced by sampling a spatial neighborhood that is\n# increased as a function of lead time. As a result, the edges of\n# the yellow square becomes more and more smooth as t increases. This represents\n# the strong loss of predictability with lead time of any extrapolation nowcast.\n\n# parameters\nprecip = np.zeros((100, 100))\nprecip[10:50, 10:50] = 1\nvelocity = np.ones((2, *precip.shape))\ntimesteps = [0, 2, 6, 12]\nthr = 0.5\nslope = 1 # pixels / timestep\n\n# compute probability forecast\nout = forecast(precip, velocity, timesteps, thr, slope=slope)\n# plot\nfor n, frame in enumerate(out):\n plt.subplot(2, 2, n + 1)\n plt.imshow(frame, interpolation=\"nearest\", vmin=0, vmax=1)\n plt.title(f\"t={timesteps[n]}\")\n plt.xticks([])\n plt.yticks([])\nplt.show()\n\n###############################################################################\n# Real-data example\n# -----------------\n#\n# We now apply the same method to real data. We use a slope of 1 km / minute\n# as suggested by Germann and Zawadzki (2004), meaning that after 30 minutes,\n# the probabilities are computed by using all pixels within a neighborhood of 30\n# kilometers.\n\nfrom datetime import datetime\n\nfrom pysteps import io, rcparams, utils\nfrom pysteps.motion.lucaskanade import dense_lucaskanade\nfrom pysteps.verification import reldiag_init, reldiag_accum, plot_reldiag\n\n# data source\nsource = rcparams.data_sources[\"mch\"]\nroot = rcparams.data_sources[\"mch\"][\"root_path\"]\nfmt = rcparams.data_sources[\"mch\"][\"path_fmt\"]\npattern = rcparams.data_sources[\"mch\"][\"fn_pattern\"]\next = rcparams.data_sources[\"mch\"][\"fn_ext\"]\ntimestep = rcparams.data_sources[\"mch\"][\"timestep\"]\nimporter_name = rcparams.data_sources[\"mch\"][\"importer\"]\nimporter_kwargs = rcparams.data_sources[\"mch\"][\"importer_kwargs\"]\n\n# read precip field\ndate = datetime.strptime(\"201607112100\", \"%Y%m%d%H%M\")\nfns = io.find_by_date(date, root, fmt, pattern, ext, timestep, num_prev_files=2)\nimporter = io.get_method(importer_name, \"importer\")\nprecip, __, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\nprecip, metadata = utils.to_rainrate(precip, metadata)\n# precip[np.isnan(precip)] = 0\n\n# motion\nmotion = dense_lucaskanade(precip)\n\n# parameters\nnleadtimes = 6\nthr = 1 # mm / h\nslope = 1 * timestep # km / min\n\n# compute probability forecast\nextrap_kwargs = dict(allow_nonfinite_values=True)\nfct = forecast(\n precip[-1], motion, nleadtimes, thr, slope=slope, extrap_kwargs=extrap_kwargs\n)\n\n# plot\nfor n, frame in enumerate(fct):\n plt.subplot(2, 3, n + 1)\n plt.imshow(frame, interpolation=\"nearest\", vmin=0, vmax=1)\n plt.xticks([])\n plt.yticks([])\nplt.show()\n\n################################################################################\n# Let's plot one single leadtime in more detail using the pysteps visualization\n# functionality.\n\nplt.close()\n# Plot the field of probabilities\nplot_precip_field(\n fct[2],\n geodata=metadata,\n ptype=\"prob\",\n probthr=thr,\n title=\"Exceedence probability (+ %i min)\" % (nleadtimes * timestep),\n)\nplt.show()\n\n###############################################################################\n# Verification\n# ------------\n\n# verifying observations\nimporter = io.get_method(importer_name, \"importer\")\nfns = io.find_by_date(\n date, root, fmt, pattern, ext, timestep, num_next_files=nleadtimes\n)\nobs, __, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\nobs, metadata = utils.to_rainrate(obs, metadata)\nobs[np.isnan(obs)] = 0\n\n# reliability diagram\nreldiag = reldiag_init(thr)\nreldiag_accum(reldiag, fct, obs[1:])\nfig, ax = plt.subplots()\nplot_reldiag(reldiag, ax)\nax.set_title(\"Reliability diagram\")\nplt.show()\n\n\n###############################################################################\n# References\n# ----------\n# Germann, U. and I. Zawadzki, 2004:\n# Scale Dependence of the Predictability of Precipitation from Continental\n# Radar Images. Part II: Probability Forecasts.\n# Journal of Applied Meteorology, 43(1), 74-89.\n\n# sphinx_gallery_thumbnail_number = 3\n" }, { "path": "examples/rainfarm_downscale.py", "content": "#!/bin/env python\r\n\"\"\"\r\nPrecipitation downscaling with RainFARM\r\n=======================================\r\n\r\nThis example script shows how to use the stochastic downscaling method RainFARM\r\navailable in pysteps.\r\n\r\nRainFARM is a downscaling algorithm for rainfall fields developed by Rebora et\r\nal. (2006). The method can represent the realistic small-scale variability of the\r\ndownscaled precipitation field by means of Gaussian random fields.\r\n\r\nSteps:\r\n 1. Read the input precipitation data.\r\n 2. Upscale the precipitation field.\r\n 3. Downscale the field to its original resolution using RainFARM with defaults.\r\n 4. Downscale with smoothing.\r\n 5. Downscale with spectral fusion.\r\n 6. Downscale with smoothing and spectral fusion.\r\n\r\nReferences:\r\n\r\n Rebora, N., L. Ferraris, J. von Hardenberg, and A. Provenzale, 2006: RainFARM:\r\n Rainfall downscaling by a filtered autoregressive model. J. Hydrometeor., 7,\r\n 724–738.\r\n\r\n D D'Onofrio, E Palazzi, J von Hardenberg, A Provenzale, and S Calmanti, 2014:\r\n Stochastic rainfall downscaling of climate models. J. Hydrometeorol., 15(2):830–843.\r\n\"\"\"\r\n\r\nimport matplotlib.pyplot as plt\r\nimport numpy as np\r\nimport os\r\nfrom pprint import pprint\r\nimport logging\r\n\r\nfrom pysteps import io, rcparams\r\nfrom pysteps.utils import aggregate_fields_space, square_domain, to_rainrate\r\nfrom pysteps.downscaling import rainfarm\r\nfrom pysteps.visualization import plot_precip_field\r\n\r\n# Configure logging\r\nlogging.basicConfig(\r\n level=logging.INFO, format=\"%(asctime)s - %(levelname)s - %(message)s\"\r\n)\r\n\r\n###############################################################################\r\n# Read the input data\r\n# -------------------\r\n#\r\n# As first step, we need to import the precipitation field that we are going\r\n# to use in this example.\r\n\r\n\r\ndef read_precipitation_data(file_path):\r\n \"\"\"Read and process precipitation data from a file.\"\"\"\r\n precip, _, metadata = io.import_mch_gif(\r\n file_path, product=\"AQC\", unit=\"mm\", accutime=5.0\r\n )\r\n precip, metadata = to_rainrate(precip, metadata)\r\n precip, metadata = square_domain(precip, metadata, \"crop\")\r\n return precip, metadata\r\n\r\n\r\n# Import the example radar composite\r\nroot_path = rcparams.data_sources[\"mch\"][\"root_path\"]\r\nfilename = os.path.join(root_path, \"20160711\", \"AQC161932100V_00005.801.gif\")\r\n\r\n# Read and process data\r\nprecip, metadata = read_precipitation_data(filename)\r\n\r\n# Nicely print the metadata\r\npprint(metadata)\r\n\r\n# Plot the original rainfall field\r\nplot_precip_field(precip, geodata=metadata)\r\nplt.title(\"Original Rainfall Field\")\r\nplt.show()\r\n\r\n# Assign the fill value to all the Nans\r\nprecip[~np.isfinite(precip)] = metadata[\"zerovalue\"]\r\n\r\n###############################################################################\r\n# Upscale the field\r\n# -----------------\r\n#\r\n# To test our downscaling method, we first need to upscale the original field to\r\n# a lower resolution. This is only for demo purposes, as we need to artificially\r\n# create a lower resolution field to apply our downscaling method.\r\n# We are going to use a factor of 16 x.\r\n\r\n\r\ndef upscale_field(precip, metadata, scale_factor):\r\n \"\"\"Upscale the precipitation field by a given scale factor.\"\"\"\r\n upscaled_resolution = metadata[\"xpixelsize\"] * scale_factor\r\n precip_lr, metadata_lr = aggregate_fields_space(\r\n precip, metadata, upscaled_resolution\r\n )\r\n return precip_lr, metadata_lr\r\n\r\n\r\nscale_factor = 16\r\nprecip_lr, metadata_lr = upscale_field(precip, metadata, scale_factor)\r\n\r\n# Plot the upscaled rainfall field\r\nplt.figure()\r\nplot_precip_field(precip_lr, geodata=metadata_lr)\r\nplt.title(\"Upscaled Rainfall Field\")\r\nplt.show()\r\n\r\n###############################################################################\r\n# Downscale the field\r\n# -------------------\r\n#\r\n# We can now use RainFARM to downscale the precipitation field.\r\n\r\n# Basic downscaling\r\nprecip_hr = rainfarm.downscale(precip_lr, ds_factor=scale_factor)\r\n\r\n# Plot the downscaled rainfall field\r\nplt.figure()\r\nplot_precip_field(precip_hr, geodata=metadata)\r\nplt.title(\"Downscaled Rainfall Field\")\r\nplt.show()\r\n\r\n###############################################################################\r\n# Downscale with smoothing\r\n# ------------------------\r\n#\r\n# Add smoothing with a Gaussian kernel during the downscaling process.\r\n\r\nprecip_hr_smooth = rainfarm.downscale(\r\n precip_lr, ds_factor=scale_factor, kernel_type=\"gaussian\"\r\n)\r\n\r\n# Plot the downscaled rainfall field with smoothing\r\nplt.figure()\r\nplot_precip_field(precip_hr_smooth, geodata=metadata)\r\nplt.title(\"Downscaled Rainfall Field with Gaussian Smoothing\")\r\nplt.show()\r\n\r\n###############################################################################\r\n# Downscale with spectral fusion\r\n# ------------------------------\r\n#\r\n# Apply spectral merging as described in D'Onofrio et al. (2014).\r\n\r\nprecip_hr_fusion = rainfarm.downscale(\r\n precip_lr, ds_factor=scale_factor, spectral_fusion=True\r\n)\r\n\r\n# Plot the downscaled rainfall field with spectral fusion\r\nplt.figure()\r\nplot_precip_field(precip_hr_fusion, geodata=metadata)\r\nplt.title(\"Downscaled Rainfall Field with Spectral Fusion\")\r\nplt.show()\r\n\r\n###############################################################################\r\n# Combined Downscale with smoothing and spectral fusion\r\n# -----------------------------------------------------\r\n#\r\n# Apply both smoothing with a Gaussian kernel and spectral fusion during the\r\n# downscaling process to observe the combined effect.\r\n\r\nprecip_hr_combined = rainfarm.downscale(\r\n precip_lr, ds_factor=scale_factor, kernel_type=\"gaussian\", spectral_fusion=True\r\n)\r\n\r\n# Plot the downscaled rainfall field with smoothing and spectral fusion\r\nplt.figure()\r\nplot_precip_field(precip_hr_combined, geodata=metadata)\r\nplt.title(\"Downscaled Rainfall Field with Gaussian Smoothing and Spectral Fusion\")\r\nplt.show()\r\n\r\n###############################################################################\r\n# Remarks\r\n# -------\r\n#\r\n# Currently, the pysteps implementation of RainFARM only covers spatial downscaling.\r\n# That is, it can improve the spatial resolution of a rainfall field. However, unlike\r\n# the original algorithm from Rebora et al. (2006), it cannot downscale the temporal\r\n# dimension.\r\n\r\n# sphinx_gallery_thumbnail_number = 2\r\n" }, { "path": "examples/steps_blended_forecast.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nBlended forecast\n====================\n\nThis tutorial shows how to construct a blended forecast from an ensemble nowcast\nusing the STEPS approach and a Numerical Weather Prediction (NWP) rainfall\nforecast. The used datasets are from the Bureau of Meteorology, Australia.\n\n\"\"\"\n\nimport os\nfrom datetime import datetime\n\nimport numpy as np\nfrom matplotlib import pyplot as plt\n\nimport pysteps\nfrom pysteps import io, rcparams, blending, nowcasts\nfrom pysteps.visualization import plot_precip_field\n\n################################################################################\n# Read the radar images and the NWP forecast\n# ------------------------------------------\n#\n# First, we import a sequence of 3 images of 10-minute radar composites\n# and the corresponding NWP rainfall forecast that was available at that time.\n#\n# You need the pysteps-data archive downloaded and the pystepsrc file\n# configured with the data_source paths pointing to data folders.\n# Additionally, the pysteps-nwp-importers plugin needs to be installed, see\n# https://github.com/pySTEPS/pysteps-nwp-importers.\n\n# Selected case\ndate_radar = datetime.strptime(\"202010310400\", \"%Y%m%d%H%M\")\n# The last NWP forecast was issued at 00:00\ndate_nwp = datetime.strptime(\"202010310000\", \"%Y%m%d%H%M\")\nradar_data_source = rcparams.data_sources[\"bom\"]\nnwp_data_source = rcparams.data_sources[\"bom_nwp\"]\n\n###############################################################################\n# Load the data from the archive\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nroot_path = radar_data_source[\"root_path\"]\npath_fmt = \"prcp-c10/66/%Y/%m/%d\"\nfn_pattern = \"66_%Y%m%d_%H%M00.prcp-c10\"\nfn_ext = radar_data_source[\"fn_ext\"]\nimporter_name = radar_data_source[\"importer\"]\nimporter_kwargs = radar_data_source[\"importer_kwargs\"]\ntimestep = 10.0\n\n# Find the radar files in the archive\nfns = io.find_by_date(\n date_radar, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_prev_files=2\n)\n\n# Read the radar composites\nimporter = io.get_method(importer_name, \"importer\")\nradar_precip, _, radar_metadata = io.read_timeseries(fns, importer, **importer_kwargs)\n\n# Import the NWP data\nfilename = os.path.join(\n nwp_data_source[\"root_path\"],\n datetime.strftime(date_nwp, nwp_data_source[\"path_fmt\"]),\n datetime.strftime(date_nwp, nwp_data_source[\"fn_pattern\"])\n + \".\"\n + nwp_data_source[\"fn_ext\"],\n)\n\nnwp_importer = io.get_method(\"bom_nwp\", \"importer\")\nnwp_precip, _, nwp_metadata = nwp_importer(filename)\n\n# Only keep the NWP forecasts from the last radar observation time (2020-10-31 04:00)\n# onwards\n\nnwp_precip = nwp_precip[24:43, :, :]\n\n\n################################################################################\n# Pre-processing steps\n# --------------------\n\n# Make sure the units are in mm/h\nconverter = pysteps.utils.get_method(\"mm/h\")\nradar_precip, radar_metadata = converter(radar_precip, radar_metadata)\nnwp_precip, nwp_metadata = converter(nwp_precip, nwp_metadata)\n\n# Threshold the data\nradar_precip[radar_precip < 0.1] = 0.0\nnwp_precip[nwp_precip < 0.1] = 0.0\n\n# Plot the radar rainfall field and the first time step of the NWP forecast.\ndate_str = datetime.strftime(date_radar, \"%Y-%m-%d %H:%M\")\nplt.figure(figsize=(10, 5))\nplt.subplot(121)\nplot_precip_field(\n radar_precip[-1, :, :],\n geodata=radar_metadata,\n title=f\"Radar observation at {date_str}\",\n colorscale=\"STEPS-NL\",\n)\nplt.subplot(122)\nplot_precip_field(\n nwp_precip[0, :, :],\n geodata=nwp_metadata,\n title=f\"NWP forecast at {date_str}\",\n colorscale=\"STEPS-NL\",\n)\nplt.tight_layout()\nplt.show()\n\n# transform the data to dB\ntransformer = pysteps.utils.get_method(\"dB\")\nradar_precip, radar_metadata = transformer(radar_precip, radar_metadata, threshold=0.1)\nnwp_precip, nwp_metadata = transformer(nwp_precip, nwp_metadata, threshold=0.1)\n\n# r_nwp has to be four dimentional (n_models, time, y, x).\n# If we only use one model:\nif nwp_precip.ndim == 3:\n nwp_precip = nwp_precip[None, :]\n\n###############################################################################\n# For the initial time step (t=0), the NWP rainfall forecast is not that different\n# from the observed radar rainfall, but it misses some of the locations and\n# shapes of the observed rainfall fields. Therefore, the NWP rainfall forecast will\n# initially get a low weight in the blending process.\n#\n# Determine the velocity fields\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\noflow_method = pysteps.motion.get_method(\"lucaskanade\")\n\n# First for the radar images\nvelocity_radar = oflow_method(radar_precip)\n\n# Then for the NWP forecast\nvelocity_nwp = []\n# Loop through the models\nfor n_model in range(nwp_precip.shape[0]):\n # Loop through the timesteps. We need two images to construct a motion\n # field, so we can start from timestep 1. Timestep 0 will be the same\n # as timestep 1.\n _v_nwp_ = []\n for t in range(1, nwp_precip.shape[1]):\n v_nwp_ = oflow_method(nwp_precip[n_model, t - 1 : t + 1, :])\n _v_nwp_.append(v_nwp_)\n v_nwp_ = None\n # Add the velocity field at time step 1 to time step 0.\n _v_nwp_ = np.insert(_v_nwp_, 0, _v_nwp_[0], axis=0)\n velocity_nwp.append(_v_nwp_)\nvelocity_nwp = np.stack(velocity_nwp)\n\n\n################################################################################\n# The blended forecast\n# ~~~~~~~~~~~~~~~~~~~~\n\nprecip_forecast = blending.steps.forecast(\n precip=radar_precip,\n precip_models=nwp_precip,\n velocity=velocity_radar,\n velocity_models=velocity_nwp,\n timesteps=18,\n timestep=timestep,\n issuetime=date_radar,\n n_ens_members=1,\n precip_thr=radar_metadata[\"threshold\"],\n kmperpixel=radar_metadata[\"xpixelsize\"] / 1000.0,\n noise_stddev_adj=\"auto\",\n vel_pert_method=None,\n)\n\n# Transform the data back into mm/h\nprecip_forecast, _ = converter(precip_forecast, radar_metadata)\nradar_precip_mmh, _ = converter(radar_precip, radar_metadata)\nnwp_precip_mmh, _ = converter(nwp_precip, nwp_metadata)\n\n\n################################################################################\n# Visualize the output\n# ~~~~~~~~~~~~~~~~~~~~\n#\n# The NWP rainfall forecast has a lower weight than the radar-based extrapolation\n# forecast at the issue time of the forecast (+0 min). Therefore, the first time\n# steps consist mostly of the extrapolation.\n# However, near the end of the forecast (+180 min), the NWP share in the blended\n# forecast has become more important and the forecast starts to resemble the\n# NWP forecast more.\n\nfig = plt.figure(figsize=(5, 12))\n\nleadtimes_min = [30, 60, 90, 120, 150, 180]\nn_leadtimes = len(leadtimes_min)\nfor n, leadtime in enumerate(leadtimes_min):\n # Nowcast with blending into NWP\n ax1 = plt.subplot(n_leadtimes, 2, n * 2 + 1)\n plot_precip_field(\n precip_forecast[0, int(leadtime / timestep) - 1, :, :],\n geodata=radar_metadata,\n title=f\"Nowcast +{leadtime} min\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n ax1.axis(\"off\")\n\n # Raw NWP forecast\n plt.subplot(n_leadtimes, 2, n * 2 + 2)\n ax2 = plot_precip_field(\n nwp_precip_mmh[0, int(leadtime / timestep) - 1, :, :],\n geodata=nwp_metadata,\n title=f\"NWP +{leadtime} min\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n ax2.axis(\"off\")\n\nplt.tight_layout()\nplt.show()\n\n\n###############################################################################\n# It is also possible to blend a deterministic or probabilistic external nowcast\n# (e.g. a pre-made nowcast or a deterministic AI-based nowcast) with NWP using\n# the STEPS algorithm. In that case, we add a `precip_nowcast` to\n# `blending.steps.forecast`. By providing an external nowcast, the STEPS blending\n# method will omit the autoregression and advection step for the extrapolation\n# cascade and use the existing external nowcast instead (which will thus be\n# decomposed into multiplicative cascades!). The weights determination and\n# possible post-processings steps will remain the same.\n#\n# Start with creating an external nowcast\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n# We go for a simple advection-only nowcast for the example, but this setup can\n# be replaced with any external deterministic or probabilistic nowcast.\nextrapolate = nowcasts.get_method(\"extrapolation\")\nradar_precip_to_advect = radar_precip.copy()\nradar_metadata_to_advect = radar_metadata.copy()\n\n# Make sure the data has no nans\nradar_precip_to_advect[~np.isfinite(radar_precip_to_advect)] = -15\nradar_precip_to_advect = radar_precip_to_advect.data\n\n# Create the extrapolation\nfc_lagrangian_extrapolation = extrapolate(\n radar_precip_to_advect[-1, :, :], velocity_radar, 18\n)\n\n# Insert an additional timestep at the start, as t0, which is the same as the current first slice.\nfc_lagrangian_extrapolation = np.insert(\n fc_lagrangian_extrapolation, 0, fc_lagrangian_extrapolation[0:1, :, :], axis=0\n)\nfc_lagrangian_extrapolation[~np.isfinite(fc_lagrangian_extrapolation)] = (\n radar_metadata_to_advect[\"zerovalue\"]\n)\n\n\n################################################################################\n# Blend the external nowcast with NWP - deterministic mode\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nprecip_forecast = blending.steps.forecast(\n precip=radar_precip,\n precip_nowcast=np.array(\n [fc_lagrangian_extrapolation]\n ), # Add an extra dimension, becuase precip_nowcast has to be 4-dimensional\n precip_models=nwp_precip,\n velocity=velocity_radar,\n velocity_models=velocity_nwp,\n timesteps=18,\n timestep=timestep,\n issuetime=date_radar,\n n_ens_members=1,\n precip_thr=radar_metadata[\"threshold\"],\n kmperpixel=radar_metadata[\"xpixelsize\"] / 1000.0,\n noise_stddev_adj=\"auto\",\n vel_pert_method=None,\n nowcasting_method=\"external_nowcast\",\n noise_method=None,\n probmatching_method=None,\n mask_method=None,\n weights_method=\"bps\",\n)\n\n# Transform the data back into mm/h\nprecip_forecast, _ = converter(precip_forecast, radar_metadata)\nradar_precip_mmh, _ = converter(radar_precip, radar_metadata)\nfc_lagrangian_extrapolation_mmh, _ = converter(\n fc_lagrangian_extrapolation, radar_metadata_to_advect\n)\nnwp_precipfc_lagrangian_extrapolation_mmh_mmh, _ = converter(nwp_precip, nwp_metadata)\n\n\n################################################################################\n# Visualize the output\n# ~~~~~~~~~~~~~~~~~~~~\n#\n# The NWP rainfall forecast has a lower weight than the radar-based extrapolation\n# forecast at the issue time of the forecast (+0 min). Therefore, the first time\n# steps consist mostly of the extrapolation.\n# However, near the end of the forecast (+180 min), the NWP share in the blended\n# forecast has become more important and the forecast starts to resemble the\n# NWP forecast more.\n\nfig = plt.figure(figsize=(6, 12))\n\nleadtimes_min = [30, 60, 90, 120, 150, 180]\nn_leadtimes = len(leadtimes_min)\n\nfor n, leadtime in enumerate(leadtimes_min):\n idx = int(leadtime / timestep) - 1\n\n # Blended nowcast\n ax1 = plt.subplot(n_leadtimes, 3, n * 3 + 1)\n plot_precip_field(\n precip_forecast[0, idx, :, :],\n geodata=radar_metadata,\n title=f\"Blended +{leadtime} min\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n ax1.axis(\"off\")\n\n # Raw extrapolated nowcast\n ax2 = plt.subplot(n_leadtimes, 3, n * 3 + 2)\n plot_precip_field(\n fc_lagrangian_extrapolation_mmh[idx, :, :],\n geodata=radar_metadata,\n title=f\"NWC +{leadtime} min\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n ax2.axis(\"off\")\n\n # Raw NWP forecast\n plt.subplot(n_leadtimes, 3, n * 3 + 3)\n ax3 = plot_precip_field(\n nwp_precip_mmh[0, idx, :, :],\n geodata=nwp_metadata,\n title=f\"NWP +{leadtime} min\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n ax3.axis(\"off\")\n\n\n################################################################################\n# Blend the external nowcast with NWP - ensemble mode\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nprecip_forecast = blending.steps.forecast(\n precip=radar_precip,\n precip_nowcast=np.array(\n [fc_lagrangian_extrapolation]\n ), # Add an extra dimension, becuase precip_nowcast has to be 4-dimensional\n precip_models=nwp_precip,\n velocity=velocity_radar,\n velocity_models=velocity_nwp,\n timesteps=18,\n timestep=timestep,\n issuetime=date_radar,\n n_ens_members=5,\n precip_thr=radar_metadata[\"threshold\"],\n kmperpixel=radar_metadata[\"xpixelsize\"] / 1000.0,\n noise_stddev_adj=\"auto\",\n vel_pert_method=None,\n nowcasting_method=\"external_nowcast\",\n noise_method=\"nonparametric\",\n probmatching_method=\"cdf\",\n mask_method=\"incremental\",\n weights_method=\"bps\",\n)\n\n# Transform the data back into mm/h\nprecip_forecast, _ = converter(precip_forecast, radar_metadata)\nradar_precip_mmh, _ = converter(radar_precip, radar_metadata)\nfc_lagrangian_extrapolation_mmh, _ = converter(\n fc_lagrangian_extrapolation, radar_metadata_to_advect\n)\nnwp_precipfc_lagrangian_extrapolation_mmh_mmh, _ = converter(nwp_precip, nwp_metadata)\n\n\n################################################################################\n# Visualize the output\n# ~~~~~~~~~~~~~~~~~~~~\n\nfig = plt.figure(figsize=(8, 12))\n\nleadtimes_min = [30, 60, 90, 120, 150, 180]\nn_leadtimes = len(leadtimes_min)\n\nfor n, leadtime in enumerate(leadtimes_min):\n idx = int(leadtime / timestep) - 1\n\n # Blended nowcast member 1\n ax1 = plt.subplot(n_leadtimes, 4, n * 4 + 1)\n plot_precip_field(\n precip_forecast[0, idx, :, :],\n geodata=radar_metadata,\n title=\"Blend Mem. 1\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n ax1.axis(\"off\")\n\n # Blended nowcast member 5\n ax2 = plt.subplot(n_leadtimes, 4, n * 4 + 2)\n plot_precip_field(\n precip_forecast[4, idx, :, :],\n geodata=radar_metadata,\n title=\"Blend Mem. 5\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n ax2.axis(\"off\")\n\n # Raw extrapolated nowcast\n ax3 = plt.subplot(n_leadtimes, 4, n * 4 + 3)\n plot_precip_field(\n fc_lagrangian_extrapolation_mmh[idx, :, :],\n geodata=radar_metadata,\n title=f\"NWC + {leadtime} min\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n ax3.axis(\"off\")\n\n # Raw NWP forecast\n ax4 = plt.subplot(n_leadtimes, 4, n * 4 + 4)\n plot_precip_field(\n nwp_precip_mmh[0, idx, :, :],\n geodata=nwp_metadata,\n title=f\"NWP + {leadtime} min\",\n axis=\"off\",\n colorscale=\"STEPS-NL\",\n colorbar=False,\n )\n ax4.axis(\"off\")\n\nplt.show()\n\nprint(\"Done.\")\n\n\n################################################################################\n# References\n# ~~~~~~~~~~\n#\n# Bowler, N. E., and C. E. Pierce, and A. W. Seed. 2004. \"STEPS: A probabilistic\n# precipitation forecasting scheme which merges an extrapolation nowcast with\n# downscaled NWP.\" Forecasting Research Technical Report No. 433. Wallingford, UK.\n#\n# Bowler, N. E., and C. E. Pierce, and A. W. Seed. 2006. \"STEPS: A probabilistic\n# precipitation forecasting scheme which merges an extrapolation nowcast with\n# downscaled NWP.\" Quarterly Journal of the Royal Meteorological Society 132(16):\n# 2127-2155. https://doi.org/10.1256/qj.04.100\n#\n# Seed, A. W., and C. E. Pierce, and K. Norman. 2013. \"Formulation and evaluation\n# of a scale decomposition-based stochastic precipitation nowcast scheme.\" Water\n# Resources Research 49(10): 6624-664. https://doi.org/10.1002/wrcr.20536\n#\n# Imhoff, R.O., L. De Cruz, W. Dewettinck, C.C. Brauer, R. Uijlenhoet, K-J. van\n# Heeringen, C. Velasco-Forero, D. Nerini, M. Van Ginderachter, and A.H. Weerts.\n# 2023. \"Scale-dependent blending of ensemble rainfall nowcasts and NWP in the\n# open-source pysteps library\". Quarterly Journal of the Royal Meteorological\n# Society 149(753): 1-30. https://doi.org/10.1002/qj.4461\n" }, { "path": "examples/thunderstorm_detection_and_tracking.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nThunderstorm Detection and Tracking - T-DaTing\n============================================\n\nThis example shows how to use the thunderstorm DaTing module. The example is based on\nMeteoSwiss radar data and uses the Cartesian composite of maximum reflectivity on a\n1 km grid. All default values are tuned to this grid, but can be modified.\nThe first section demonstrates thunderstorm cell detection and how to plot contours.\nThe second section demonstrates detection and tracking in combination,\nas well as how to plot the resulting tracks.\nThis module was implemented following the procedures used in the TRT Thunderstorms\nRadar Tracking algorithm (:cite:`TRT2004`) used operationally at MeteoSwiss.\nModifications include advecting the identified thunderstorms with the optical flow\nobtained from pysteps, as well as additional options in the thresholding. A detailed\ndescription is published in Appendix A of :cite:`Feldmann2021`.\n\nReferences\n..........\n:cite:`TRT2004`\n:cite:`Feldmann2021`\n\n@author: feldmann-m\n\"\"\"\n\n################################################################################\n# Import all required functions\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nfrom datetime import datetime\nfrom pprint import pprint\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nfrom pysteps import io, rcparams\nfrom pysteps.feature import tstorm as tstorm_detect\nfrom pysteps.tracking import tdating as tstorm_dating\nfrom pysteps.utils import to_reflectivity\nfrom pysteps.visualization import plot_precip_field, plot_track, plot_cart_contour\n\n################################################################################\n# Read the radar input images\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n#\n# A series of 20 files containing Swiss Cartesian gridded rain rates are imported. Since\n# the algorithm is tuned to Swiss max-reflectivity data, the rain rates are transformed\n# to reflectivity fields using the 'to_reflectivity' utility in pysteps.utils.\n\n# Select the input data\ndate = datetime.strptime(\"201607112100\", \"%Y%m%d%H%M\")\ndata_source = rcparams.data_sources[\"mch\"]\n\n# Extract corresponding settings\nroot_path = data_source[\"root_path\"]\npath_fmt = data_source[\"path_fmt\"]\nfn_pattern = data_source[\"fn_pattern\"]\nfn_ext = data_source[\"fn_ext\"]\nimporter_name = data_source[\"importer\"]\nimporter_kwargs = data_source[\"importer_kwargs\"]\ntimestep = data_source[\"timestep\"]\n\n# Load the data from the archive\nfns = io.archive.find_by_date(\n date, root_path, path_fmt, fn_pattern, fn_ext, timestep, num_next_files=20\n)\nimporter = io.get_method(importer_name, \"importer\")\nR, _, metadata = io.read_timeseries(fns, importer, **importer_kwargs)\n\n# Convert to reflectivity (it is possible to give the a- and b- parameters of the\n# Marshall-Palmer relationship here: zr_a = and zr_b =).\nZ, metadata = to_reflectivity(R, metadata)\n\n# Extract the list of timestamps\ntimelist = metadata[\"timestamps\"]\n\npprint(metadata)\n\n###############################################################################\n# Example of thunderstorm identification in a single timestep\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n# The function tstorm_detect.detection requires a 2-D input image, all further inputs are\n# optional.\n\ninput_image = Z[2, :, :].copy()\ntime = timelist[2]\ncells_id, labels = tstorm_detect.detection(input_image, time=time)\n\n###############################################################################\n# Properties of one of the identified cells:\nprint(cells_id.iloc[0])\n\n###############################################################################\n# Optionally, one can also ask to consider splits and merges of thunderstorm cells.\n# A cell at time t is considered to split if it will verlap more than 10% with more than\n# one cell at time t+1. Conversely, a cell is considered to be a merge, if more\n# than one cells fron time t will overlap more than 10% with it.\n\ncells_id, labels = tstorm_detect.detection(\n input_image, time=time, output_splits_merges=True\n)\nprint(cells_id.iloc[0])\n\n###############################################################################\n# Example of thunderstorm tracking over a timeseries\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n# The tstorm-dating function requires the entire pre-loaded time series.\n# The first two timesteps are required to initialize the\n# flow prediction and are not used to compute tracks.\n\ntrack_list, cell_list, label_list = tstorm_dating.dating(\n input_video=Z, timelist=timelist\n)\n\n###############################################################################\n# Plotting the results\n# ~~~~~~~~~~~~~~~~~~~~\n\n# Plot precipitation field\nplot_precip_field(Z[2, :, :], geodata=metadata, units=metadata[\"unit\"])\nplt.xlabel(\"Swiss easting [m]\")\nplt.ylabel(\"Swiss northing [m]\")\n\n# Add the identified cells\nplot_cart_contour(cells_id.cont, geodata=metadata)\n\n# Filter the tracks to only contain cells existing in this timestep\nIDs = cells_id.ID.values\ntrack_filt = []\nfor track in track_list:\n if np.unique(track.ID) in IDs:\n track_filt.append(track)\n\n# Add their tracks\nplot_track(track_filt, geodata=metadata)\nplt.show()\n\n################################################################################\n# Evaluating temporal behaviour of cell\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n# Maximum reflectivity of cells in time\n\n# Make an empty list\ntlen = []\n# Get a list of colors that we will use for the plot\ncolor = iter(plt.cm.ocean(np.linspace(0, 0.8, len(track_filt))))\n# Now, loop through all the tracks and plot the maximum reflectivity of the cell\n# in time.\nfor track in track_filt:\n plt.plot(np.arange(len(track)), track.max_ref, c=next(color))\n tlen.append(len(track))\nplt.xticks(np.arange(max(tlen) + 1), labels=np.arange(max(tlen) + 1) * 5)\nplt.ylabel(\"Maximum reflectivity (dBZ)\")\nplt.xlabel(\"Time since cell detection (min)\")\nplt.legend(IDs, loc=\"lower right\", ncol=3, title=\"Track number\")\nplt.show()\n\n###############################################################################\n# The size of the thunderstorm cells in time\n\n# Make an empty list\ntlen = []\n# Get a list of colors that we will use for the plot\ncolor = iter(plt.cm.ocean(np.linspace(0, 0.8, len(track_filt))))\n# Now, loop through all the tracks and plot the cell size of the thunderstorms\n# in time.\nfor track in track_filt:\n size = []\n for ID, t in track.iterrows():\n size.append(len(t.x))\n plt.plot(np.arange(len(track)), size, c=next(color))\n tlen.append(len(track))\nplt.xticks(np.arange(max(tlen) + 1), labels=np.arange(max(tlen) + 1) * 5)\nplt.ylabel(\"Thunderstorm cell size (pixels)\")\nplt.xlabel(\"Time since cell detection (min)\")\nplt.legend(IDs, loc=\"upper left\", ncol=3, title=\"Track number\")\nplt.show()\n" }, { "path": "pyproject.toml", "content": "[build-system]\nrequires = [\n \"wheel\",\n \"setuptools>=40.8.0\",\n \"Cython>=0.29.2\",\n \"numpy>=1.13\"\n]\n# setuptools 40.8.0 is the first version of setuptools that offers\n# a PEP 517 backend that closely mimics directly executing setup.py.\nbuild-backend = \"setuptools.build_meta:__legacy__\"\n#https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support\n\n\n# Define black parameters for the project\n# https://black.readthedocs.io/en/stable/pyproject_toml.html#configuration-format\n[tool.black]\ntarget-version = ['py36']\nline-length = 88\nexclude = '''\n/(\n \\.eggs\n | \\.git\n | _build\n | build\n | dist\n)/\n'''\n" }, { "path": "pysteps/__init__.py", "content": "import json\nimport os\nimport stat\nimport sys\nimport warnings\n\nfrom jsmin import jsmin\nfrom jsonschema import Draft4Validator\n\n# import subpackages\nfrom . import blending\nfrom . import cascade\nfrom . import datasets\nfrom . import decorators\nfrom . import downscaling\nfrom . import exceptions\nfrom . import extrapolation\nfrom . import io\nfrom . import motion\nfrom . import noise\nfrom . import nowcasts\nfrom . import postprocessing\nfrom . import timeseries\nfrom . import utils\nfrom . import verification as vf\nfrom . import visualization as plt\n\n\ndef _get_config_file_schema():\n \"\"\"\n Return the path to the parameters file json schema.\n \"\"\"\n module_file = _decode_filesystem_path(__file__)\n return os.path.join(os.path.dirname(module_file), \"pystepsrc_schema.json\")\n\n\ndef _fconfig_candidates_generator():\n \"\"\"\n Configuration files candidates generator.\n\n See :py:func:~config_fname for more details.\n \"\"\"\n\n yield os.path.join(os.getcwd(), \"pystepsrc\")\n\n try:\n pystepsrc = os.environ[\"PYSTEPSRC\"]\n except KeyError:\n pass\n else:\n yield pystepsrc\n yield os.path.join(pystepsrc, \"pystepsrc\")\n\n if os.name == \"nt\":\n # Windows environment\n env_variable = \"USERPROFILE\"\n subdir = \"pysteps\"\n else:\n # UNIX like\n env_variable = \"HOME\"\n subdir = \".pysteps\"\n\n try:\n pystepsrc = os.environ[env_variable]\n except KeyError:\n pass\n else:\n yield os.path.join(pystepsrc, subdir, \"pystepsrc\")\n\n module_file = _decode_filesystem_path(__file__)\n yield os.path.join(os.path.dirname(module_file), \"pystepsrc\")\n yield None\n\n\n# Function adapted from matplotlib's *matplotlib_fname* function.\ndef config_fname():\n \"\"\"\n Get the location of the config file.\n\n Looks for pystepsrc file in the following order:\n - $PWD/pystepsrc: Looks for the file in the current directory\n - $PYSTEPSRC: If the system variable $PYSTEPSRC is defined and it points\n to a file, it is used..\n - $PYSTEPSRC/pystepsrc: If $PYSTEPSRC points to a directory, it looks for\n the pystepsrc file inside that directory.\n - $HOME/.pysteps/pystepsrc (unix and Mac OS X) :\n If the system variable $HOME is defined, it looks\n for the configuration file in this path.\n - $USERPROFILE/pysteps/pystepsrc (windows only): It looks for the\n configuration file in the pysteps directory located user's home directory.\n - Lastly, it looks inside the library in pysteps/pystepsrc for a\n system-defined copy.\n \"\"\"\n\n file_name = None\n for file_name in _fconfig_candidates_generator():\n if file_name is not None:\n if os.path.exists(file_name):\n st_mode = os.stat(file_name).st_mode\n if stat.S_ISREG(st_mode) or stat.S_ISFIFO(st_mode):\n return file_name\n\n # Return first candidate that is a file,\n # or last candidate if none is valid\n # (in that case, a warning is raised at startup by `rc_params`).\n\n return file_name\n\n\ndef _decode_filesystem_path(path):\n if not isinstance(path, str):\n return path.decode(sys.getfilesystemencoding())\n else:\n return path\n\n\nclass _DotDictify(dict):\n \"\"\"\n Class used to recursively access dict via attributes as well\n as index access.\n This is introduced to maintain backward compatibility with older pysteps\n configuration parameters implementations.\n\n Code adapted from:\n https://stackoverflow.com/questions/3031219/recursively-access-dict-via-attributes-as-well-as-index-access\n\n Credits: `Curt Hagenlocher`_\n\n .. _`Curt Hagenlocher`: https://stackoverflow.com/users/533/curt-hagenlocher\n \"\"\"\n\n def __setitem__(self, key, value):\n if isinstance(value, dict) and not isinstance(value, _DotDictify):\n value = _DotDictify(value)\n super().__setitem__(key, value)\n\n def __getitem__(self, key):\n value = super().__getitem__(key)\n if isinstance(value, dict) and not isinstance(value, _DotDictify):\n value = _DotDictify(value)\n super().__setitem__(key, value)\n return value\n\n __setattr__, __getattr__ = __setitem__, __getitem__\n\n\nrcparams = dict()\n\n\ndef load_config_file(params_file=None, verbose=False, dryrun=False):\n \"\"\"\n Load the pysteps configuration file. The configuration parameters are available\n as a DotDictify instance in the `pysteps.rcparams` variable.\n\n Parameters\n ----------\n params_file: str\n Path to the parameters file to load. If `params_file=None`, it looks\n for a configuration file in the default locations.\n verbose: bool\n Print debugging information. False by default.\n This flag is overwritten by the silent_import=False in the\n pysteps configuration file.\n dryrun: bool\n If False, perform a dry run that does not update the `pysteps.rcparams`\n attribute.\n\n Returns\n -------\n rcparams: _DotDictify\n Configuration parameters loaded from file.\n \"\"\"\n\n global rcparams\n\n if params_file is None:\n # Load default configuration\n params_file = config_fname()\n\n if params_file is None:\n warnings.warn(\n \"pystepsrc file not found.\" + \"The defaults parameters are left empty\",\n category=ImportWarning,\n )\n\n _rcparams = dict()\n return\n\n with open(params_file, \"r\") as f:\n _rcparams = json.loads(jsmin(f.read()))\n\n if (not _rcparams.get(\"silent_import\", False)) or verbose:\n print(\"Pysteps configuration file found at: \" + params_file + \"\\n\")\n\n with open(_get_config_file_schema(), \"r\") as f:\n schema = json.loads(jsmin(f.read()))\n validator = Draft4Validator(schema)\n\n error_msg = \"Error reading pystepsrc file.\"\n error_count = 0\n for error in validator.iter_errors(_rcparams):\n error_msg += \"\\nError in \" + \"/\".join(list(error.path))\n error_msg += \": \" + error.message\n error_count += 1\n if error_count > 0:\n raise RuntimeError(error_msg)\n\n _rcparams = _DotDictify(_rcparams)\n\n if not dryrun:\n rcparams = _rcparams\n\n return _rcparams\n\n\n# Load default configuration\nload_config_file()\n\n# After the sub-modules are loaded, register the discovered importers plugin.\nio.interface.discover_importers()\npostprocessing.interface.discover_postprocessors()\n" }, { "path": "pysteps/blending/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Methods for blending NWP model(s) with nowcasts.\"\"\"\n\nfrom pysteps.blending.interface import get_method\nfrom .clim import *\nfrom .skill_scores import *\nfrom .utils import *\n" }, { "path": "pysteps/blending/clim.py", "content": "\"\"\"\npysteps.blending.clim\n=====================\n\nModule with methods to read, write and compute past and climatological NWP model\nskill scores. The module stores the average daily skill score for the past t days\nand updates it every day. The resulting average climatological skill score is\nthe skill the NWP model skill regresses to during the blended forecast. If no\nclimatological values are present, the default skill from :cite:`BPS2006` is used.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_default_skill\n save_skill\n calc_clim_skill\n\"\"\"\n\nimport pickle\nfrom pathlib import Path\n\nimport numpy as np\n\n\ndef get_default_skill(n_cascade_levels=6, n_models=1):\n \"\"\"\n Get the default climatological skill values as given in :cite:`BPS2006`.\n Take subset of n_cascade_levels or add entries with small values (1e-4) if\n n_cascade_levels differs from 8.\n\n Parameters\n ----------\n n_cascade_levels: int, optional\n Number of cascade levels. Defaults to 6, see issue #385 on GitHub.\n n_models: int, optional\n Number of NWP models. Defaults to 1.\n\n Returns\n -------\n default_skill: array-like\n Array of shape [model, scale_level] containing the climatological skill\n values.\n\n \"\"\"\n\n default_skill = np.array(\n [0.848, 0.537, 0.237, 0.065, 0.020, 0.0044, 0.0052, 0.0040]\n )\n n_skill = default_skill.shape[0]\n if n_cascade_levels < n_skill:\n default_skill = default_skill[0:n_cascade_levels]\n elif n_cascade_levels > n_skill:\n default_skill = np.append(\n default_skill, np.repeat(1e-4, n_cascade_levels - n_skill)\n )\n return np.resize(default_skill, (n_models, n_cascade_levels))\n\n\ndef save_skill(\n current_skill,\n validtime,\n outdir_path,\n window_length=30,\n **kwargs,\n):\n \"\"\"\n Add the current NWP skill to update today's daily average skill. If the day\n is over, update the list of daily average skill covering a rolling window.\n\n Parameters\n ----------\n current_skill: array-like\n Array of shape [model, scale_level, ...]\n containing the current skill of the different NWP models per cascade\n level.\n validtime: datetime\n Datetime object containing the date and time for which the current\n skill are valid.\n outdir_path: string\n Path to folder where the historical skill are stored. Defaults to\n path_workdir from rcparams.\n window_length: int, optional\n Length of window (in days) of daily skill that should be retained.\n Defaults to 30.\n\n Returns\n -------\n None\n\n \"\"\"\n\n n_cascade_levels = current_skill.shape[1]\n\n # Load skill_today, a dictionary containing {mean_skill, n, last_validtime}\n new_skill_today_file = False\n\n skill_today_file = Path(outdir_path) / \"NWP_skill_today.pkl\"\n if skill_today_file.is_file():\n with open(skill_today_file, \"rb\") as f:\n skill_today = pickle.load(f)\n if skill_today[\"mean_skill\"].shape != current_skill.shape:\n new_skill_today_file = True\n else:\n new_skill_today_file = True\n\n if new_skill_today_file:\n skill_today = {\n \"mean_skill\": np.copy(current_skill),\n \"n\": 0,\n \"last_validtime\": validtime,\n }\n\n # Load the past skill which is an array with dimensions day x model x scale_level\n past_skill_file = Path(outdir_path) / \"NWP_skill_window.npy\"\n past_skill = None\n if past_skill_file.is_file():\n past_skill = np.load(past_skill_file)\n # First check if we have started a new day wrt the last written skill, in which\n # case we should update the daily skill file and reset daily statistics.\n if skill_today[\"last_validtime\"].date() < validtime.date():\n # Append skill to the list of the past X daily averages.\n if (\n past_skill is not None\n and past_skill.shape[2] == n_cascade_levels\n and past_skill.shape[1] == skill_today[\"mean_skill\"].shape[0]\n ):\n past_skill = np.append(past_skill, [skill_today[\"mean_skill\"]], axis=0)\n else:\n past_skill = np.array([skill_today[\"mean_skill\"]])\n\n # Remove oldest if the number of entries exceeds the window length.\n if past_skill.shape[0] > window_length:\n past_skill = np.delete(past_skill, 0, axis=0)\n # FIXME also write out last_validtime.date() in this file?\n # In that case it will need pickling or netcdf.\n # Write out the past skill within the rolling window.\n np.save(past_skill_file, past_skill)\n # Reset statistics for today.\n skill_today[\"n\"] = 0\n skill_today[\"mean_skill\"] = 0\n\n # Reset today's skill if needed and/or compute online average from the\n # current skill using numerically stable algorithm\n skill_today[\"n\"] += 1\n skill_today[\"mean_skill\"] += (\n current_skill - skill_today[\"mean_skill\"]\n ) / skill_today[\"n\"]\n skill_today[\"last_validtime\"] = validtime\n # Make path if path does not exist\n skill_today_file.parent.mkdir(exist_ok=True, parents=True)\n # Open and write to skill file\n with open(skill_today_file, \"wb\") as f:\n pickle.dump(skill_today, f)\n\n return None\n\n\ndef calc_clim_skill(\n outdir_path,\n n_cascade_levels=6,\n n_models=1,\n window_length=30,\n):\n \"\"\"\n Return the climatological skill based on the daily average skill in the\n rolling window. This is done using a geometric mean.\n\n Parameters\n ----------\n n_cascade_levels: int, optional\n Number of cascade levels. Defaults to 6, see issue #385 on GitHub.\n outdir_path: string\n Path to folder where the historical skill are stored. Defaults to\n path_workdir from rcparams.\n n_models: int, optional\n Number of NWP models. Defaults to 1.\n window_length: int, optional\n Length of window (in days) over which to compute the climatological\n skill. Defaults to 30.\n\n Returns\n -------\n climatological_mean_skill: array-like\n Array of shape [model, scale_level, ...] containing the climatological\n (geometric) mean skill.\n\n \"\"\"\n past_skill_file = Path(outdir_path) / \"NWP_skill_window.npy\"\n # past_skill has dimensions date x model x scale_level x ....\n if past_skill_file.is_file():\n past_skill = np.load(past_skill_file)\n else:\n past_skill = np.array(None)\n # check if there is enough data to compute the climatological skill\n if not past_skill.any():\n return get_default_skill(n_cascade_levels, n_models)\n elif past_skill.shape[0] < window_length:\n return get_default_skill(n_cascade_levels, n_models)\n # reduce window if necessary\n else:\n past_skill = past_skill[-window_length:]\n\n # Make sure past_skill cannot be lower than 10e-5\n past_skill = np.where(past_skill < 10e-5, 10e-5, past_skill)\n\n # Calculate climatological skill from the past_skill using the\n # geometric mean.\n geomean_skill = np.exp(np.log(past_skill).mean(axis=0))\n\n # Make sure skill is always a positive value and a finite value\n geomean_skill = np.where(geomean_skill < 10e-5, 10e-5, geomean_skill)\n geomean_skill = np.nan_to_num(\n geomean_skill, copy=True, nan=10e-5, posinf=10e-5, neginf=10e-5\n )\n\n return geomean_skill\n" }, { "path": "pysteps/blending/ens_kalman_filter_methods.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.blending.ens_kalman_filter_methods\n=============================================\nMethods to calculate the ensemble Kalman filter based correction methods for blending\nbetween nowcast and NWP.\nThe core of the method occurs in the EnsembleKalmanFilter class. The specific method\nto use this core class can be selected. Currently, only the implementation of the\nensemble Kalman filter from :cite:`Nerini2019MWR` is available.\n\nAdditional keyword arguments for the ensemble Kalman filter are:\n\nn_tapering: int, default=0\n Tapering parameter controlling the number of covariance pairs (i, i ± n_tapering)\n retained in the covariance matrix. With n_tapering=0, only the variances\n (main diagonal) of the principal components are kept.\nnon_precip_mask: bool, (True)\n Flag to specify whether the computation should be truncated on grid boxes where at\n least a minimum number (configurable) of ensemble members forecast precipitation.\n Defaults to True.\nn_ens_prec: int, (1)\n Minimum number of ensemble members that forecast precipitation for the above\n mentioned mask. Defaults to 1.\nlien_criterion: bool, (True)\n Flag to specify whether Lien criterion (Lien et al., 2013) should be applied for\n the computation of the update step within the ensemble Kalman filter. Defaults to\n True.\nn_lien: int, (n_ens_members/2)\n Minimum number of ensemble members that forecast precipitation for the Lien\n criterion. Defaults to half of the ensemble members.\nprob_matching: str, {'iterative','post_forecast','none'}\n Specify the probability matching method that should be applied as an additional\n processing step of the forecast computation. Defaults to 'iterative'.\ninflation_factor_bg: float, (1.0)\n Inflation factor of the background (NWC) covariance matrix. This factor increases\n the covariances of the background ensemble and, thus, supports a faster convergence\n towards the observation ensemble (NWP). Defaults to 1.0.\ninflation_factor_obs: float, (1.0)\n Inflation factor of the observation (NWP) covariance matrix. This factor increases\n the covariances of the observation ensemble (NWP) and, thus, supports a slower\n convergence towards the observation ensemble. Defaults to 1.0.\noffset_bg: float, (0.0)\n Offset of the background (NWC) covariance matrix. This offset supports a faster\n convergence towards the observation ensemble (NWP) by linearly increasing all\n elements of the background covariance matrix. Defaults to 0.0.\noffset_obs: float, (0.0)\n Offset of the observation (NWP) covariance matrix. This offset supports a slower\n convergence towards the observation ensemble (NWP) by linearly incerasing all\n elements of the observation covariance matrix. Defaults to 0.0.\nnwp_hres_eff: float\n Effective horizontal resolution of the utilized NWP model.\nsampling_prob_source: str, {'ensemble','explained_var'}\n Computation method of the sampling probability for the probability matching.\n 'ensemble' computes this probability as the ratio between the ensemble differences\n of analysis_ensemble - background_ensemble and observation_ensemble - background_ensemble.\n 'explained_var' uses the sum of the Kalman gain weighted by the explained variance ratio.\nuse_accum_sampling_prob: bool, (False)\n Flag to specify whether the current sampling probability should be used for the\n probability matching or a probability integrated over the previous forecast time.\n Defaults to True.\nensure_full_nwp_weight: bool, (True)\n Flag to specify whether the end of the combination should be represent the pure NWP\n forecast. Defaults to True.\n\"\"\"\n\nimport numpy as np\n\nfrom pysteps import utils\nfrom pysteps.postprocessing import probmatching\n\ntry:\n import dask\n\n DASK_IMPORTED = True\nexcept ImportError:\n DASK_IMPORTED = False\n\n\nclass EnsembleKalmanFilter:\n\n def __init__(self, config, params):\n\n self._config = config\n\n # Check for combination kwargs in params\n self.__n_tapering = params.combination_kwargs.get(\"n_tapering\", 0)\n self.__non_precip_mask = params.combination_kwargs.get(\"non_precip_mask\", True)\n self.__n_ens_prec = params.combination_kwargs.get(\"n_ens_prec\", 1)\n self.__lien_criterion = params.combination_kwargs.get(\"lien_criterion\", True)\n self.__n_lien = params.combination_kwargs.get(\n \"n_lien\", self._config.n_ens_members // 2\n )\n\n print(\"Initialize ensemble Kalman filter\")\n print(\"=================================\")\n print(\"\")\n\n print(f\"Non-tapered diagonals: {self.__n_tapering}\")\n print(f\"Non precip mask: {self.__non_precip_mask}\")\n print(f\"No. ens mems with precipitation: {self.__n_ens_prec}\")\n print(f\"Lien Criterion: {self.__lien_criterion}\")\n print(f\"No. ens mems with precip (Lien): {self.__n_lien}\")\n print(\"\")\n\n def update(\n self,\n background_ensemble: np.ndarray,\n observation_ensemble: np.ndarray,\n inflation_factor_bg: float,\n inflation_factor_obs: float,\n offset_bg: float,\n offset_obs: float,\n background_ensemble_valid_lien: np.ndarray | None = None,\n observation_ensemble_valid_lien: np.ndarray | None = None,\n ):\n \"\"\"\n Compute the ensemble Kalman filter update step.\n\n Parameters\n ----------\n background_ensemble: np.ndarray\n Two-dimensional array of shape (n_ens, n_pc) containing the background\n ensemble that corresponds to the Nowcast ensemble forecast.\n observation_ensemble: np.ndarray\n Two-dimensional array of shape (n_ens, n_pc) containing the observations\n that correspond to the NWP ensemble forecast.\n inflation_factor_bg: float\n Inflation factor of the background ensemble covariance matrix.\n inflation_factor_obs: float\n Inflation factor of the observation covariance matrix.\n offset_bg: float\n Offset of the background ensemble covariance matrix.\n offset_obs: float\n Offset of the observation covariance matrix.\n\n Other Parameters\n ----------------\n background_ensemble_valid_lien: np.ndarray\n Two-dimensional array of shape (n_ens, n_pc) containing the background\n ensemble that consists only of grid boxes at which the Lien criterion is\n satisfied.\n observation_ensemble_valid_lien: np.ndarray\n Two-dimensional array of shape (n_ens, n_pc) containing the observations\n that consists only of grid boxes at which the Lien criterion is satisfied.\n\n Returns\n -------\n analysis_ensemble: np.ndarray\n Two-dimensional array of shape (n_ens, n_pc) containing the updated\n analysis matrix.\n \"\"\"\n\n # If the masked background and observation arrays are given, compute the\n # covariance matrices P and R only on these values.\n if (\n background_ensemble_valid_lien is not None\n and observation_ensemble_valid_lien is not None\n ):\n # Equation 13 in Nerini et al. (2019)\n P = self.get_covariance_matrix(\n background_ensemble_valid_lien,\n inflation_factor=inflation_factor_bg,\n offset=offset_bg,\n )\n # Equation 14 in Nerini et al. (2019)\n R = self.get_covariance_matrix(\n observation_ensemble_valid_lien,\n inflation_factor=inflation_factor_obs,\n offset=offset_obs,\n )\n # Otherwise use the complete arrays.\n else:\n # Equation 13 in Nerini et al. (2019)\n P = self.get_covariance_matrix(\n background_ensemble,\n inflation_factor=inflation_factor_bg,\n offset=offset_bg,\n )\n # Equation 14 in Nerini et al. (2019)\n R = self.get_covariance_matrix(\n observation_ensemble,\n inflation_factor=inflation_factor_obs,\n offset=offset_obs,\n )\n\n # Estimate the Kalman gain (eq. 15 in Nerini et al., 2019)\n self.K = np.dot(P, np.linalg.inv(P + R))\n\n # Update the background ensemble (eq. 16 in Nerini et al., 2019)\n analysis_ensemble = background_ensemble.T + np.dot(\n self.K, (observation_ensemble - background_ensemble).T\n )\n\n return analysis_ensemble\n\n def get_covariance_matrix(\n self, forecast_array: np.ndarray, inflation_factor: float, offset: float\n ):\n \"\"\"\n Compute the covariance matrix of a given ensemble forecast along the grid boxes\n or principal components as it is done by Eq. 13 and 14 in Nerini et al., 2019.\n\n Parameters\n ----------\n forecast_array: np.ndarray\n Two-dimensional array of shape (n_ens, n_pc) containing an ensemble\n forecast of one lead time.\n inflation_factor: float\n Factor to increase the covariance and therefore the ensemble spread.\n offset: float\n Offset to shift the covariance.\n\n Returns\n -------\n Cov: np.ndarray\n Two-dimensional array of shape (n_pc, n_pc) containg the covariance matrix\n of the given ensemble forecast.\n \"\"\"\n\n # Compute the ensemble mean\n ensemble_mean = np.mean(forecast_array, axis=0)\n # Center the ensemble forecast and multiply with the given inflation factor\n centered_ensemble = (forecast_array - ensemble_mean) * inflation_factor\n # Compute the covariance matrix and add the respective offset and filter\n # unwanted diagonals, respectively.\n Cov = (\n 1\n / (forecast_array.shape[0] - 1)\n * np.dot(centered_ensemble.T, centered_ensemble)\n + offset\n ) * self.get_tapering(forecast_array.shape[1])\n\n return Cov\n\n def get_tapering(self, n: int):\n \"\"\"\n Create a window function to clip unwanted diagonals of the covariance matrix.\n\n Parameters\n ----------\n n: integer\n Number of grid boxes/principal components of the ensemble forecast for that\n the covariance matrix is computed.\n\n Returns\n -------\n window_function: np.ndarray\n Two-dimensional array of shape (n_pc, n_pc) containing the window function\n to filter unwanted diagonals of the covariance matrix.\n \"\"\"\n\n # Create an n-dimensional I-matrix as basis of the window function\n window_function = np.eye(n)\n # Get the weightings of a hanning window function with respect to the number of\n # diagonals that on want to keep\n hanning_values = np.hanning(self.__n_tapering * 2 + 1)[\n (self.__n_tapering + 1) :\n ]\n\n # Add the respective values to I\n for d in range(self.__n_tapering):\n window_function += np.diag(np.ones(n - d - 1) * hanning_values[d], k=d + 1)\n window_function += np.diag(np.ones(n - d - 1) * hanning_values[d], k=-d - 1)\n\n return window_function\n\n def get_precipitation_mask(self, forecast_array: np.ndarray):\n \"\"\"\n Create the set of grid boxes where at least a minimum number (configurable)\n of ensemble members forecast precipitation.\n\n Parameters\n ----------\n forecast_array: np.ndarray\n Two-dimensional array of shape (n_ens, n_grid) containg the ensemble\n forecast for one lead time.\n\n Returns\n -------\n idx_prec: np.ndarray\n One-dimensional array of shape (n_grid) that is set to True if the minimum\n number of ensemble members predict precipitation.\n \"\"\"\n\n # Check the number of ensemble members forecast precipitation at each grid box.\n forecast_array_sum = np.sum(\n forecast_array >= self._config.precip_threshold, axis=0\n )\n\n # If the masking of areas without precipitation is requested, mask grid boxes\n # where less ensemble members predict precipitation than the set limit n_ens_prec.\n if self.__non_precip_mask == True:\n idx_prec = forecast_array_sum >= self.__n_ens_prec\n # Else, set all to True.\n else:\n idx_prec = np.ones_like(forecast_array_sum).astype(bool)\n\n return idx_prec\n\n def get_lien_criterion(self, nwc_ensemble: np.ndarray, nwp_ensemble: np.ndarray):\n \"\"\"\n Create the set of grid boxes where the Lien criterion is satisfied (Lien et\n al., 2013) and thus, at least half (configurable) of the ensemble members of\n each forecast (Nowcast and NWP) predict precipitation.\n\n Parameters\n ----------\n nwc_ensemble: np.ndarray\n Two-dimensional array (n_ens, n_grid) containing the nowcast ensemble\n forecast for one lead time.\n nwp_ensemble: np.ndarray\n Two-dimensional array (n_ens, n_grid) containg the NWP ensemble forecast\n for one lead time.\n\n Returns\n -------\n idx_lien: np.ndarray\n One-dimensional array of shape (n_grid) that is set to True at grid boxes\n where the Lien criterion is satisfied.\n \"\"\"\n\n # Check the number of ensemble members forecast precipitation at each grid box.\n nwc_ensemble_sum = np.sum(nwc_ensemble >= self._config.precip_threshold, axis=0)\n nwp_ensemble_sum = np.sum(nwp_ensemble >= self._config.precip_threshold, axis=0)\n\n # If the masking of areas without precipitation is requested, mask grid boxes\n # where less ensemble members predict precipitation than the set limit of n_ens_fc_prec.\n if self.__lien_criterion:\n idx_lien = np.logical_and(\n nwc_ensemble_sum >= self.__n_lien, nwp_ensemble_sum >= self.__n_lien\n )\n # Else, set all to True.\n else:\n idx_lien = np.ones_like(nwc_ensemble_sum).astype(bool)\n\n return idx_lien\n\n def get_weighting_for_probability_matching(\n self,\n background_ensemble: np.ndarray,\n analysis_ensemble: np.ndarray,\n observation_ensemble: np.ndarray,\n ):\n \"\"\"\n Compute the weighting between background (nowcast) and observation (NWP) ensemble\n that results to the updated analysis ensemble in physical space for an optional\n probability matching. See equation 17 in Nerini et al. (2019).\n\n Parameters\n ----------\n background_ensemble: np.ndarray\n Two-dimensional array of shape (n_ens, n_grid) containing the background\n ensemble (Original nowcast).\n analysis_ensemble: np.ndarray\n Two-dimensional array of shape (n_ens, n_grid) containing the updated\n analysis ensemble.\n observation_ensemble: np.ndarray\n Two-dimensional array of shape (n_ens, n_grid) containing the observation\n ensemble (NWP).\n\n Returns\n -------\n prob_matching_weight: float\n A weighting of which elements of the input ensemble contributed to the\n updated analysis ensemble with respect to observation_ensemble. Therefore, 0\n means that the contribution comes entirely from the background_ensemble (the\n original nowcast). 1 means that the contribution comes entirely from the\n observation_ensemble (the NWP forecast).\n \"\"\"\n\n # Compute the sum of differences between analysis_ensemble and background_ensemble\n # as well as observation_ensemble and background_ensemble along the grid boxes.\n w1 = np.sum(analysis_ensemble - background_ensemble, axis=0)\n w2 = np.sum(observation_ensemble - background_ensemble, axis=0)\n\n # Check for infinitesimal differences between w1 and w2 as well as 0.\n w_close = np.isclose(w1, w2)\n w_zero = np.logical_and(w_close, np.isclose(w2, 0.0))\n\n # Compute the fraction of w1 and w2 and set values on grid boxes marked by\n # w_close or w_zero to 1 and 0, respectively.\n prob_matching_weight = np.zeros_like(w1)\n prob_matching_weight[~w_zero] = w1[~w_zero] / w2[~w_zero]\n prob_matching_weight[w_close] = 1.0\n\n # Even now we have at some grid boxes weights outside the range between 0\n # and 1. Therefore, we leave them out in the calculation of the averaged\n # weighting.\n valid_values = np.logical_and(\n prob_matching_weight >= 0.0, prob_matching_weight <= 1.0\n )\n prob_matching_weight = np.nanmean(prob_matching_weight[valid_values])\n\n # If there is no finite prob_matching_weight, we are switching to the NWP\n if not np.isfinite(prob_matching_weight):\n prob_matching_weight = 1.0\n\n return prob_matching_weight\n\n\nclass MaskedEnKF(EnsembleKalmanFilter):\n\n def __init__(self, config, params):\n\n EnsembleKalmanFilter.__init__(self, config, params)\n self.__params = params\n\n # Read arguments from combination kwargs or set standard values if kwargs not\n # given\n self.__iterative_prob_matching = self.__params.combination_kwargs.get(\n \"iterative_prob_matching\", True\n )\n self.__inflation_factor_bg = self.__params.combination_kwargs.get(\n \"inflation_factor_bg\", 1.0\n )\n self.__inflation_factor_obs = self.__params.combination_kwargs.get(\n \"inflation_factor_obs\", 1.0\n )\n self.__offset_bg = self.__params.combination_kwargs.get(\"offset_bg\", 0.0)\n self.__offset_obs = self.__params.combination_kwargs.get(\"offset_obs\", 0.0)\n self.__sampling_prob_source = self.__params.combination_kwargs.get(\n \"sampling_prob_source\", \"ensemble\"\n )\n self.__use_accum_sampling_prob = self.__params.combination_kwargs.get(\n \"use_accum_sampling_prob\", False\n )\n self.__ensure_full_nwp_weight = self.__params.combination_kwargs.get(\n \"ensure_full_nwp_weight\", True\n )\n\n self.__sampling_probability = 0.0\n self.__accumulated_sampling_prob = 0.0\n self.__degradation_timestep = 0.2\n self.__inflation_factor_obs_tmp = 1.0\n\n print(\"Initialize masked ensemble Kalman filter\")\n print(\"========================================\")\n print(\"\")\n\n print(f\"Iterative probability matching: {self.__iterative_prob_matching}\")\n print(f\"Background inflation factor: {self.__inflation_factor_bg}\")\n print(f\"Observation inflation factor: {self.__inflation_factor_obs}\")\n print(f\"Background offset: {self.__offset_bg}\")\n print(f\"Observation offset: {self.__offset_obs}\")\n print(f\"Sampling probability source: {self.__sampling_prob_source}\")\n print(f\"Use accum. sampling probability: {self.__use_accum_sampling_prob}\")\n print(f\"Ensure full NWP weight: {self.__ensure_full_nwp_weight}\")\n\n return\n\n def correct_step(\n self, background_ensemble, observation_ensemble, resampled_forecast\n ):\n \"\"\"\n Prepare input ensembles of Nowcast and NWP for the ensemble Kalman filter\n update step.\n\n Parameters\n ----------\n background_ensemble: np.ndarray\n Three-dimensional array of shape (n_ens, m, n) containing the background\n (Nowcast) ensemble forecast for one timestep. This data is used as background\n ensemble in the ensemble Kalman filter.\n observation_ensemble: np.ndarray\n Three-dimensional array of shape (n_ens, m, n) containing the observation\n (NWP) ensemble forecast for one timestep. This data is used as observation\n ensemble in the ensemble Kalman filter.\n resampled_forecast: np.ndarray\n Three-dimensional array of shape (n_ens, m, n) containing the resampled (post-\n processed) ensemble forecast for one timestep.\n\n Returns\n -------\n analysis_ensemble: np.ndarray\n Three-dimensional array of shape (n_ens, m, n) containing the Nowcast\n ensemble forecast corrected by NWP ensemble data.\n resampled_forecast: np.ndarray\n Three-dimensional array of shape (n_ens, m, n) containing the resampled (post-\n processed) ensemble forecast for one timestep.\n \"\"\"\n\n # Get indices with predicted precipitation.\n idx_prec = np.logical_or(\n self.get_precipitation_mask(background_ensemble),\n self.get_precipitation_mask(observation_ensemble),\n )\n\n # Get indices with satisfied Lien criterion and truncate the array onto the\n # precipitation area.\n idx_lien = self.get_lien_criterion(background_ensemble, observation_ensemble)[\n idx_prec\n ]\n\n # Stack both ensemble forecasts and truncate them onto the precipitation area.\n forecast_ens_stacked = np.vstack((background_ensemble, observation_ensemble))[\n :, idx_prec\n ]\n\n # Remove possible non-finite values\n forecast_ens_stacked[~np.isfinite(forecast_ens_stacked)] = (\n self._config.norain_threshold\n )\n\n # Check whether there are more rainy grid boxes as two times the ensemble\n # members\n if np.sum(idx_prec) <= forecast_ens_stacked.shape[0]:\n # If this is the case, the NWP ensemble forecast is returned\n return observation_ensemble\n\n # Transform both ensemble forecasts into the PC space.\n kwargs = {\"n_components\": forecast_ens_stacked.shape[0], \"svd_solver\": \"full\"}\n forecast_ens_stacked_pc, pca_params = utils.pca.pca_transform(\n forecast_ens=forecast_ens_stacked, get_params=True, **kwargs\n )\n\n # And do that transformation also for the Lien criterion masked values.\n forecast_ens_lien_pc = utils.pca.pca_transform(\n forecast_ens=forecast_ens_stacked,\n mask=idx_lien,\n pca_params=pca_params,\n **kwargs,\n )\n\n if not np.isclose(self.__accumulated_sampling_prob, 1.0, rtol=1e-2):\n self.__inflation_factor_obs_tmp = (\n self.__inflation_factor_obs\n - self.__accumulated_sampling_prob * (self.__inflation_factor_obs - 1.0)\n )\n else:\n self.__inflation_factor_obs_tmp = np.cos(self.__degradation_timestep)\n self.__degradation_timestep += 0.2\n\n # Get the updated background ensemble (Nowcast ensemble) in PC space.\n analysis_ensemble_pc = self.update(\n background_ensemble=forecast_ens_stacked_pc[: background_ensemble.shape[0]],\n observation_ensemble=forecast_ens_stacked_pc[\n background_ensemble.shape[0] :\n ],\n inflation_factor_bg=self.__inflation_factor_bg,\n inflation_factor_obs=self.__inflation_factor_obs_tmp,\n offset_bg=self.__offset_bg,\n offset_obs=self.__offset_obs,\n background_ensemble_valid_lien=forecast_ens_lien_pc[\n : background_ensemble.shape[0]\n ],\n observation_ensemble_valid_lien=forecast_ens_lien_pc[\n background_ensemble.shape[0] :\n ],\n )\n\n # Transform the analysis ensemble back into physical space.\n analysis_ensemble = utils.pca.pca_backtransform(\n forecast_ens_pc=analysis_ensemble_pc.T, pca_params=pca_params\n )\n\n # Get the sampling probability either based on the ensembles...\n if self.__sampling_prob_source == \"ensemble\":\n sampling_probability_single_step = (\n self.get_weighting_for_probability_matching(\n background_ensemble=forecast_ens_stacked[\n : background_ensemble.shape[0]\n ][:, idx_lien],\n analysis_ensemble=analysis_ensemble[:, idx_lien],\n observation_ensemble=forecast_ens_stacked[\n background_ensemble.shape[0] :\n ][:, idx_lien],\n )\n )\n # ...or based on the explained variance weighted Kalman gain.\n elif self.__sampling_prob_source == \"explained_var\":\n sampling_probability_single_step = np.sum(\n np.diag(self.K) * pca_params[\"explained_variance\"]\n )\n else:\n raise ValueError(\n f\"Sampling probability source should be either 'ensemble' or 'explained_var', but is {self.__sampling_prob_source}!\"\n )\n\n # Adjust sampling probability when the accumulation flag is set\n if self.__use_accum_sampling_prob == True:\n self.__sampling_probability = (\n 1 - sampling_probability_single_step\n ) * self.__sampling_probability + sampling_probability_single_step\n else:\n self.__sampling_probability = sampling_probability_single_step\n\n # The accumulation is divided for cases one would not use the accumulated\n # sampling probability for the probability matching, but still wants to have\n # the pure NWP forecast at the end of a combined forecast.\n if self.__ensure_full_nwp_weight == True:\n self.__accumulated_sampling_prob = (\n 1 - sampling_probability_single_step\n ) * self.__accumulated_sampling_prob + sampling_probability_single_step\n\n print(f\"Sampling probability: {self.__sampling_probability:1.4f}\")\n\n # Apply probability matching to the analysis ensemble\n if self.__iterative_prob_matching:\n\n def worker(j):\n # Get the combined distribution based on the input weight\n resampled_forecast[j] = probmatching.resample_distributions(\n first_array=background_ensemble[j],\n second_array=observation_ensemble[j],\n probability_first_array=1 - self.__sampling_probability,\n ).reshape(self.__params.len_y, self.__params.len_x)\n\n dask_worker_collection = []\n\n if DASK_IMPORTED and self._config.n_ens_members > 1:\n for j in range(self._config.n_ens_members):\n dask_worker_collection.append(dask.delayed(worker)(j))\n dask.compute(\n *dask_worker_collection,\n num_workers=self.__params.num_ensemble_workers,\n )\n else:\n for j in range(self._config.n_ens_members):\n worker(j)\n\n dask_worker_collection = None\n\n # Set analysis ensemble into the Nowcast ensemble\n background_ensemble[:, idx_prec] = analysis_ensemble\n\n return background_ensemble, resampled_forecast\n\n def get_inflation_factor_obs(self):\n \"\"\"\n Helper function for ensuring the full NWP weight at the end of a combined\n forecast. If an accumulated sampling probability of 1 is reached, the\n observation inflation factor is reduced to 0 by a cosine function.\n \"\"\"\n\n return self.__inflation_factor_obs_tmp\n" }, { "path": "pysteps/blending/interface.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.blending.interface\n==========================\nInterface for the blending module. It returns a callable function for computing\nblended nowcasts with NWP models.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nfrom functools import partial\n\nfrom pysteps.blending import linear_blending\nfrom pysteps.blending import steps\nfrom pysteps.blending import pca_ens_kalman_filter\n\n_blending_methods = dict()\n_blending_methods[\"linear_blending\"] = linear_blending.forecast\n_blending_methods[\"salient_blending\"] = partial(linear_blending.forecast, saliency=True)\n_blending_methods[\"steps\"] = steps.forecast\n_blending_methods[\"pca_enkf\"] = pca_ens_kalman_filter.forecast\n\n\ndef get_method(name):\n \"\"\"\n Return a callable function for computing nowcasts blending into an NWP\n forecast.\n\n Implemented methods:\n\n +------------------+------------------------------------------------------+\n | Name | Description |\n +==================+======================================================+\n | linear_blending | the linear blending of a nowcast method with other |\n | | data (e.g. NWP data). |\n +------------------+------------------------------------------------------+\n | salient_blending | the salient blending of a nowcast method with other |\n | | data (e.g. NWP data) described in :cite:`Hwang2015`. |\n | | The blending is based on intensities and forecast |\n | | times. The blended product preserves pixel |\n | | intensities with time if they are strong enough based|\n | | on their ranked salience. |\n +------------------+------------------------------------------------------+\n | steps | the STEPS stochastic nowcasting blending method |\n | | described in :cite:`Seed2003`, :cite:`BPS2006` and |\n | | :cite:`SPN2013`. The blending weights approach |\n | | currently follows :cite:`BPS2006`. |\n +------------------+------------------------------------------------------+\n | pca_enkf | the reduced-space EnKF combination method described |\n | | in :cite:`Nerini2019MWR`. |\n +------------------+------------------------------------------------------+\n \"\"\"\n if isinstance(name, str):\n name = name.lower()\n else:\n raise TypeError(\n \"Only strings supported for the method's names.\\n\"\n + \"Available names:\"\n + str(list(_blending_methods.keys()))\n ) from None\n\n try:\n return _blending_methods[name]\n except KeyError:\n raise ValueError(\n f\"Unknown blending method {name}.\"\n \"The available methods are: \"\n f\"{*list(_blending_methods.keys()),}\"\n ) from None\n" }, { "path": "pysteps/blending/linear_blending.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.nowcasts.linear_blending\n================================\n\nLinear blending method to blend (ensemble) nowcasts and (ensemble) Numerical\nWeather Prediction (NWP) forecasts. The method uses a predefined start and end\ntime for the blending. Before this start time, the resulting blended forecasts only\nconsists of the nowcast(s). In between the start and end time, the nowcast(s)\nweight decreases and NWP forecasts weight increases linearly from 1(0) to\n0(1). After the end time, the blended forecast entirely consists of the NWP\nforecasts. The saliency-based blending method also takes into account the pixel\nintensities and preserves them if they are strong enough based on their ranked salience.\n\nImplementation of the linear blending and saliency-based blending between nowcast and NWP data.\n\n.. autosummary::\n :toctree: ../generated/\n\n forecast\n\"\"\"\n\nimport numpy as np\nfrom pysteps import nowcasts\nfrom pysteps.utils import conversion\nfrom scipy.stats import rankdata\n\n\ndef forecast(\n precip,\n precip_metadata,\n velocity,\n timesteps,\n timestep,\n nowcast_method,\n precip_nwp=None,\n precip_nwp_metadata=None,\n start_blending=120,\n end_blending=240,\n fill_nwp=True,\n saliency=False,\n nowcast_kwargs=None,\n):\n \"\"\"Generate a forecast by linearly or saliency-based blending of nowcasts with NWP data\n\n Parameters\n ----------\n precip: array_like\n Array containing the input precipitation field(s) ordered by timestamp\n from oldest to newest. The time steps between the inputs are assumed\n to be regular.\n precip_metadata: dict\n Metadata dictionary containing (at least) the transform, unit and threshold\n attributes as described in the documentation of :py:mod:`pysteps.io.importers`.\n velocity; array_like\n Array of shape (2, m, n) containing the x- and y-components of the advection\n field. The velocities are assumed to represent one time step between the\n inputs. All values are required to be finite.\n timesteps: int\n Number of time steps to forecast.\n timestep: int or float\n The time difference (in minutes) between consecutive forecast fields.\n nowcast_method: str\n Name of the nowcasting method. See :py:mod:`pysteps.nowcasts.interface`\n for the list of available methods.\n precip_nwp: array_like or NoneType, optional\n Array of shape (timesteps, m, n) in the case of no ensemble or\n of shape (n_ens_members, timesteps, m, n) in the case of an ensemble\n containing the NWP precipitation fields ordered by timestamp from oldest\n to newest. The time steps between the inputs are assumed to be regular\n (and identical to the time step between the nowcasts). If no NWP\n data is given the value of precip_nwp is None and no blending will be performed.\n precip_nwp_metadata: dict or NoneType, optional\n NWP metadata dictionary containing (at least) the transform, unit and threshold\n attributes as described in the documentation of :py:mod:`pysteps.io.importers`.\n start_blending: int, optional\n Time stamp (in minutes) after which the blending should start. Before this\n only the nowcast data is used.\n end_blending: int, optional\n Time stamp (in minutes) after which the blending should end. Between\n start_blending and end_blending the nowcasts and NWP data are linearly\n merged with each other. After end_blending only the NWP data is used.\n fill_nwp: bool, optional\n Standard value is True. If True, the NWP data will be used to fill in the\n no data mask of the nowcast.\n saliency: bool, optional\n Default value is False. If True, saliency will be used for blending. The blending\n is based on intensities and forecast times as described in :cite:`Hwang2015`. The blended\n product preserves pixel intensities with time if they are strong enough based on their ranked\n salience.\n nowcast_kwargs: dict, optional\n Dictionary containing keyword arguments for the nowcast method.\n\n\n Returns\n -------\n precip_blended: ndarray\n Array of shape (timesteps, m, n) in the case of no ensemble or\n of shape (n_ens_members, timesteps, m, n) in the case of an ensemble\n containing the precipation forecast generated by linearly blending\n the nowcasts and the NWP data. n_ens_members equals the maximum no. of\n ensemble members in either the nowcast or nwp model(s).\n \"\"\"\n\n if nowcast_kwargs is None:\n nowcast_kwargs = dict()\n\n # Ensure that only the most recent precip timestep is used\n if len(precip.shape) == 3:\n precip = precip[-1, :, :]\n\n # First calculate the number of needed timesteps (up to end_blending) for the nowcast\n # to ensure that the nowcast calculation time is limited.\n timesteps_nowcast = int(end_blending / timestep)\n\n nowcast_method_func = nowcasts.get_method(nowcast_method)\n\n # Check if NWP data is given as input\n if precip_nwp is not None:\n # Calculate the nowcast\n precip_nowcast = nowcast_method_func(\n precip,\n velocity,\n timesteps_nowcast,\n **nowcast_kwargs,\n )\n\n # Make sure that precip_nowcast and precip_nwp are in mm/h\n precip_nowcast, _ = conversion.to_rainrate(\n precip_nowcast, metadata=precip_metadata\n )\n precip_nwp, _ = conversion.to_rainrate(precip_nwp, metadata=precip_nwp_metadata)\n\n if len(precip_nowcast.shape) == 4:\n n_ens_members_nowcast = precip_nowcast.shape[0]\n if n_ens_members_nowcast == 1:\n precip_nowcast = np.squeeze(precip_nowcast)\n else:\n n_ens_members_nowcast = 1\n\n if len(precip_nwp.shape) == 4:\n # Ensure precip_nwp has t = n_timesteps\n precip_nwp = precip_nwp[:, 0:timesteps, :, :]\n # Set the number of ensemble members\n n_ens_members_nwp = precip_nwp.shape[0]\n if n_ens_members_nwp == 1:\n precip_nwp = np.squeeze(precip_nwp)\n\n else:\n # Ensure precip_nwp has t = n_timesteps\n precip_nwp = precip_nwp[0:timesteps, :, :]\n # Set the number of ensemble members\n n_ens_members_nwp = 1\n\n # Now, repeat the nowcast ensemble members or the nwp models/members until\n # it has the same amount of members as n_ens_members_max. For instance, if\n # you have 10 ensemble nowcasts members and 3 NWP members, the output will\n # be an ensemble of 10 members. Hence, the three NWP members are blended\n # with the first three members of the nowcast (member one with member one,\n # two with two, etc.), subsequently, the same NWP members are blended with\n # the next three members (NWP member one with member 4, NWP member 2 with\n # member 5, etc.), until 10 is reached.\n n_ens_members_max = max(n_ens_members_nowcast, n_ens_members_nwp)\n n_ens_members_min = min(n_ens_members_nowcast, n_ens_members_nwp)\n\n if n_ens_members_min != n_ens_members_max:\n if n_ens_members_nwp == 1:\n precip_nwp = np.repeat(\n precip_nwp[np.newaxis, :, :], n_ens_members_max, axis=0\n )\n elif n_ens_members_nowcast == 1:\n precip_nowcast = np.repeat(\n precip_nowcast[np.newaxis, :, :], n_ens_members_max, axis=0\n )\n else:\n repeats = [\n (n_ens_members_max + i) // n_ens_members_min\n for i in range(n_ens_members_min)\n ]\n\n if n_ens_members_nwp == n_ens_members_min:\n precip_nwp = np.repeat(precip_nwp, repeats, axis=0)\n elif n_ens_members_nowcast == n_ens_members_min:\n precip_nowcast = np.repeat(precip_nowcast, repeats, axis=0)\n\n # Check if dimensions are correct\n assert (\n precip_nwp.shape[-2:] == precip_nowcast.shape[-2:]\n ), \"The x and y dimensions of precip_nowcast and precip_nwp need to be identical: dimension of precip_nwp = {} and dimension of precip_nowcast = {}\".format(\n precip_nwp.shape[-2:], precip_nowcast.shape[-2:]\n )\n\n # Ensure we are not working with nans in the bleding.\n # Check if the NWP data contains any nans. If so, fill them with 0.0.\n precip_nwp = np.nan_to_num(precip_nwp, nan=0.0)\n\n # Fill nans in precip_nowcast\n nan_mask = np.isnan(precip_nowcast)\n if fill_nwp:\n if len(precip_nwp.shape) == 4:\n precip_nowcast[nan_mask] = precip_nwp[:, 0:timesteps_nowcast, :, :][\n nan_mask\n ]\n else:\n precip_nowcast[nan_mask] = precip_nwp[0:timesteps_nowcast, :, :][\n nan_mask\n ]\n else:\n precip_nowcast[nan_mask] = 0.0\n\n # Initialise output\n precip_blended = np.zeros_like(precip_nwp)\n\n # Calculate the weights\n for i in range(timesteps):\n # Calculate what time we are at\n t = (i + 1) * timestep\n\n if n_ens_members_max == 1:\n ref_dim = 0\n else:\n ref_dim = 1\n\n # apply blending\n # compute the slice indices\n slc_id = _get_slice(precip_blended.ndim, ref_dim, i)\n\n # Calculate the weight with a linear relation (weight_nwp at start_blending = 0.0)\n # and (weight_nwp at end_blending = 1.0)\n weight_nwp = (t - start_blending) / (end_blending - start_blending)\n\n # Set weights at times before start_blending and after end_blending\n if weight_nwp <= 0.0:\n weight_nwp = 0.0\n precip_blended[slc_id] = precip_nowcast[slc_id]\n\n elif weight_nwp >= 1.0:\n weight_nwp = 1.0\n precip_blended[slc_id] = precip_nwp[slc_id]\n\n else:\n # Calculate weight_nowcast\n weight_nowcast = 1.0 - weight_nwp\n\n # Calculate output by combining precip_nwp and precip_nowcast,\n # while distinguishing between ensemble and non-ensemble methods\n if saliency:\n ranked_salience = _get_ranked_salience(\n precip_nowcast[slc_id], precip_nwp[slc_id]\n )\n ws = _get_ws(weight_nowcast, ranked_salience)\n precip_blended[slc_id] = (\n ws * precip_nowcast[slc_id] + (1 - ws) * precip_nwp[slc_id]\n )\n\n else:\n precip_blended[slc_id] = (\n weight_nwp * precip_nwp[slc_id]\n + weight_nowcast * precip_nowcast[slc_id]\n )\n\n else:\n # Calculate the nowcast\n precip_nowcast = nowcast_method_func(\n precip,\n velocity,\n timesteps,\n **nowcast_kwargs,\n )\n\n # Make sure that precip_nowcast and precip_nwp are in mm/h\n precip_nowcast, _ = conversion.to_rainrate(\n precip_nowcast, metadata=precip_metadata\n )\n\n # If no NWP data is given, the blended field is simply equal to the nowcast field\n precip_blended = precip_nowcast\n\n return precip_blended\n\n\ndef _get_slice(n_dims, ref_dim, ref_id):\n \"\"\"source: https://stackoverflow.com/a/24399139/4222370\"\"\"\n slc = [slice(None)] * n_dims\n slc[ref_dim] = ref_id\n return tuple(slc)\n\n\ndef _get_ranked_salience(precip_nowcast, precip_nwp):\n \"\"\"Calculate ranked salience, which show how close the pixel is to the maximum intensity difference [r(x,y)=1]\n or the minimum intensity difference [r(x,y)=0]\n\n Parameters\n ----------\n precip_nowcast: array_like\n Array of shape (m,n) containing the extrapolated precipitation field at a specified timestep\n precip_nwp: array_like\n Array of shape (m,n) containing the NWP fields at a specified timestep\n\n Returns\n -------\n ranked_salience:\n Array of shape (m,n) containing ranked salience\n \"\"\"\n\n # calcutate normalized intensity\n if np.max(precip_nowcast) == 0:\n norm_nowcast = np.zeros_like(precip_nowcast)\n else:\n norm_nowcast = precip_nowcast / np.max(precip_nowcast)\n\n if np.max(precip_nwp) == 0:\n norm_nwp = np.zeros_like(precip_nwp)\n else:\n norm_nwp = precip_nwp / np.max(precip_nwp)\n\n diff = norm_nowcast - norm_nwp\n\n # Calculate ranked salience, based on dense ranking method, in which equally comparable values receive the same ranking number\n ranked_salience = rankdata(diff, method=\"dense\").reshape(diff.shape).astype(\"float\")\n ranked_salience /= ranked_salience.max()\n\n return ranked_salience\n\n\ndef _get_ws(weight, ranked_salience):\n \"\"\"Calculate salience weight based on linear weight and ranked salience as described in :cite:`Hwang2015`.\n Cells with higher intensities result in larger weights.\n\n Parameters\n ----------\n weight: int\n Varying between 0 and 1\n ranked_salience: array_like\n Array of shape (m,n) containing ranked salience\n\n Returns\n -------\n ws: array_like\n Array of shape (m,n) containing salience weight, which preserves pixel intensties with time if they are strong\n enough based on the ranked salience.\n \"\"\"\n\n # Calculate salience weighte\n ws = 0.5 * (\n (weight * ranked_salience)\n / (weight * ranked_salience + (1 - weight) * (1 - ranked_salience))\n + (\n np.sqrt(ranked_salience**2 + weight**2)\n / (\n np.sqrt(ranked_salience**2 + weight**2)\n + np.sqrt((1 - ranked_salience) ** 2 + (1 - weight) ** 2)\n )\n )\n )\n return ws\n" }, { "path": "pysteps/blending/pca_ens_kalman_filter.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.blending.pca_ens_kalman_filter\n======================================\n\nImplementation of the reduced-space ensemble Kalman filter method described in\n:cite:`Nerini2019MWR`. The nowcast is iteratively corrected by NWP data using\nan ensemble Kalman filter in principal component (PC) space. The reduced-space\nensemble Kalman filter method consists of the following main steps:\n\nInitialization step\n-------------------\n1. Set the radar rainfall fields in a Lagrangian space.\n2. Perform the cascade decomposition for the input radar rainfall fields.\n3. Estimate AR parameters for the extrapolation nowcast and noise cascade.\n4. Initialize the noise method and precompute a set of noise fields.\n5. Initialize forecast models equal to the number of ensemble members.\n6. Initialize the ensemble Kalman filter method.\n7. Start the forecasting loop.\n\nForecast step\n-------------\n1. Decompose the rainfall forecast field of the previous timestep.\n2. Update the common precipitation mask of nowcast and NWP fields for noise imprint.\n3. Iterate the AR model.\n4. Recompose the rainfall forecast field.\n5. (Optional) Apply probability matching.\n6. Extrapolate the recomposed rainfall field to the current timestep.\n\nCorrection step\n---------------\n1. Identify grid boxes where rainfall is forecast.\n2. Reduce nowcast and NWP ensembles onto these grid boxes and apply principal\n component analysis to further reduce the dimensionality.\n3. Apply the update step of the ensemble Kalman filter.\n\nFinalization\n------------\n1. Set no-data values in the final forecast fields.\n2. The original approach iterates between forecast and correction steps.\n However, to reduce smoothing effects in this implementation, a pure\n forecast step is computed at the first forecast timestep, and afterwards\n the method alternates between correction and forecast steps. The smoothing\n effects arise due to the NWP effective horizontal resolution and due to\n the spatial decomposition at each forecast timestep.\n\n.. autosummary::\n :toctree: ../generated/\n\n forecast\n\"\"\"\n\nimport datetime\nimport time\nfrom copy import deepcopy\n\nimport numpy as np\nfrom scipy.ndimage import (\n binary_dilation,\n gaussian_filter,\n)\n\nfrom pysteps import blending, cascade, extrapolation, noise, utils\nfrom pysteps.blending.ens_kalman_filter_methods import MaskedEnKF\nfrom pysteps.nowcasts import utils as nowcast_utils\nfrom pysteps.postprocessing import probmatching\nfrom pysteps.timeseries import autoregression, correlation\nfrom pysteps.utils.check_norain import check_norain\n\ntry:\n import dask\n\n DASK_IMPORTED = True\nexcept ImportError:\n DASK_IMPORTED = False\n\nfrom dataclasses import dataclass, field\nfrom typing import Any, Callable\n\n\n@dataclass(frozen=True)\nclass EnKFCombinationConfig:\n \"\"\"\n Parameters\n ----------\n\n n_ens_members: int\n The number of ensemble members to generate. This number should always be\n equal to the number of NWP ensemble members / number of NWP models.\n n_cascade_levels: int\n The number of cascade levels to use. Defaults to 6,\n see issue #385 on GitHub.\n precip_threshold: float\n Specifies the threshold value for minimum observable precipitation\n intensity.\n norain_threshold: float\n Specifies the threshold value for the fraction of rainy (see above) pixels\n in the radar rainfall field below which we consider there to be no rain.\n Depends on the amount of clutter typically present.\n precip_mask_dilation: int\n Number of grid boxes by which the precipitation mask should be extended per\n timestep.\n smooth_radar_mask_range: int, Default is 0.\n Method to smooth the transition between the radar-NWP-noise blend and the NWP-noise\n blend near the edge of the radar domain (radar mask), where the radar data is either\n not present anymore or is not reliable. If set to 0 (grid cells), this generates a\n normal forecast without smoothing. To create a smooth mask, this range should be a\n positive value, representing a buffer band of a number of pixels by which the mask\n is cropped and smoothed. The smooth radar mask removes the hard edges between NWP\n and radar in the final blended product. Typically, a value between 50 and 100 km\n can be used. 80 km generally gives good results.\n extrapolation_method: str\n Name of the extrapolation method to use. See the documentation of\n :py:mod:`pysteps.extrapolation.interface`.\n decomposition_method: str, {'fft'}\n Name of the cascade decomposition method to use. See the documentation\n of :py:mod:`pysteps.cascade.interface`.\n bandpass_filter_method: str, {'gaussian', 'uniform'}\n Name of the bandpass filter method to use with the cascade decomposition.\n See the documentation of :py:mod:`pysteps.cascade.interface`.\n noise_method: str, {'parametric','nonparametric','ssft','nested',None}\n Name of the noise generator to use for perturbating the precipitation\n field. See the documentation of :py:mod:`pysteps.noise.interface`. If set to\n None, no noise is generated.\n enkf_method: str, {'masked_enkf'}\n Name of the ensemble Kalman filter method to use for the correction step.\n Currently, only 'masked_enkf' is implemented. This method corresponds to the\n reduced-space ensemble Kalman filter method described by Nerini et al., 2019.\n enable_combination: bool\n Flag to specify whether the correction step or only the forecast steps should\n be processed.\n noise_stddev_adj: str, {'auto','fixed',None}\n Optional adjustment for the standard deviations of the noise fields added\n to each cascade level. This is done to compensate incorrect std. dev.\n estimates of casace levels due to presence of no-rain areas. 'auto'=use\n the method implemented in :py:func:`pysteps.noise.utils.\n compute_noise_stddev_adjs`.\n 'fixed'= use the formula given in :cite:`BPS2006` (eq. 6), None=disable\n noise std. dev adjustment.\n ar_order: int\n The order of the autoregressive model to use. Currently, only AR(1) is\n implemented.\n seed: int\n Optional seed number for the random generators.\n num_workers: int\n The number of workers to use for parallel computation. Applicable if dask\n is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it\n is advisable to disable OpenMP by setting the environment variable\n OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous\n threads.\n fft_method: str\n A string defining the FFT method to use (see FFT methods in\n :py:func:`pysteps.utils.interface.get_method`).\n Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed,\n the recommended method is 'pyfftw'.\n domain: str, {\"spatial\", \"spectral\"}\n If \"spatial\", all computations are done in the spatial domain (the\n classical STEPS model). If \"spectral\", the AR(2) models and stochastic\n perturbations are applied directly in the spectral domain to reduce\n memory footprint and improve performance :cite:`PCH2019b`.\n extrapolation_kwargs: dict\n Optional dictionary containing keyword arguments for the extrapolation\n method. See the documentation of :py:func:`pysteps.extrapolation.interface`.\n filter_kwargs: dict\n Optional dictionary containing keyword arguments for the filter method.\n See the documentation of :py:mod:`pysteps.cascade.bandpass_filters`.\n noise_kwargs: dict\n Optional dictionary containing keyword arguments for the initializer of\n the noise generator. See the documentation of :py:mod:`pysteps.noise.\n fftgenerators`.\n combination_kwargs: dict\n Optional dictionary containing keyword arguments for the initializer of the\n correction step. Options are: {nwp_hres_eff: float, the effective horizontal\n resolution of the utilized NWP model; prob_matching: str, specifies the\n probability matching method that should be applied}. See the documentation of\n :py:mod:`pysteps.blending.ens_kalman_filter_methods`.\n measure_time: bool\n If set to True, measure, print and return the computation time.\n verbose_output: bool\n If set to True, return additionally the background ensemble of the EnKF for further statistics.\n callback: function, optional\n Optional function that is called after computation of each time step of\n the nowcast. The function takes one argument: a three-dimensional array\n of shape (n_ens_members,h,w), where h and w are the height and width\n of the input field precip, respectively. This can be used, for instance,\n writing the outputs into files.\n return_output: bool\n Set to False to disable returning the outputs as numpy arrays. This can\n save memory if the intermediate results are written to output files using\n the callback function. (Call back function is currently not implemented.)\n n_noise_fields: int\n Number of precomputed noise fields. A number of 30 is adequate to generate\n sufficient spread in the Nowcast.\n \"\"\"\n\n n_ens_members: int\n n_cascade_levels: int\n precip_threshold: float | None\n norain_threshold: float\n precip_mask_dilation: int\n smooth_radar_mask_range: int\n extrapolation_method: str\n decomposition_method: str\n bandpass_filter_method: str\n noise_method: str | None\n enkf_method: str | None\n enable_combination: bool\n noise_stddev_adj: str | None\n ar_order: int\n seed: int | None\n num_workers: int\n fft_method: str\n domain: str\n extrapolation_kwargs: dict[str, Any] = field(default_factory=dict)\n filter_kwargs: dict[str, Any] = field(default_factory=dict)\n noise_kwargs: dict[str, Any] = field(default_factory=dict)\n combination_kwargs: dict[str, Any] = field(default_factory=dict)\n measure_time: bool = False\n verbose_output: bool = False\n callback: Any | None = None\n return_output: bool = True\n n_noise_fields: int = 30\n\n\n@dataclass\nclass EnKFCombinationParams:\n noise_std_coeffs: np.ndarray | None = None\n bandpass_filter: Any | None = None\n fft: Any | None = None\n perturbation_generator: Callable[..., np.ndarray] | None = None\n noise_generator: Callable[..., np.ndarray] | None = None\n PHI: np.ndarray | None = None\n extrapolation_method: Callable[..., Any] | None = None\n decomposition_method: Callable[..., dict] | None = None\n recomposition_method: Callable[..., np.ndarray] | None = None\n fft_objs: list[Any] = field(default_factory=list)\n xy_coordinates: np.ndarray | None = None\n precip_threshold: float | None = None\n mask_threshold: np.ndarray | None = None\n num_ensemble_workers: int | None = None\n domain_mask: np.ndarray | None = None\n extrapolation_kwargs: dict | None = None\n filter_kwargs: dict | None = None\n noise_kwargs: dict | None = None\n combination_kwargs: dict | None = None\n len_y: int | None = None\n len_x: int | None = None\n no_rain_case: str | None = None\n\n\nclass ForecastInitialization:\n \"\"\"\n Class to bundle the steps necessary for the forecast initialization.\n These steps are:\n\n #. Set the radar rainfall fields in a Lagrangian space.\n #. Perform the cascade decomposition for the input radar rainfall fields.\n #. Estimate AR parameters for the extrapolation nowcast and noise cascade.\n #. Initialize the noise method and precompute a set of noise fields.\n \"\"\"\n\n def __init__(\n self,\n enkf_combination_config: EnKFCombinationConfig,\n enkf_combination_params: EnKFCombinationParams,\n obs_precip: np.ndarray,\n obs_velocity: np.ndarray,\n ):\n self.__config = enkf_combination_config\n self.__params = enkf_combination_params\n\n self.__obs_precip = obs_precip\n self.__obs_velocity = obs_velocity\n\n # Measure time for initialization.\n if self.__config.measure_time:\n self.__start_time_init = time.time()\n\n self.__initialize_nowcast_components()\n\n self.__prepare_radar_data_and_ar_parameters()\n\n self.__initialize_noise()\n\n self.__initialize_noise_field_pool()\n\n if self.__config.measure_time:\n print(\n f\"Elapsed time for initialization: {time.time() - self.__start_time_init}\"\n )\n\n # Initialize FFT, bandpass filters, decomposition methods, and extrapolation\n # method.\n def __initialize_nowcast_components(self):\n # Initialize number of ensemble workers\n self.__params.num_ensemble_workers = min(\n self.__config.n_ens_members,\n self.__config.num_workers,\n )\n\n # Extract the spatial dimensions of the observed precipitation (x, y)\n self.__params.len_y, self.__params.len_x = self.__obs_precip.shape[1:]\n\n # Generate the mesh grid for spatial coordinates\n x_values, y_values = np.meshgrid(\n np.arange(self.__params.len_x),\n np.arange(self.__params.len_y),\n )\n self.__params.xy_coordinates = np.stack([x_values, y_values])\n\n # Initialize FFT method\n self.__params.fft = utils.get_method(\n self.__config.fft_method,\n shape=(\n self.__params.len_y,\n self.__params.len_x,\n ),\n n_threads=self.__config.num_workers,\n )\n\n # Initialize the band-pass filter for the cascade decomposition\n filter_method = cascade.get_method(self.__config.bandpass_filter_method)\n self.__params.bandpass_filter = filter_method(\n (self.__params.len_y, self.__params.len_x),\n self.__config.n_cascade_levels,\n **(self.__params.filter_kwargs or {}),\n )\n\n # Get the decomposition method (e.g., FFT)\n (\n self.__params.decomposition_method,\n self.__params.recomposition_method,\n ) = cascade.get_method(self.__config.decomposition_method)\n\n # Get the extrapolation method (e.g., semilagrangian)\n self.__params.extrapolation_method = extrapolation.get_method(\n self.__config.extrapolation_method\n )\n\n # Determine the domain mask from non-finite values in the precipitation data\n self.__params.domain_mask = np.logical_or.reduce(\n [\n ~np.isfinite(self.__obs_precip[i, :])\n for i in range(self.__obs_precip.shape[0])\n ]\n )\n\n print(\"Nowcast components initialized successfully.\")\n\n # Prepare radar precipitation fields for nowcasting and estimate the AR\n # parameters.\n def __prepare_radar_data_and_ar_parameters(self):\n \"\"\"\n Prepare radar and NWP precipitation fields for nowcasting.\n This includes generating a threshold mask, transforming fields into\n Lagrangian coordinates, cascade decomposing/recomposing, and checking\n for zero-precip areas. The results are stored in class attributes.\n\n Estimate autoregressive (AR) parameters for the radar rainfall field. If\n precipitation exists, compute temporal auto-correlations; otherwise, use\n predefined climatological values. Adjust coefficients if necessary and\n estimate AR model parameters.\n \"\"\"\n\n # Start with the radar rainfall fields. We want the fields in a Lagrangian\n # space. Advect the previous precipitation fields to the same position with\n # the most recent one (i.e. transform them into the Lagrangian coordinates).\n self.__params.extrapolation_kwargs[\"xy_coords\"] = self.__params.xy_coordinates\n self.__params.extrapolation_kwargs[\"outval\"] = (\n self.__config.precip_threshold - 2.0\n )\n res = []\n\n def transform_to_lagrangian(precip, i):\n return self.__params.extrapolation_method(\n precip[i, :, :],\n self.__obs_velocity,\n self.__config.ar_order - i,\n allow_nonfinite_values=True,\n **self.__params.extrapolation_kwargs.copy(),\n )[-1]\n\n if not DASK_IMPORTED:\n # Process each earlier precipitation field directly\n for i in range(self.__config.ar_order):\n self.__obs_precip[i, :, :] = transform_to_lagrangian(\n self.__obs_precip, i\n )\n else:\n # Use Dask delayed for parallelization if DASK_IMPORTED is True\n for i in range(self.__config.ar_order):\n res.append(dask.delayed(transform_to_lagrangian)(self.__obs_precip, i))\n num_workers_ = (\n len(res)\n if self.__config.num_workers > len(res)\n else self.__config.num_workers\n )\n self.__obs_precip = np.stack(\n list(dask.compute(*res, num_workers=num_workers_))\n + [self.__obs_precip[-1, :, :]]\n )\n\n # Mask the observations\n obs_mask = np.logical_or(\n ~np.isfinite(self.__obs_precip),\n self.__obs_precip < self.__config.precip_threshold,\n )\n self.__obs_precip[obs_mask] = self.__config.precip_threshold - 2.0\n\n # Compute the cascade decompositions of the input precipitation fields\n precip_forecast_decomp = []\n for i in range(self.__config.ar_order + 1):\n precip_forecast = self.__params.decomposition_method(\n self.__obs_precip[i, :, :],\n self.__params.bandpass_filter,\n mask=self.__params.mask_threshold,\n fft_method=self.__params.fft,\n output_domain=self.__config.domain,\n normalize=True,\n compute_stats=True,\n compact_output=False,\n )\n precip_forecast_decomp.append(precip_forecast)\n\n # Rearrange the cascaded into a four-dimensional array of shape\n # (n_cascade_levels,ar_order+1,m,n) for the autoregressive model\n self.precip_cascades = nowcast_utils.stack_cascades(\n precip_forecast_decomp, self.__config.n_cascade_levels\n )\n\n # Set the mean and standard deviations based on the most recent field.\n precip_forecast_decomp = precip_forecast_decomp[-1]\n self.mean_extrapolation = np.array(precip_forecast_decomp[\"means\"])\n self.std_extrapolation = np.array(precip_forecast_decomp[\"stds\"])\n\n if self.__params.no_rain_case == \"obs\":\n GAMMA = np.ones((self.__config.n_cascade_levels, self.__config.ar_order))\n\n else:\n # If there are values in the radar fields, compute the auto-correlations\n GAMMA = np.empty((self.__config.n_cascade_levels, self.__config.ar_order))\n\n # compute lag-l temporal auto-correlation coefficients for each cascade level\n for i in range(self.__config.n_cascade_levels):\n GAMMA[i, :] = correlation.temporal_autocorrelation(\n self.precip_cascades[i], mask=self.__params.mask_threshold\n )\n\n # Print the GAMMA value\n nowcast_utils.print_corrcoefs(GAMMA)\n\n if self.__config.ar_order == 2:\n # Adjust the lag-2 correlation coefficient to ensure that the AR(p)\n # process is stationary\n for i in range(self.__config.n_cascade_levels):\n GAMMA[i, 1] = autoregression.adjust_lag2_corrcoef2(\n GAMMA[i, 0], GAMMA[i, 1]\n )\n\n # Estimate the parameters of the AR(p) model from the auto-correlation\n # coefficients\n self.__params.PHI = np.empty(\n (self.__config.n_cascade_levels, self.__config.ar_order + 1)\n )\n for i in range(self.__config.n_cascade_levels):\n self.__params.PHI[i, :] = autoregression.estimate_ar_params_yw(GAMMA[i, :])\n\n nowcast_utils.print_ar_params(self.__params.PHI)\n\n # Initialize the noise generation and get n_noise_fields.\n def __initialize_noise(self):\n \"\"\"\n Initialize noise-based perturbations if configured, computing any required\n adjustment coefficients and setting up the perturbation generator.\n \"\"\"\n if (\n self.__config.noise_method is not None\n and self.__params.no_rain_case != \"obs\"\n ):\n # get methods for perturbations\n init_noise, self.__params.noise_generator = noise.get_method(\n self.__config.noise_method\n )\n\n self.__precip_noise_input = self.__obs_precip.copy()\n\n # initialize the perturbation generator for the precipitation field\n self.__params.perturbation_generator = init_noise(\n self.__precip_noise_input,\n fft_method=self.__params.fft,\n **self.__params.noise_kwargs,\n )\n\n if self.__config.noise_stddev_adj == \"auto\":\n print(\"Computing noise adjustment coefficients... \", end=\"\", flush=True)\n precip_forecast_min = np.min(self.__precip_noise_input)\n self.__params.noise_std_coeffs = noise.utils.compute_noise_stddev_adjs(\n self.__precip_noise_input[-1, :, :],\n self.__params.precip_threshold,\n precip_forecast_min,\n self.__params.bandpass_filter,\n self.__params.decomposition_method,\n self.__params.perturbation_generator,\n self.__params.noise_generator,\n 20,\n conditional=True,\n num_workers=self.__config.num_workers,\n seed=self.__config.seed,\n )\n\n elif self.__config.noise_stddev_adj == \"fixed\":\n f = lambda k: 1.0 / (0.75 + 0.09 * k)\n self.__params.noise_std_coeffs = [\n f(k) for k in range(1, self.__config.n_cascade_levels + 1)\n ]\n else:\n self.__params.noise_std_coeffs = np.ones(self.__config.n_cascade_levels)\n\n if self.__config.noise_stddev_adj is not None:\n print(f\"noise std. dev. coeffs: {self.__params.noise_std_coeffs}\")\n\n else:\n self.__params.perturbation_generator = None\n self.__params.noise_generator = None\n self.__params.noise_std_coeffs = None\n\n # Create a pool of n noise fields.\n def __initialize_noise_field_pool(self):\n \"\"\"\n Initialize a pool of noise fields avoiding the separate generation of noise fields for each\n time step and ensemble member. A pool of 30 fields is sufficient to generate adequate spread\n in the nowcast for combination.\n \"\"\"\n self.noise_field_pool = np.zeros(\n (\n self.__config.n_noise_fields,\n self.__config.n_cascade_levels,\n self.__params.len_y,\n self.__params.len_x,\n )\n )\n\n # Get a seed value for each ensemble member\n seed = self.__config.seed\n if self.__config.noise_method is not None:\n self.__randgen_precip = []\n # for j in range(self.__config.n_ens_members):\n for j in range(self.__config.n_noise_fields):\n rs = np.random.RandomState(seed)\n self.__randgen_precip.append(rs)\n seed = rs.randint(0, high=1e9)\n\n # Get the decomposition method\n self.__params.fft_objs = []\n for _ in range(self.__config.n_noise_fields):\n self.__params.fft_objs.append(\n utils.get_method(\n self.__config.fft_method,\n shape=self.precip_cascades.shape[-2:],\n )\n )\n\n if self.__params.noise_generator is not None:\n # Determine the noise field for each ensemble member\n for j in range(self.__config.n_noise_fields):\n epsilon = self.__params.noise_generator(\n self.__params.perturbation_generator,\n randstate=self.__randgen_precip[j],\n fft_method=self.__params.fft_objs[j],\n domain=self.__config.domain,\n )\n # Decompose the noise field into a cascade\n self.noise_field_pool[j] = self.__params.decomposition_method(\n epsilon,\n self.__params.bandpass_filter,\n fft_method=self.__params.fft_objs[j],\n input_domain=self.__config.domain,\n output_domain=self.__config.domain,\n compute_stats=False,\n normalize=True,\n compact_output=True,\n )[\"cascade_levels\"]\n\n\nclass ForecastState:\n \"\"\"\n Common memory of ForecastModel instances.\n \"\"\"\n\n def __init__(\n self,\n enkf_combination_config: EnKFCombinationConfig,\n enkf_combination_params: EnKFCombinationParams,\n noise_field_pool: np.ndarray,\n latest_obs: np.ndarray,\n precip_mask: np.ndarray,\n ):\n self.config = enkf_combination_config\n self.params = enkf_combination_params\n self.noise_field_pool = noise_field_pool\n self.precip_mask = np.repeat(\n precip_mask[None, :], self.config.n_ens_members, axis=0\n )\n\n latest_obs[~np.isfinite(latest_obs)] = self.config.precip_threshold - 2.0\n self.nwc_prediction = np.repeat(\n latest_obs[None, :, :], self.config.n_ens_members, axis=0\n )\n self.fc_resampled = np.repeat(\n latest_obs[None, :, :], self.config.n_ens_members, axis=0\n )\n self.nwc_prediction_btf = self.nwc_prediction.copy()\n\n self.final_combined_forecast = []\n self.background_ensemble = {}\n\n return\n\n\nclass ForecastModel:\n \"\"\"\n Class to manage the forecast step of each ensemble member.\n \"\"\"\n\n def __init__(\n self,\n forecast_state: ForecastState,\n precip_cascades: np.ndarray,\n velocity: np.ndarray,\n mu: np.ndarray,\n sigma: np.ndarray,\n ens_member: int,\n ):\n # Initialize instance variables\n self.__forecast_state = forecast_state\n self.__precip_cascades = precip_cascades\n self.__velocity = velocity\n\n self.__mu = mu\n self.__sigma = sigma\n\n self.__previous_displacement = np.zeros(\n (2, self.__forecast_state.params.len_y, self.__forecast_state.params.len_x)\n )\n\n # Get NWP effective horizontal resolution and type of probability matching from\n # combination kwargs.\n # It's not the best practice to mix parameters. Maybe the cascade mask as well\n # as the probability matching should be implemented at another location.\n self.__nwp_hres_eff = self.__forecast_state.params.combination_kwargs.get(\n \"nwp_hres_eff\", 0.0\n )\n self.__prob_matching = self.__forecast_state.params.combination_kwargs.get(\n \"prob_matching\", \"iterative\"\n )\n\n # Get spatial scales whose central wavelengths are above the effective\n # horizontal resolution of the NWP model.\n # Factor 3 on the effective resolution is similar to that factor of the\n # localization of AR parameters and scaling parameters.\n self.__resolution_mask = (\n self.__forecast_state.params.len_y\n / self.__forecast_state.params.bandpass_filter[\"central_wavenumbers\"]\n >= self.__nwp_hres_eff * 3.0\n )\n\n self.__ens_member = ens_member\n\n # Bundle single steps of the forecast.\n def run_forecast_step(self, nwp, is_correction_timestep=False):\n # Decompose precipitation field.\n self.__decompose(is_correction_timestep)\n\n # Update precipitation mask.\n self.__update_precip_mask(nwp=nwp)\n\n # Iterate through the AR process.\n self.__iterate()\n\n # Recompose the precipitation field for the correction step.\n self.__forecast_state.nwc_prediction[self.__ens_member] = (\n blending.utils.recompose_cascade(\n combined_cascade=self.__precip_cascades[:, -1],\n combined_mean=self.__mu,\n combined_sigma=self.__sigma,\n )\n )\n\n # Apply probability matching\n if self.__prob_matching == \"iterative\":\n self.__probability_matching()\n\n # Extrapolate the precipitation field onto the position of the current timestep.\n # If smooth_radar_mask_range is not zero, ensure the extrapolation kwargs use\n # a constant value instead of \"nearest\" for the coordinate mapping, otherwise\n # there are possibly no nans in the domain.\n if self.__forecast_state.config.smooth_radar_mask_range != 0:\n self.__forecast_state.params.extrapolation_kwargs[\n \"map_coordinates_mode\"\n ] = \"constant\"\n self.__advect()\n\n # The extrapolation components are NaN outside the advected\n # radar domain. This results in NaN values in the blended\n # forecast outside the radar domain. Therefore, fill these\n # areas with the defined minimum value, if requested.\n nan_mask = np.isnan(self.__forecast_state.nwc_prediction[self.__ens_member])\n self.__forecast_state.nwc_prediction[self.__ens_member][nan_mask] = (\n self.__forecast_state.config.precip_threshold - 2.0\n )\n\n # Create the resulting precipitation field and set no data area. In future, when\n # transformation between linear and logarithmic scale will be necessary, it will be\n # implemented in this function.\n # TODO: once this transformation is needed, adjust the smoothed transition between\n # radar mask and NWP as performed at the end of the run_forecast_step function.\n def backtransform(self):\n # Set the resulting field as shallow copy of the field that is used\n # continuously for forecast computation.\n if self.__forecast_state.config.smooth_radar_mask_range == 0:\n self.__forecast_state.nwc_prediction_btf[self.__ens_member] = (\n self.__forecast_state.nwc_prediction[self.__ens_member]\n )\n\n # Set no data area\n self.__set_no_data()\n\n # Call spatial decomposition function and compute an adjusted standard deviation of\n # each spatial scale at timesteps where NWP information is incorporated.\n def __decompose(self, is_correction_timestep):\n # Call spatial decomposition method.\n precip_extrap_decomp = self.__forecast_state.params.decomposition_method(\n self.__forecast_state.nwc_prediction[self.__ens_member],\n self.__forecast_state.params.bandpass_filter,\n fft_method=self.__forecast_state.params.fft_objs[self.__ens_member],\n input_domain=self.__forecast_state.config.domain,\n output_domain=self.__forecast_state.config.domain,\n compute_stats=False,\n normalize=True,\n compact_output=False,\n )\n\n # Set decomposed field onto the latest precipitation cascade.\n self.__precip_cascades[:, -1] = precip_extrap_decomp[\"cascade_levels\"]\n\n # If NWP information is incorporated, use the current mean of the decomposed\n # field and adjust standard deviation on spatial scales that have a central\n # wavelength below the effective horizontal resolution of the NWP model.\n if is_correction_timestep:\n # Set the mean of the spatial scales onto the mean values of the currently\n # decomposed field.\n self.__mu = np.array(precip_extrap_decomp[\"means\"])\n # Compute the standard deviation evolved by an AR(1)-process.\n self.__sigma = np.sqrt(\n self.__forecast_state.params.PHI[:, 0] ** 2.0 * self.__sigma**2.0\n + self.__forecast_state.params.PHI[:, 1] ** 2.0\n * self.__forecast_state.params.noise_std_coeffs**2.0\n )\n\n # Use the standard deviations of the currently decomposed field for spatial\n # scales above the effective horizontal resolution of the NWP model.\n self.__sigma[self.__resolution_mask] = np.array(\n precip_extrap_decomp[\"stds\"]\n )[self.__resolution_mask]\n # Else, keep mean and standard deviation constant for pure nowcasting forecast steps.\n # It's not necessary but describes better the handling of the scaling\n # parameters.\n else:\n self.__mu = self.__mu\n self.__sigma = self.__sigma\n\n # Call extrapolation function to extrapolate the precipitation field onto the\n # position of the current timestep.\n def __advect(self):\n # Since previous displacement is the sum of displacement over all previous\n # timesteps, we have to compute the differences between the displacements to\n # get the motion vector field for one time step.\n displacement_tmp = self.__previous_displacement.copy()\n\n # Call the extrapolation method\n (\n self.__forecast_state.nwc_prediction[self.__ens_member],\n self.__previous_displacement,\n ) = self.__forecast_state.params.extrapolation_method(\n self.__forecast_state.nwc_prediction[self.__ens_member],\n self.__velocity,\n [1],\n allow_nonfinite_values=True,\n displacement_previous=self.__previous_displacement,\n **self.__forecast_state.params.extrapolation_kwargs,\n )\n if (\n self.__forecast_state.config.smooth_radar_mask_range > 0\n and self.__ens_member == 0\n ):\n self.__forecast_state.params.domain_mask = (\n self.__forecast_state.params.extrapolation_method(\n self.__forecast_state.params.domain_mask,\n self.__velocity,\n [1],\n interp_order=1,\n outval=True,\n )[0]\n )\n\n # Get the difference of the previous displacement field.\n self.__previous_displacement -= displacement_tmp\n\n # Get a noise field out of the respective pool and iterate through the AR(1)\n # process.\n def __iterate(self):\n # Get a noise field out of the noise field pool and multiply it with\n # precipitation mask and the standard deviation coefficients.\n epsilon = (\n self.__forecast_state.noise_field_pool[\n np.random.randint(self.__forecast_state.config.n_noise_fields)\n ]\n * self.__forecast_state.precip_mask[self.__ens_member][None, :, :]\n * self.__forecast_state.params.noise_std_coeffs[:, None, None]\n )\n\n # Iterate through the AR(1) process for each cascade level.\n for i in range(self.__forecast_state.config.n_cascade_levels):\n self.__precip_cascades[i] = autoregression.iterate_ar_model(\n self.__precip_cascades[i],\n self.__forecast_state.params.PHI[i],\n epsilon[i],\n )\n\n # Update the precipitation mask for the forecast step by incorporating areas\n # where the NWP model forecast precipitation.\n def __update_precip_mask(self, nwp):\n # Get the area where the NWP ensemble member forecast precipitation above\n # precipitation threshold and dilate it by a configurable range.\n precip_mask = (\n binary_dilation(\n nwp > self.__forecast_state.config.precip_threshold,\n structure=np.ones(\n (\n self.__forecast_state.config.precip_mask_dilation,\n self.__forecast_state.config.precip_mask_dilation,\n ),\n dtype=int,\n ),\n )\n * 1.0\n )\n # Get the area where the combined member forecast precipitation above the\n # precipitation threshold and dilate it by a configurable range.\n precip_mask += (\n binary_dilation(\n self.__forecast_state.nwc_prediction[self.__ens_member]\n > self.__forecast_state.config.precip_threshold,\n structure=np.ones(\n (\n self.__forecast_state.config.precip_mask_dilation,\n self.__forecast_state.config.precip_mask_dilation,\n ),\n dtype=int,\n ),\n )\n * 1.0\n )\n # Set values above 1 to 1 for conversion into bool.\n precip_mask[precip_mask >= 1.0] = 1.0\n # Some additional dilation of the precipitation mask.\n precip_mask = gaussian_filter(precip_mask, (1, 1))\n # Set the mask outside the radar domain to 0.\n precip_mask[self.__forecast_state.params.domain_mask] = 0.0\n # Convert mask into bool.\n self.__forecast_state.precip_mask[self.__ens_member] = np.array(\n precip_mask, dtype=bool\n )\n\n # Apply probability matching\n def __probability_matching(self):\n # Apply probability matching\n self.__forecast_state.nwc_prediction[self.__ens_member] = (\n probmatching.nonparam_match_empirical_cdf(\n self.__forecast_state.nwc_prediction[self.__ens_member],\n self.__forecast_state.fc_resampled[self.__ens_member],\n )\n )\n\n # Set no data area in the resulting precipitation field.\n def __set_no_data(self):\n self.__forecast_state.nwc_prediction_btf[self.__ens_member][\n self.__forecast_state.params.domain_mask\n ] = np.nan\n\n # Fill edge zones of the domain with NWP data if smooth_radar_mask_range is > 0\n def fill_backtransform(self, nwp):\n # For a smoother transition at the edge, we can slowly dilute the nowcast\n # component into NWP at the edges\n\n # Compute the smooth dilated mask\n new_mask = blending.utils.compute_smooth_dilated_mask(\n self.__forecast_state.params.domain_mask,\n max_padding_size_in_px=self.__forecast_state.config.smooth_radar_mask_range,\n )\n new_mask = np.nan_to_num(new_mask, nan=0)\n\n # Ensure mask values are between 0 and 1\n mask_model = np.clip(new_mask, 0, 1)\n mask_radar = np.clip(1 - new_mask, 0, 1)\n\n # Handle NaNs in precip_forecast_new and precip_forecast_new_mod_only by setting NaNs to 0 in the blending step\n nwp_temp = np.nan_to_num(nwp, nan=0)\n nwc_temp = np.nan_to_num(\n self.__forecast_state.nwc_prediction[self.__ens_member], nan=0\n )\n\n # Perform the blending of radar and model inside the radar domain using a weighted combination\n self.__forecast_state.nwc_prediction_btf[self.__ens_member] = np.nansum(\n [\n mask_model * nwp_temp,\n mask_radar * nwc_temp,\n ],\n axis=0,\n )\n\n\nclass EnKFCombinationNowcaster:\n def __init__(\n self,\n obs_precip: np.ndarray,\n obs_timestamps: np.ndarray,\n nwp_precip: np.ndarray,\n nwp_timestamps: np.ndarray,\n obs_velocity: np.ndarray,\n fc_period: int,\n fc_init: datetime.datetime,\n enkf_combination_config: EnKFCombinationConfig,\n ):\n \"\"\"\n Initialize EnKFCombinationNowcaster with inputs and configurations.\n \"\"\"\n # Store inputs\n self.__obs_precip = obs_precip\n self.__nwp_precip = nwp_precip\n self.__obs_velocity = obs_velocity\n self.__fc_period = fc_period\n self.__fc_init = fc_init\n\n # Store config\n self.__config = enkf_combination_config\n\n # Initialize Params\n self.__params = EnKFCombinationParams()\n\n # Store input timestamps\n self.__obs_timestamps = obs_timestamps\n self.__nwp_timestamps = nwp_timestamps\n\n def compute_forecast(self):\n \"\"\"\n Generate a combined nowcast ensemble by using the reduced-space ensemble Kalman\n filter method.\n\n Parameters\n ----------\n obs_precip: np.ndarray\n Array of shape (ar_order+1,m,n) containing the observed input precipitation\n fields ordered by timestamp from oldest to newst. The time steps between\n the inputs are assumed to be regular.\n obs_timestamps: np.ndarray\n Array of shape (ar_order+1) containing the corresponding time stamps of\n observed input precipitation fields as datetime objects.\n nwp_precip: np.ndarray\n Array of shape (n_ens,n_times,m,n) containing the (NWP) ensemble model\n forecast.\n nwp_timestamps: np.ndarray\n Array of shape (n_times) containing the corresponding time stamps of the\n (NWP) ensemble model forecast as datetime objects.\n obs_velocity: np.ndarray\n Array of shape (2,m,n) containing the x- and y-components of the advection\n field. The velocities are based on the observed input precipitation fields\n and are assumed to represent one time step between the inputs. All values\n are required to be finite.\n fc_period: int\n Forecast range in minutes.\n fc_init: datetime object\n Issuetime of the combined forecast to compute.\n enkf_combination_config: EnKFCombinationConfig\n Provides a set of configuration parameters for the nowcast ensemble\n generation.\n\n Returns\n -------\n out: np.ndarray\n If return_output is True, a four-dimensional array of shape\n (n_ens_members,num_timesteps,m,n) containing a time series of forecast\n precipitation fields for each ensemble member. Otherwise, a None value\n is returned. The time series starts from t0. The timestep is taken from the\n input precipitation fields precip.\n\n See also\n --------\n :py:mod:`pysteps.extrapolation.interface`, :py:mod:`pysteps.cascade.interface`,\n :py:mod:`pysteps.noise.interface`, :py:func:`pysteps.noise.utils.\n compute_noise_stddev_adjs`\n\n References\n ----------\n :cite:`Nerini2019MWR`\n\n Notes\n -----\n 1. The combination method currently supports only an AR(1) process for the\n forecast step.\n \"\"\"\n\n # Check for the inputs.\n self.__check_inputs()\n\n # Check timestamps of radar and nwp input and determine forecast and correction\n # timesteps as well as the temporal resolution\n self.__check_input_timestamps()\n\n # Check wehther there is no precipitation in observation, but in NWP or the other way around\n self.__check_no_rain_case()\n\n # Print forecast information.\n self.__print_forecast_info()\n\n # Initialize and compute the forecast initialization.\n self.FI = ForecastInitialization(\n self.__config, self.__params, self.__obs_precip, self.__obs_velocity\n )\n\n # NWP: Set values below precip thr and nonfinite values to norain thr.\n nwp_mask = np.logical_or(\n ~np.isfinite(self.__nwp_precip),\n self.__nwp_precip < self.__config.precip_threshold,\n )\n self.__nwp_precip[nwp_mask] = self.__config.precip_threshold - 2.0\n\n # Set an initial precipitation mask for the NWC models.\n precip_mask = binary_dilation(\n self.__obs_precip[-1] > self.__config.precip_threshold,\n structure=np.ones(\n (self.__config.precip_mask_dilation, self.__config.precip_mask_dilation)\n ),\n )\n\n # Initialize an instance of NWC forecast model class for each ensemble member.\n self.FS = ForecastState(\n enkf_combination_config=self.__config,\n enkf_combination_params=self.__params,\n noise_field_pool=self.FI.noise_field_pool,\n latest_obs=self.__obs_precip[-1, :, :],\n precip_mask=precip_mask.copy(),\n )\n\n self.FC_Models = {}\n for j in range(self.__config.n_ens_members):\n FC = ForecastModel(\n forecast_state=self.FS,\n precip_cascades=deepcopy(self.FI.precip_cascades),\n velocity=self.__obs_velocity,\n mu=deepcopy(self.FI.mean_extrapolation),\n sigma=deepcopy(self.FI.std_extrapolation),\n ens_member=j,\n )\n self.FC_Models[j] = FC\n\n # Initialize the combination model.\n if self.__config.enkf_method == \"masked_enkf\":\n kalman_filter_model = MaskedEnKF\n else:\n raise ValueError(\n \"Currently, only 'masked_enkf' is implemented as ensemble\"\n \"Kalman filter method!\"\n )\n self.KalmanFilterModel = kalman_filter_model(self.__config, self.__params)\n\n # Start the main forecast loop.\n self.__integrated_nowcast_main_loop()\n\n # Stack and return the forecast output.\n if self.__config.return_output:\n self.FS.final_combined_forecast = np.array(\n self.FS.final_combined_forecast\n ).swapaxes(0, 1)\n\n if self.__config.measure_time:\n return (\n self.FS.final_combined_forecast,\n self.__fc_init,\n self.__mainloop_time,\n )\n if self.__config.verbose_output:\n return self.FS.final_combined_forecast, self.FS.background_ensemble\n return self.FS.final_combined_forecast\n\n # Else, return None\n return None\n\n def __check_inputs(self):\n \"\"\"\n Validates user's input.\n \"\"\"\n\n # Check dimensions of obs precip\n if self.__obs_precip.ndim != 3:\n raise ValueError(\n \"Precipitation observation must be a three-dimensional \"\n \"array of shape (ar_order + 1, m, n)\"\n )\n if self.__obs_precip.shape[0] < self.__config.ar_order + 1:\n raise ValueError(\n f\"Precipitation observation must have at least \"\n f\"{self.__config.ar_order + 1} time steps in the first\"\n f\"dimension to match the autoregressive order \"\n f\"(ar_order={self.__config.ar_order})\"\n )\n\n # If it is necessary, slice the precipitation field to only use the last\n # ar_order +1 time steps.\n if self.__obs_precip.shape[0] > self.__config.ar_order + 1:\n self.__obs_precip = np.delete(\n self.__obs_precip,\n np.arange(\n 0, self.__obs_precip.shape[0] - (self.__config.ar_order + 1), 1\n ),\n axis=0,\n )\n\n # Check NWP data dimensions\n NWP_shape = self.__nwp_precip.shape\n NWP_timestamps_len = len(self.__nwp_timestamps)\n if not NWP_timestamps_len in NWP_shape:\n raise ValueError(\n f\"nwp_timestamps has not the same length as NWP data!\"\n f\"nwp_timestamps length: {NWP_timestamps_len}\"\n f\"nwp_precip shape: {NWP_shape}\"\n )\n\n # Ensure that model has shape: [n_ens_members, t, y, x]\n # n_ens_members and t can sometimes be swapped when using grib datasets.\n # Check for temporal resolution of NWP data\n if NWP_shape[0] == NWP_timestamps_len:\n self.__nwp_precip = self.__nwp_precip.swapaxes(0, 1)\n\n # Check dimensions of obs velocity\n if self.__obs_velocity.ndim != 3:\n raise ValueError(\n \"The velocity field must be a three-dimensional array of shape (2, m, n)\"\n )\n\n # Check whether the spatial dimensions match between obs precip and\n # obs velocity\n if self.__obs_precip.shape[1:3] != self.__obs_velocity.shape[1:3]:\n raise ValueError(\n f\"Spatial dimension of Precipitation observation and the\"\n \"velocity field do not match: \"\n f\"{self.__obs_precip.shape[1:3]} vs. {self.__obs_velocity.shape[1:3]}\"\n )\n\n # Check velocity field for non-finite values\n if np.any(~np.isfinite(self.__obs_velocity)):\n raise ValueError(\"Velocity contains non-finite values\")\n\n # Check whether there are extrapolation kwargs\n if self.__config.extrapolation_kwargs is None:\n self.__params.extrapolation_kwargs = dict()\n else:\n self.__params.extrapolation_kwargs = deepcopy(\n self.__config.extrapolation_kwargs\n )\n\n # Check whether there are filter kwargs\n if self.__config.filter_kwargs is None:\n self.__params.filter_kwargs = dict()\n else:\n self.__params.filter_kwargs = deepcopy(self.__config.filter_kwargs)\n\n # Check for noise kwargs\n if self.__config.noise_kwargs is None:\n self.__params.noise_kwargs = {\"win_fun\": \"tukey\"}\n else:\n self.__params.noise_kwargs = deepcopy(self.__config.noise_kwargs)\n\n # Check for combination kwargs\n if self.__config.combination_kwargs is None:\n self.__params.combination_kwargs = dict()\n else:\n self.__params.combination_kwargs = deepcopy(\n self.__config.combination_kwargs\n )\n\n # Set the precipitation threshold also in params\n self.__params.precip_threshold = self.__config.precip_threshold\n\n # Check for the standard deviation adjustment of the noise fields\n if self.__config.noise_stddev_adj not in [\"auto\", \"fixed\", None]:\n raise ValueError(\n f\"Unknown noise_std_dev_adj method {self.__config.noise_stddev_adj}. \"\n \"Must be 'auto', 'fixed', or None\"\n )\n\n def __check_input_timestamps(self):\n \"\"\"\n Check for timestamps of radar data and NWP data, determine forecasts and\n correction timesteps as well as the temporal resolution of the combined forecast\n \"\"\"\n\n # Check for temporal resolution of radar data\n obs_time_diff = np.unique(np.diff(self.__obs_timestamps))\n if obs_time_diff.size > 1:\n raise ValueError(\n \"Observation data has a different temporal resolution or \"\n \"observations are missing!\"\n )\n self.__temporal_res = int(obs_time_diff[0].total_seconds() / 60)\n\n # Check for temporal resolution of NWP data\n nwp_time_diff = np.unique(np.diff(self.__nwp_timestamps))\n if nwp_time_diff.size > 1:\n raise ValueError(\n \"NWP data has a different temporal resolution or some time steps are missing!\"\n )\n nwp_temporal_res = int(nwp_time_diff[0].total_seconds() / 60)\n\n # Check whether all necessary timesteps are included in the observation\n if self.__obs_timestamps[-1] != self.__fc_init:\n raise ValueError(\n \"The last observation timestamp differs from forecast issue time!\"\n )\n if self.__obs_timestamps.size < self.__config.ar_order + 1:\n raise ValueError(\n f\"Precipitation observation must have at least \"\n f\"{self.__config.ar_order + 1} time steps in the first\"\n f\"dimension to match the autoregressive order \"\n f\"(ar_order={self.__config.ar_order})\"\n )\n\n # Check whether the NWP forecasts includes the combined forecast range\n if np.logical_or(\n self.__fc_init < self.__nwp_timestamps[0],\n self.__fc_init > self.__nwp_timestamps[-1],\n ):\n raise ValueError(\"Forecast issue time is not included in the NWP forecast!\")\n\n max_nwp_fc_period = (\n self.__nwp_timestamps.size\n - np.where(self.__nwp_timestamps == self.__fc_init)[0][0]\n - 1\n ) * nwp_temporal_res\n if max_nwp_fc_period < self.__fc_period - nwp_temporal_res:\n raise ValueError(\n \"The remaining NWP forecast is not sufficient for the combined forecast period\"\n )\n\n # Truncate the NWP dataset if there sufficient remaining timesteps are available\n self.__nwp_precip = np.delete(\n self.__nwp_precip,\n np.logical_or(\n self.__nwp_timestamps < self.__fc_init,\n self.__nwp_timestamps\n > self.__fc_init + datetime.timedelta(minutes=self.__fc_period),\n ),\n axis=1,\n )\n\n # Define forecast and correction timesteps assuming that temporal resolution of\n # the combined forecast is equal to that of the radar data\n self.__forecast_leadtimes = np.arange(\n 0, self.__fc_period + 1, self.__temporal_res\n )\n trunc_nwp_timestamps = (\n self.__nwp_timestamps[\n np.logical_and(\n self.__nwp_timestamps >= self.__fc_init,\n self.__nwp_timestamps\n <= self.__fc_init + datetime.timedelta(minutes=self.__fc_period),\n )\n ]\n - self.__fc_init\n )\n self.__correction_leadtimes = np.array(\n [int(timestamp.total_seconds() / 60) for timestamp in trunc_nwp_timestamps]\n )\n\n def __check_no_rain_case(self):\n print(\"Test for no rain cases\")\n print(\"======================\")\n print(\"\")\n\n # Check for zero input fields in the radar and NWP data.\n zero_precip_radar = check_norain(\n self.__obs_precip,\n self.__config.precip_threshold,\n self.__config.norain_threshold,\n self.__params.noise_kwargs[\"win_fun\"],\n )\n # The norain fraction threshold used for nwp is the default value of 0.0,\n # since nwp does not suffer from clutter.\n zero_precip_nwp_forecast = check_norain(\n self.__nwp_precip,\n self.__config.precip_threshold,\n self.__config.norain_threshold,\n self.__params.noise_kwargs[\"win_fun\"],\n )\n\n # If there is no precipitation in the observation, set no_rain_case to \"obs\"\n # and use only the NWP ensemble forecast\n if zero_precip_radar:\n self.__params.no_rain_case = \"obs\"\n # If there is no precipitation at the first usable NWP forecast timestep, but\n # in the observation, compute an extrapolation forecast\n elif zero_precip_nwp_forecast:\n self.__params.no_rain_case = \"nwp\"\n # Otherwise, set no_rain_case to 'none' and compute combined forecast as usual\n else:\n self.__params.no_rain_case = \"none\"\n\n return\n\n def __print_forecast_info(self):\n \"\"\"\n Print information about the forecast configuration, including inputs, methods,\n and parameters.\n \"\"\"\n print(\"Reduced-space ensemble Kalman filter\")\n print(\"====================================\")\n print(\"\")\n\n print(\"Inputs\")\n print(\"------\")\n print(f\"Forecast issue time: {self.__fc_init.isoformat()}\")\n print(\n f\"Input dimensions: {self.__obs_precip.shape[1]}x{self.__obs_precip.shape[2]}\"\n )\n print(f\"Temporal resolution: {self.__temporal_res} minutes\")\n print(\"\")\n\n print(\"NWP and blending inputs\")\n print(\"-----------------------\")\n print(f\"Number of (NWP) models: {self.__nwp_precip.shape[0]}\")\n print(\"\")\n\n print(\"Methods\")\n print(\"-------\")\n print(\n f\"Extrapolation: {self.__config.extrapolation_method}\"\n )\n print(\n f\"Bandpass filter: {self.__config.bandpass_filter_method}\"\n )\n print(\n f\"Decomposition: {self.__config.decomposition_method}\"\n )\n print(f\"Noise generator: {self.__config.noise_method}\")\n print(\n f\"Noise adjustment: {'yes' if self.__config.noise_stddev_adj else 'no'}\"\n )\n\n print(f\"EnKF implementation: {self.__config.enkf_method}\")\n\n print(f\"FFT method: {self.__config.fft_method}\")\n print(f\"Domain: {self.__config.domain}\")\n print(\"\")\n\n print(\"Parameters\")\n print(\"----------\")\n print(f\"Forecast length in min: {self.__fc_period}\")\n print(f\"Ensemble size: {self.__config.n_ens_members}\")\n print(f\"Parallel threads: {self.__config.num_workers}\")\n print(f\"Number of cascade levels: {self.__config.n_cascade_levels}\")\n print(f\"Order of the AR(p) model: {self.__config.ar_order}\")\n print(\"\")\n\n print(f\"No rain forecast: {self.__params.no_rain_case}\")\n\n def __integrated_nowcast_main_loop(self):\n if self.__config.measure_time:\n starttime_mainloop = time.time()\n\n self.__params.extrapolation_kwargs[\"return_displacement\"] = True\n is_correction_timestep = False\n\n for t, fc_leadtime in enumerate(self.__forecast_leadtimes):\n if self.__config.measure_time:\n starttime = time.time()\n\n # Check whether forecast time step is also a correction time step.\n is_correction_timestep = (\n self.__forecast_leadtimes[t - 1] in self.__correction_leadtimes\n and t > 1\n and np.logical_and(\n self.__config.enable_combination,\n self.__params.no_rain_case != \"nwp\",\n )\n )\n\n # Check whether forecast time step is a nowcasting time step.\n is_nowcasting_timestep = t > 0\n\n # Check whether full NWP weight is reached.\n is_full_nwp_weight = (\n self.KalmanFilterModel.get_inflation_factor_obs() <= 0.02\n or self.__params.no_rain_case == \"obs\"\n )\n\n # If full NWP weight is reached, set pure NWP ensemble forecast in combined\n # forecast output\n if is_full_nwp_weight:\n # Set t_corr to the first available NWP data timestep and that is 0\n try:\n t_corr\n except NameError:\n t_corr = 0\n\n print(f\"Full NWP weight is reached for lead time + {fc_leadtime} min\")\n if is_correction_timestep:\n t_corr = np.where(\n self.__correction_leadtimes == self.__forecast_leadtimes[t]\n )[0][0]\n self.FS.nwc_prediction = self.__nwp_precip[:, t_corr]\n\n # Otherwise compute the combined forecast.\n else:\n print(f\"Computing combination for lead time + {fc_leadtime} min\")\n self.__forecast_loop(t, is_correction_timestep, is_nowcasting_timestep)\n\n # Apply back transformation\n if self.__config.smooth_radar_mask_range == 0:\n for j, FC_Model in enumerate(self.FC_Models.values()):\n FC_Model.backtransform()\n else:\n try:\n t_fill_nwp\n except NameError:\n t_fill_nwp = 0\n if self.__forecast_leadtimes[t] in self.__correction_leadtimes:\n t_fill_nwp = np.where(\n self.__correction_leadtimes == self.__forecast_leadtimes[t]\n )[0][0]\n\n def worker(j):\n self.FC_Models[j].fill_backtransform(\n self.__nwp_precip[j, t_fill_nwp]\n )\n\n dask_worker_collection = []\n\n if DASK_IMPORTED and self.__config.n_ens_members > 1:\n for j in range(self.__config.n_ens_members):\n dask_worker_collection.append(dask.delayed(worker)(j))\n dask.compute(\n *dask_worker_collection,\n num_workers=self.__params.num_ensemble_workers,\n )\n else:\n for j in range(self.__config.n_ens_members):\n worker(j)\n\n dask_worker_collection = None\n\n self.__write_output()\n\n if self.__config.measure_time:\n _ = self.__measure_time(\"timestep\", starttime)\n else:\n print(\"...done.\")\n\n if self.__config.measure_time:\n self.__mainloop_time = time.time() - starttime_mainloop\n print(\n f\"Elapsed time for computing forecast: {(self.__mainloop_time / 60.0):.4} min\"\n )\n\n return\n\n def __forecast_loop(self, t, is_correction_timestep, is_nowcasting_timestep):\n # If the temporal resolution of the NWP data is equal to those of the\n # observation, the correction step can be applied after the forecast\n # step for the current forecast leadtime.\n # However, if the temporal resolution is different, the correction step\n # has to be applied before the forecast step to avoid smoothing effects\n # in the resulting precipitation fields.\n if is_correction_timestep:\n t_corr = np.where(\n self.__correction_leadtimes == self.__forecast_leadtimes[t - 1]\n )[0][0]\n\n if self.__config.verbose_output:\n self.FS.background_ensemble[self.__correction_leadtimes[t_corr]] = (\n self.FS.nwc_prediction.copy()\n )\n\n self.FS.nwc_prediction, self.FS.fc_resampled = (\n self.KalmanFilterModel.correct_step(\n self.FS.nwc_prediction,\n self.__nwp_precip[:, t_corr],\n self.FS.fc_resampled,\n )\n )\n\n # Run nowcasting time step\n if is_nowcasting_timestep:\n # Set t_corr to the first available NWP data timestep and that is 0\n try:\n t_corr\n except NameError:\n t_corr = 0\n\n def worker(j):\n self.FC_Models[j].run_forecast_step(\n nwp=self.__nwp_precip[j, t_corr],\n is_correction_timestep=is_correction_timestep,\n )\n\n dask_worker_collection = []\n\n if DASK_IMPORTED and self.__config.n_ens_members > 1:\n for j in range(self.__config.n_ens_members):\n dask_worker_collection.append(dask.delayed(worker)(j))\n dask.compute(\n *dask_worker_collection,\n num_workers=self.__params.num_ensemble_workers,\n )\n else:\n for j in range(self.__config.n_ens_members):\n worker(j)\n\n dask_worker_collection = None\n\n def __write_output(self):\n if (\n self.__config.callback is not None\n and self.FS.nwc_prediction_btf.shape[1] > 0\n ):\n self.__config.callback(self.FS.nwc_prediction_btf)\n\n if self.__config.return_output:\n self.FS.final_combined_forecast.append(self.FS.nwc_prediction_btf.copy())\n\n def __measure_time(self, label, start_time):\n \"\"\"\n Measure and print the time taken for a specific part of the process.\n\n Parameters:\n - label: A description of the part of the process being measured.\n - start_time: The timestamp when the process started (from time.time()).\n \"\"\"\n if self.__config.measure_time:\n elapsed_time = time.time() - start_time\n print(f\"{label} took {elapsed_time:.2f} seconds.\")\n return elapsed_time\n return None\n\n\ndef forecast(\n obs_precip,\n obs_timestamps,\n nwp_precip,\n nwp_timestamps,\n velocity,\n forecast_horizon,\n issuetime,\n n_ens_members,\n precip_mask_dilation=1,\n smooth_radar_mask_range=0,\n n_cascade_levels=6,\n precip_thr=-10.0,\n norain_thr=0.01,\n extrap_method=\"semilagrangian\",\n decomp_method=\"fft\",\n bandpass_filter_method=\"gaussian\",\n noise_method=\"nonparametric\",\n enkf_method=\"masked_enkf\",\n enable_combination=True,\n noise_stddev_adj=None,\n ar_order=1,\n callback=None,\n return_output=True,\n seed=None,\n num_workers=1,\n fft_method=\"numpy\",\n domain=\"spatial\",\n extrap_kwargs=None,\n filter_kwargs=None,\n noise_kwargs=None,\n combination_kwargs=None,\n measure_time=False,\n verbose_output=False,\n):\n \"\"\"\n Generate a combined nowcast ensemble by using the reduced-space ensemble Kalman\n filter method described in Nerini et al. 2019.\n\n Parameters\n ----------\n obs_precip: np.ndarray\n Array of shape (ar_order+1,m,n) containing the observed input precipitation\n fields ordered by timestamp from oldest to newst. The time steps between\n the inputs are assumed to be regular.\n obs_timestamps: np.ndarray\n Array of shape (ar_order+1) containing the corresponding time stamps of\n observed input precipitation fields as datetime objects.\n nwp_precip: np.ndarray\n Array of shape (n_ens,n_times,m,n) containing the (NWP) ensemble model\n forecast.\n nwp_timestamps: np.ndarray\n Array of shape (n_times) containing the corresponding time stamps of the\n (NWP) ensemble model forecast as datetime objects.\n velocity: np.ndarray\n Array of shape (2,m,n) containing the x- and y-components of the advection\n field. The velocities are based on the observed input precipitation fields\n and are assumed to represent one time step between the inputs. All values\n are required to be finite.\n forecast_horizon: int\n The length of the forecast horizon (the length of the forecast) in minutes.\n issuetime: datetime object\n Issuetime of the combined forecast to compute.\n n_ens_members: int\n The number of ensemble members to generate. This number should always be\n equal to or larger than the number of NWP ensemble members / number of\n NWP models.\n precip_mask_dilation: int\n Range by which the precipitation mask within the forecast step should be\n extended per time step. Defaults to 1.\n smooth_radar_mask_range: int, Default is 0.\n Method to smooth the transition between the radar-NWP-noise blend and the NWP-noise\n blend near the edge of the radar domain (radar mask), where the radar data is either\n not present anymore or is not reliable. If set to 0 (grid cells), this generates a\n normal forecast without smoothing. To create a smooth mask, this range should be a\n positive value, representing a buffer band of a number of pixels by which the mask\n is cropped and smoothed. The smooth radar mask removes the hard edges between NWP\n and radar in the final blended product. Typically, a value between 50 and 100 km\n can be used. 80 km generally gives good results.\n n_cascade_levels: int, optional\n The number of cascade levels to use. Defaults to 6, see issue #385 on GitHub.\n precip_thr: float, optional\n pecifies the threshold value for minimum observable precipitation\n intensity. Required if mask_method is not None or conditional is True.\n Defaults to -10.0.\n norain_thr: float\n Specifies the threshold value for the fraction of rainy (see above) pixels\n in the radar rainfall field below which we consider there to be no rain.\n Depends on the amount of clutter typically present. Defaults to -15.0.\n extrap_method: str, optional\n Name of the extrapolation method to use. See the documentation of\n :py:mod:`pysteps.extrapolation.interface`. Defaults to 'semilagrangian'.\n decomp_method: {'fft'}, optional\n Name of the cascade decomposition method to use. See the documentation\n of :py:mod:`pysteps.cascade.interface`. Defaults to 'fft'.\n bandpass_filter_method: {'gaussian', 'uniform'}, optional\n Name of the bandpass filter method to use with the cascade decomposition.\n See the documentation of :py:mod:`pysteps.cascade.interface`. Defaults to\n 'guassian'.\n noise_method: {'parametric','nonparametric','ssft','nested',None}, optional\n Name of the noise generator to use for perturbating the precipitation\n field. See the documentation of :py:mod:`pysteps.noise.interface`. If set to\n None, no noise is generated. Defaults to 'nonparametric'.\n enkf_method: {'masked_enkf}, optional\n Name of the ensemble Kalman filter method to use for the correction step.\n Currently, only 'masked_enkf' method is implemented that corresponds to the\n reduced-space ensemble Kalman filter technique described in Nerini et al. 2019.\n Defaults to 'masked_enkf'.\n enable_combination: bool, optional\n Flag to specify whether the correction step should be applied or a pure\n nowcasting ensemble should be computed. Defaults to True.\n noise_stddev_adj: {'auto','fixed',None}, optional\n Optional adjustment for the standard deviations of the noise fields added\n to each cascade level. This is done to compensate incorrect std. dev.\n estimates of casace levels due to presence of no-rain areas. 'auto'=use\n the method implemented in :py:func:`pysteps.noise.utils.\n compute_noise_stddev_adjs`.\n 'fixed'= use the formula given in :cite:`BPS2006` (eq. 6), None=disable\n noise std. dev adjustment.\n ar_order: int, optional\n The order of the autoregressive model to use. Must be 1, since only this order\n is currently implemented.\n callback: function, optional\n Optional function that is called after computation of each time step of\n the nowcast. The function takes one argument: a three-dimensional array\n of shape (n_ens_members,h,w), where h and w are the height and width\n of the input field precip, respectively. This can be used, for instance,\n writing the outputs into files.\n return_output: bool, optional\n Set to False to disable returning the outputs as numpy arrays. This can\n save memory if the intermediate results are written to output files using\n the callback function.\n num_workers: int, optional\n The number of workers to use for parallel computation. Applicable if dask\n is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it\n is advisable to disable OpenMP by setting the environment variable\n OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous\n threads.\n fft_method: str, optional\n A string defining the FFT method to use (see FFT methods in\n :py:func:`pysteps.utils.interface.get_method`).\n Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed,\n the recommended method is 'pyfftw'.\n domain: {\"spatial\", \"spectral\"}\n If \"spatial\", all computations are done in the spatial domain (the\n classical STEPS model). If \"spectral\", the AR(2) models and stochastic\n perturbations are applied directly in the spectral domain to reduce\n memory footprint and improve performance :cite:`PCH2019b`.\n extrap_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the extrapolation\n method. See the documentation of :py:func:`pysteps.extrapolation.interface`.\n filter_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the filter method.\n See the documentation of :py:mod:`pysteps.cascade.bandpass_filters`.\n noise_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the initializer of\n the noise generator. See the documentation of :py:mod:`pysteps.noise.\n fftgenerators`.\n combination_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the initializer of the\n ensemble Kalman filter method. See the documentation of\n :py:mod:`pysteps.blending.ens_kalman_filter_methods`.\n measure_time: bool\n If set to True, measure, print and return the computation time.\n verbose_output: bool\n If set to True, return additionally the background ensemble of the EnKF for further statistics.\n\n Returns\n -------\n out: np.ndarray\n If return_output is True, a four-dimensional array of shape\n (n_ens_members,num_timesteps,m,n) containing a time series of forecast\n precipitation fields for each ensemble member. Otherwise, a None value\n is returned. The time series starts from t0. The timestep is taken from the\n input precipitation fields precip.\n\n See also\n --------\n :py:mod:`pysteps.extrapolation.interface`, :py:mod:`pysteps.cascade.interface`,\n :py:mod:`pysteps.noise.interface`, :py:func:`pysteps.noise.utils.\n compute_noise_stddev_adjs`\n\n References\n ----------\n :cite:`Nerini2019MWR`\n\n Notes\n -----\n 1. The combination method currently supports only an AR(1) process for the\n forecast step.\n \"\"\"\n\n combination_config = EnKFCombinationConfig(\n n_ens_members=n_ens_members,\n n_cascade_levels=n_cascade_levels,\n precip_threshold=precip_thr,\n norain_threshold=norain_thr,\n precip_mask_dilation=precip_mask_dilation,\n smooth_radar_mask_range=smooth_radar_mask_range,\n extrapolation_method=extrap_method,\n decomposition_method=decomp_method,\n bandpass_filter_method=bandpass_filter_method,\n noise_method=noise_method,\n enkf_method=enkf_method,\n enable_combination=enable_combination,\n noise_stddev_adj=noise_stddev_adj,\n ar_order=ar_order,\n seed=seed,\n num_workers=num_workers,\n fft_method=fft_method,\n domain=domain,\n extrapolation_kwargs=extrap_kwargs,\n filter_kwargs=filter_kwargs,\n noise_kwargs=noise_kwargs,\n combination_kwargs=combination_kwargs,\n measure_time=measure_time,\n verbose_output=verbose_output,\n callback=callback,\n return_output=return_output,\n n_noise_fields=30,\n )\n\n combination_nowcaster = EnKFCombinationNowcaster(\n obs_precip=obs_precip,\n obs_timestamps=obs_timestamps,\n nwp_precip=nwp_precip,\n nwp_timestamps=nwp_timestamps,\n obs_velocity=velocity,\n fc_period=forecast_horizon,\n fc_init=issuetime,\n enkf_combination_config=combination_config,\n )\n\n forecast_enkf_combination = combination_nowcaster.compute_forecast()\n\n return forecast_enkf_combination\n" }, { "path": "pysteps/blending/skill_scores.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.blending.skill_scores\n==============================\n\nMethods for computing skill scores, needed for the blending weights, of two-\ndimensional model fields with the latest observation field.\n\n.. autosummary::\n :toctree: ../generated/\n\n spatial_correlation\n lt_dependent_cor_nwp\n lt_dependent_cor_extrapolation\n clim_regr_values\n\"\"\"\n\nimport numpy as np\nfrom pysteps.blending import clim\n\n\ndef spatial_correlation(obs, mod, domain_mask):\n \"\"\"Determine the spatial correlation between the cascade of the latest\n available observed (radar) rainfall field and a time-synchronous cascade\n derived from a model (generally NWP) field. Both fields are assumed to use\n the same grid.\n\n\n Parameters\n ----------\n obs : array-like\n Array of shape [cascade_level, y, x] with per cascade_level the\n normalized cascade of the observed (radar) rainfall field.\n mod : array-like\n Array of shape [cascade_level, y, x] with per cascade_level the\n normalized cascade of the model field.\n domain_mask : array-like\n Boolean array of shape [y, x] indicating which cells fall outside the\n radar domain.\n\n Returns\n -------\n rho : array-like\n Array of shape [n_cascade_levels] containing per cascade_level the\n correlation between the normalized cascade of the observed (radar)\n rainfall field and the normalized cascade of the model field.\n\n References\n ----------\n :cite:`BPS2006`\n :cite:`SPN2013`\n\n \"\"\"\n rho = []\n # Fill rho per cascade level, so loop through the cascade levels\n for cascade_level in range(0, obs.shape[0]):\n # Only calculate the skill for the pixels within the radar domain\n # (as that is where there are observations)\n obs_cascade_level = obs[cascade_level, :, :]\n mod_cascade_level = mod[cascade_level, :, :]\n obs_cascade_level[domain_mask] = np.nan\n mod_cascade_level[domain_mask] = np.nan\n\n # Flatten both arrays\n obs_1d = obs_cascade_level.flatten()\n mod_1d = mod_cascade_level.flatten()\n # Calculate the correlation between the two\n cov = np.nansum(\n (mod_1d - np.nanmean(mod_1d)) * (obs_1d - np.nanmean(obs_1d))\n ) # Without 1/n, as this cancels out (same for stdx and -y)\n std_obs = np.sqrt(np.nansum((obs_1d - np.nanmean(obs_1d)) ** 2.0))\n std_mod = np.sqrt(np.nansum((mod_1d - np.nanmean(mod_1d)) ** 2.0))\n rho.append(cov / (std_mod * std_obs))\n\n # Make sure rho is always a (finite) number\n rho = np.nan_to_num(rho, copy=True, nan=10e-5, posinf=10e-5, neginf=10e-5)\n\n return rho\n\n\ndef lt_dependent_cor_nwp(lt, correlations, outdir_path, n_model=0, skill_kwargs=None):\n \"\"\"Determine the correlation of a model field for lead time lt and\n cascade k, by assuming that the correlation determined at t=0 regresses\n towards the climatological values.\n\n\n Parameters\n ----------\n lt : int\n The lead time of the forecast in minutes.\n correlations : array-like\n Array of shape [n_cascade_levels] containing per cascade_level the\n correlation between the normalized cascade of the observed (radar)\n rainfall field and the normalized cascade of the model field.\n outdir_path: string\n Path to folder where the historical skill are stored. Defaults to\n path_workdir from rcparams.\n n_model: int, optional\n The index number of the (NWP) model when the climatological skill of\n multiple (NWP) models is stored. For calculations with one model, or\n when n_model is not provided, n_model = 0.\n skill_kwargs : dict, optional\n Dictionary containing e.g. the outdir_path, nmodels and window_length\n parameters.\n\n Returns\n -------\n rho : array-like\n Array of shape [n_cascade_levels] containing, for lead time lt, per\n cascade_level the correlation between the normalized cascade of the\n observed (radar) rainfall field and the normalized cascade of the\n model field.\n\n References\n ----------\n :cite:`BPS2004`\n :cite:`BPS2006`\n \"\"\"\n\n if skill_kwargs is None:\n skill_kwargs = dict()\n\n # Obtain the climatological values towards which the correlations will\n # regress\n clim_cor_values, regr_pars = clim_regr_values(\n n_cascade_levels=len(correlations),\n outdir_path=outdir_path,\n n_model=n_model,\n skill_kwargs=skill_kwargs,\n )\n # Determine the speed of the regression (eq. 24 in BPS2004)\n qm = np.exp(-lt / regr_pars[0, :]) * (2 - np.exp(-lt / regr_pars[1, :]))\n # Determine the correlation for lead time lt\n rho = qm * correlations + (1 - qm) * clim_cor_values\n\n return rho\n\n\ndef lt_dependent_cor_extrapolation(PHI, correlations=None, correlations_prev=None):\n \"\"\"Determine the correlation of the extrapolation (nowcast) component for\n lead time lt and cascade k, by assuming that the correlation determined at\n t=0 regresses towards the climatological values.\n\n\n Parameters\n ----------\n PHI : array-like\n Array of shape [n_cascade_levels, ar_order + 1] containing per\n cascade level the autoregression parameters.\n correlations : array-like, optional\n Array of shape [n_cascade_levels] containing per cascade_level the\n latest available correlation from the extrapolation component that can\n be found from the AR-2 model.\n correlations_prev : array-like, optional\n Similar to correlations, but from the timestep before that.\n\n Returns\n -------\n rho : array-like\n Array of shape [n_cascade_levels] containing, for lead time lt, per\n cascade_level the correlation of the extrapolation component.\n\n References\n ----------\n :cite:`BPS2004`\n :cite:`BPS2006`\n\n \"\"\"\n # Check if correlations_prev exists, if not, we set it to 1.0\n if correlations_prev is None:\n correlations_prev = np.repeat(1.0, PHI.shape[0])\n # Same for correlations at first time step, we set it to\n # phi1 / (1 - phi2), see BPS2004\n if correlations is None:\n correlations = PHI[:, 0] / (1.0 - PHI[:, 1])\n\n # Calculate the correlation for lead time lt\n rho = PHI[:, 0] * correlations + PHI[:, 1] * correlations_prev\n\n # Finally, set the current correlations array as the previous one for the\n # next time step\n rho_prev = correlations\n\n return rho, rho_prev\n\n\ndef clim_regr_values(n_cascade_levels, outdir_path, n_model=0, skill_kwargs=None):\n \"\"\"Obtains the climatological correlation values and regression parameters\n from a file called NWP_weights_window.bin in the outdir_path. If this file\n is not present yet, the values from :cite:`BPS2004` are used.\n\n\n Parameters\n ----------\n n_cascade_levels : int\n The number of cascade levels to use.\n outdir_path: string\n Path to folder where the historical skill are stored. Defaults to\n path_workdir from rcparams.\n n_model: int, optional\n The index number of the (NWP) model when the climatological skill of\n multiple (NWP) models is stored. For calculations with one model, or\n when n_model is not provided, n_model = 0.\n skill_kwargs : dict, optional\n Dictionary containing e.g. the outdir_path, nmodels and window_length\n parameters.\n\n Returns\n -------\n clim_cor_values : array-like\n Array of shape [n_cascade_levels] containing the\n climatological values of the lag 1 and lag 2 auto-correlation\n coefficients, obtained by calling a method implemented in\n pysteps.blending.skill_scores.get_clim_skill_scores.\n regr_pars : array-like\n Array of shape [2, n_cascade_levels] containing the regression\n parameters. These are fixed values that should be hard-coded in this\n function. Unless changed by the user, the standard values from\n `BPS2004` are used.\n\n Notes\n -----\n The literature climatological values assume 8 cascade levels. In case\n less or more cascade levels are used, the script will handle this by taking\n the first n values or extending the array with a small value. This is not\n ideal, but will be fixed once the clim_regr_file is made. Hence, this\n requires a warm-up period of the model.\n In addition, the regression parameter values (eq. 24 in BPS2004) are hard-\n coded and can only be optimized by the user after (re)fitting of the\n equation.\n\n \"\"\"\n\n if skill_kwargs is None:\n skill_kwargs = {\"n_models\": 1}\n\n # First, obtain climatological skill values\n try:\n clim_cor_values = clim.calc_clim_skill(\n outdir_path=outdir_path, n_cascade_levels=n_cascade_levels, **skill_kwargs\n )\n except FileNotFoundError:\n # The climatological skill values file does not exist yet, so we'll\n # use the default values from BPS2004.\n clim_cor_values = clim.get_default_skill(\n n_cascade_levels=n_cascade_levels, n_models=skill_kwargs[\"n_models\"]\n )\n\n clim_cor_values = clim_cor_values[n_model, :]\n\n # Check if clim_cor_values has only one model, otherwise it has\n # returned the skill values for multiple models\n if clim_cor_values.ndim != 1:\n raise IndexError(\n \"The climatological cor. values of multiple models were returned, but only one model should be specified. Make sure to pass the argument nmodels in the function\"\n )\n\n # Also check whether the number of cascade_levels is correct\n if clim_cor_values.shape[0] > n_cascade_levels:\n clim_cor_values = clim_cor_values[0:n_cascade_levels]\n elif clim_cor_values.shape[0] < n_cascade_levels:\n # Get the number of cascade levels that is missing\n n_extra_lev = n_cascade_levels - clim_cor_values.shape[0]\n # Append the array with correlation values of 10e-4\n clim_cor_values = np.append(clim_cor_values, np.repeat(1e-4, n_extra_lev))\n\n # Get the regression parameters (L in eq. 24 in BPS2004) - Hard coded,\n # change to own values when present.\n regr_pars = np.array(\n [\n [130.0, 165.0, 120.0, 55.0, 50.0, 15.0, 15.0, 10.0],\n [155.0, 220.0, 200.0, 75.0, 10e4, 10e4, 10e4, 10e4],\n ]\n )\n\n # Check whether the number of cascade_levels is correct\n if regr_pars.shape[1] > n_cascade_levels:\n regr_pars = regr_pars[:, 0:n_cascade_levels]\n elif regr_pars.shape[1] < n_cascade_levels:\n # Get the number of cascade levels that is missing\n n_extra_lev = n_cascade_levels - regr_pars.shape[1]\n # Append the array with correlation values of 10e-4\n regr_pars = np.append(\n regr_pars,\n [np.repeat(10.0, n_extra_lev), np.repeat(10e4, n_extra_lev)],\n axis=1,\n )\n\n return clim_cor_values, regr_pars\n" }, { "path": "pysteps/blending/steps.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.blending.steps\n======================\n\nImplementation of the STEPS stochastic blending method as described in\n:cite:`BPS2004`, :cite:`BPS2006`, :cite:`SPN2013` and :cite:`Imhoff2023`.\nThe STEPS blending method consists of the following main steps:\n\n #. Set the radar rainfall fields in a Lagrangian space.\n #. Perform the cascade decomposition for the input radar rainfall fields.\n The method assumes that the cascade decomposition of the NWP model fields is\n already done prior to calling the function, as the NWP model fields are\n generally not updated with the same frequency (which is more efficient). A\n method to decompose and store the NWP model fields whenever a new NWP model\n field is present, is present in pysteps.blending.utils.decompose_NWP.\n #. Initialize the noise method (this will be bypassed if a deterministic nowcast\n is provided and n_ens_members is 1).\n #. Estimate AR parameters for the extrapolation nowcast and noise cascade.\n #. Initialize all the random generators.\n #. Calculate the initial skill of the NWP model forecasts at t=0.\n #. Start the forecasting loop:\n #. Determine which NWP models will be combined with which nowcast ensemble\n member. The number of output ensemble members equals the maximum number\n of (ensemble) members in the input, which can be either the defined\n number of (nowcast) ensemble members or the number of NWP models/members.\n #. Determine the skill and weights of the forecasting components\n (extrapolation, NWP and noise) for that lead time.\n #. Regress the extrapolation and noise cascades separately to the subsequent\n time step.\n #. Extrapolate the extrapolation and noise cascades to the current time step.\n #. Blend the cascades.\n #. Recompose the cascade to a rainfall field.\n #. Post-processing steps (masking and probability matching, which are\n different from the original blended STEPS implementation).\n\n.. autosummary::\n :toctree: ../generated/\n\n forecast\n calculate_ratios\n calculate_weights_bps\n calculate_weights_spn\n blend_means_sigmas\n\"\"\"\n\nimport math\nimport time\nfrom copy import copy, deepcopy\nfrom functools import partial\nfrom multiprocessing.pool import ThreadPool\n\nimport numpy as np\nfrom scipy.linalg import inv\nfrom scipy.ndimage import binary_dilation, generate_binary_structure, iterate_structure\n\nfrom pysteps import blending, cascade, extrapolation, noise, utils\nfrom pysteps.nowcasts import utils as nowcast_utils\nfrom pysteps.postprocessing import probmatching\nfrom pysteps.timeseries import autoregression, correlation\nfrom pysteps.utils.check_norain import check_norain\n\ntry:\n import dask\n\n DASK_IMPORTED = True\nexcept ImportError:\n DASK_IMPORTED = False\n\nfrom dataclasses import dataclass, field\nfrom typing import Any, Callable\n\n\n@dataclass(frozen=True)\nclass StepsBlendingConfig:\n \"\"\"\n Parameters\n ----------\n\n precip_threshold: float, optional\n Specifies the threshold value for minimum observable precipitation\n intensity. Required if mask_method is not None or conditional is True.\n norain_threshold: float\n Specifies the threshold value for the fraction of rainy (see above) pixels\n in the radar rainfall field below which we consider there to be no rain.\n Depends on the amount of clutter typically present.\n Standard set to 0.0\n kmperpixel: float, optional\n Spatial resolution of the input data (kilometers/pixel). Required if\n vel_pert_method is not None or mask_method is 'incremental'.\n timestep: float\n Time step of the motion vectors (minutes). Required if vel_pert_method is\n not None or mask_method is 'incremental'.\n n_ens_members: int\n The number of ensemble members to generate. This number should always be\n equal to or larger than the number of NWP ensemble members / number of\n NWP models.\n n_cascade_levels: int, optional\n The number of cascade levels to use. Defaults to 6,\n see issue #385 on GitHub.\n blend_nwp_members: bool\n Check if NWP models/members should be used individually, or if all of\n them are blended together per nowcast ensemble member. Standard set to\n false.\n extrapolation_method: str, optional\n Name of the extrapolation method to use. See the documentation of\n :py:mod:`pysteps.extrapolation.interface`.\n decomposition_method: {'fft'}, optional\n Name of the cascade decomposition method to use. See the documentation\n of :py:mod:`pysteps.cascade.interface`.\n bandpass_filter_method: {'gaussian', 'uniform'}, optional\n Name of the bandpass filter method to use with the cascade decomposition.\n See the documentation of :py:mod:`pysteps.cascade.interface`.\n nowcasting_method: {'steps', 'external_nowcast'},\n Name of the nowcasting method used to generate the nowcasts. If an external\n nowcast is provided, the script will use this as input and bypass the\n autoregression and advection of the extrapolation cascade. Defaults to 'steps',\n which follows the method described in :cite:`Imhoff2023`. Note, if\n nowcasting_method is 'external_nowcast', precip_nowcast cannot be None.\n noise_method: {'parametric','nonparametric','ssft','nested',None}, optional\n Name of the noise generator to use for perturbating the precipitation\n field. See the documentation of :py:mod:`pysteps.noise.interface`. If set to None,\n no noise is generated.\n noise_stddev_adj: {'auto','fixed',None}, optional\n Optional adjustment for the standard deviations of the noise fields added\n to each cascade level. This is done to compensate incorrect std. dev.\n estimates of casace levels due to presence of no-rain areas. 'auto'=use\n the method implemented in :py:func:`pysteps.noise.utils.compute_noise_stddev_adjs`.\n 'fixed'= use the formula given in :cite:`BPS2006` (eq. 6), None=disable\n noise std. dev adjustment.\n ar_order: int, optional\n The order of the autoregressive model to use. Must be >= 1.\n velocity_perturbation_method: {'bps',None}, optional\n Name of the noise generator to use for perturbing the advection field. See\n the documentation of :py:mod:`pysteps.noise.interface`. If set to None, the advection\n field is not perturbed.\n weights_method: {'bps','spn'}, optional\n The calculation method of the blending weights. Options are the method\n by :cite:`BPS2006` and the covariance-based method by :cite:`SPN2013`.\n Defaults to bps.\n timestep_start_full_nwp_weight: int, optional.\n The timestep, which should be smaller than timesteps, at which a linear\n transition takes place from the calculated weights to full NWP weight\n (and zero extrapolation and noise weight) to ensure the blending\n procedure becomes equal to the NWP forecast(s) at the last timestep\n of the blending procedure. If not provided, the blending stick to the\n theoretical weights provided by the chosen weights_method for a given\n lead time and skill of each blending component.\n conditional: bool, optional\n If set to True, compute the statistics of the precipitation field\n conditionally by excluding pixels where the values are below the threshold\n precip_thr.\n probmatching_method: {'cdf','mean',None}, optional\n Method for matching the statistics of the forecast field with those of\n the most recently observed one. 'cdf'=map the forecast CDF to the observed\n one, 'mean'=adjust only the conditional mean value of the forecast field\n in precipitation areas, None=no matching applied. Using 'mean' requires\n that mask_method is not None.\n mask_method: {'obs','incremental',None}, optional\n The method to use for masking no precipitation areas in the forecast field.\n The masked pixels are set to the minimum value of the observations.\n 'obs' = apply precip_thr to the most recently observed precipitation intensity\n field, 'incremental' = iteratively buffer the mask with a certain rate\n (currently it is 1 km/min), None=no masking.\n resample_distribution: bool, optional\n Method to resample the distribution from the extrapolation and NWP cascade as input\n for the probability matching. Not resampling these distributions may lead to losing\n some extremes when the weight of both the extrapolation and NWP cascade is similar.\n Defaults to True.\n smooth_radar_mask_range: int, Default is 0.\n Method to smooth the transition between the radar-NWP-noise blend and the NWP-noise\n blend near the edge of the radar domain (radar mask), where the radar data is either\n not present anymore or is not reliable. If set to 0 (grid cells), this generates a\n normal forecast without smoothing. To create a smooth mask, this range should be a\n positive value, representing a buffer band of a number of pixels by which the mask\n is cropped and smoothed. The smooth radar mask removes the hard edges between NWP\n and radar in the final blended product. Typically, a value between 50 and 100 km\n can be used. 80 km generally gives good results.\n seed: int, optional\n Optional seed number for the random generators.\n num_workers: int, optional\n The number of workers to use for parallel computation. Applicable if dask\n is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it\n is advisable to disable OpenMP by setting the environment variable\n OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous\n threads.\n fft_method: str, optional\n A string defining the FFT method to use (see FFT methods in\n :py:func:`pysteps.utils.interface.get_method`).\n Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed,\n the recommended method is 'pyfftw'.\n domain: {\"spatial\", \"spectral\"}\n If \"spatial\", all computations are done in the spatial domain (the\n classical STEPS model). If \"spectral\", the AR(2) models and stochastic\n perturbations are applied directly in the spectral domain to reduce\n memory footprint and improve performance :cite:`PCH2019b`.\n outdir_path_skill: string, optional\n Path to folder where the historical skill are stored. Defaults to\n path_workdir from rcparams. If no path is given, './tmp' will be used.\n extrapolation_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the extrapolation\n method. See the documentation of :py:func:`pysteps.extrapolation.interface`.\n filter_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the filter method.\n See the documentation of :py:mod:`pysteps.cascade.bandpass_filters`.\n noise_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the initializer of\n the noise generator. See the documentation of :py:mod:`pysteps.noise.fftgenerators`.\n velocity_perturbation_kwargs: dict, optional\n Optional dictionary containing keyword arguments 'p_par' and 'p_perp' for\n the initializer of the velocity perturbator. The choice of the optimal\n parameters depends on the domain and the used optical flow method.\n\n Default parameters from :cite:`BPS2006`:\n p_par = [10.88, 0.23, -7.68]\n p_perp = [5.76, 0.31, -2.72]\n\n Parameters fitted to the data (optical flow/domain):\n\n darts/fmi:\n p_par = [13.71259667, 0.15658963, -16.24368207]\n p_perp = [8.26550355, 0.17820458, -9.54107834]\n\n darts/mch:\n p_par = [24.27562298, 0.11297186, -27.30087471]\n p_perp = [-7.80797846e+01, -3.38641048e-02, 7.56715304e+01]\n\n darts/fmi+mch:\n p_par = [16.55447057, 0.14160448, -19.24613059]\n p_perp = [14.75343395, 0.11785398, -16.26151612]\n\n lucaskanade/fmi:\n p_par = [2.20837526, 0.33887032, -2.48995355]\n p_perp = [2.21722634, 0.32359621, -2.57402761]\n\n lucaskanade/mch:\n p_par = [2.56338484, 0.3330941, -2.99714349]\n p_perp = [1.31204508, 0.3578426, -1.02499891]\n\n lucaskanade/fmi+mch:\n p_par = [2.31970635, 0.33734287, -2.64972861]\n p_perp = [1.90769947, 0.33446594, -2.06603662]\n\n vet/fmi:\n p_par = [0.25337388, 0.67542291, 11.04895538]\n p_perp = [0.02432118, 0.99613295, 7.40146505]\n\n vet/mch:\n p_par = [0.5075159, 0.53895212, 7.90331791]\n p_perp = [0.68025501, 0.41761289, 4.73793581]\n\n vet/fmi+mch:\n p_par = [0.29495222, 0.62429207, 8.6804131 ]\n p_perp = [0.23127377, 0.59010281, 5.98180004]\n\n fmi=Finland, mch=Switzerland, fmi+mch=both pooled into the same data set\n\n The above parameters have been fitted by using run_vel_pert_analysis.py\n and fit_vel_pert_params.py located in the scripts directory.\n\n See :py:mod:`pysteps.noise.motion` for additional documentation.\n climatology_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the climatological\n skill file. Arguments can consist of: 'outdir_path', 'n_models'\n (the number of NWP models) and 'window_length' (the minimum number of\n days the clim file should have, otherwise the default is used).\n mask_kwargs: dict\n Optional dictionary containing mask keyword arguments 'mask_f',\n 'mask_rim' and 'max_mask_rim', the factor defining the the mask\n increment and the (maximum) rim size, respectively.\n The mask increment is defined as mask_f*timestep/kmperpixel.\n measure_time: bool\n If set to True, measure, print and return the computation time.\n callback: function, optional\n Optional function that is called after computation of each time step of\n the nowcast. The function takes one argument: a three-dimensional array\n of shape (n_ens_members,h,w), where h and w are the height and width\n of the input field precip, respectively. This can be used, for instance,\n writing the outputs into files.\n return_output: bool, optional\n Set to False to disable returning the outputs as numpy arrays. This can\n save memory if the intermediate results are written to output files using\n the callback function.\n \"\"\"\n\n precip_threshold: float | None\n norain_threshold: float\n kmperpixel: float\n timestep: float\n n_ens_members: int\n n_cascade_levels: int\n blend_nwp_members: bool\n extrapolation_method: str\n decomposition_method: str\n bandpass_filter_method: str\n nowcasting_method: str\n noise_method: str | None\n noise_stddev_adj: str | None\n ar_order: int\n velocity_perturbation_method: str | None\n weights_method: str\n timestep_start_full_nwp_weight: int | None\n conditional: bool\n probmatching_method: str | None\n mask_method: str | None\n resample_distribution: bool\n smooth_radar_mask_range: int\n seed: int | None\n num_workers: int\n fft_method: str\n domain: str\n outdir_path_skill: str\n extrapolation_kwargs: dict[str, Any] = field(default_factory=dict)\n filter_kwargs: dict[str, Any] = field(default_factory=dict)\n noise_kwargs: dict[str, Any] = field(default_factory=dict)\n velocity_perturbation_kwargs: dict[str, Any] = field(default_factory=dict)\n climatology_kwargs: dict[str, Any] = field(default_factory=dict)\n mask_kwargs: dict[str, Any] = field(default_factory=dict)\n measure_time: bool = False\n callback: Any | None = None\n return_output: bool = True\n\n\n@dataclass\nclass StepsBlendingParams:\n noise_std_coeffs: np.ndarray | None = None\n bandpass_filter: Any | None = None\n fft: Any | None = None\n perturbation_generator: Callable[..., np.ndarray] | None = None\n noise_generator: Callable[..., np.ndarray] | None = None\n PHI: np.ndarray | None = None\n extrapolation_method: Callable[..., Any] | None = None\n decomposition_method: Callable[..., dict] | None = None\n recomposition_method: Callable[..., np.ndarray] | None = None\n velocity_perturbations: Any | None = None\n generate_velocity_noise: Callable[[Any, float], np.ndarray] | None = None\n velocity_perturbations_parallel: np.ndarray | None = None\n velocity_perturbations_perpendicular: np.ndarray | None = None\n fft_objs: list[Any] = field(default_factory=list)\n mask_rim: int | None = None\n struct: np.ndarray | None = None\n time_steps_is_list: bool = False\n precip_models_provided_is_cascade: bool = False\n xy_coordinates: np.ndarray | None = None\n precip_zerovalue: float | None = None\n precip_threshold: float | None = None\n mask_threshold: np.ndarray | None = None\n zero_precip_radar: bool = False\n zero_precip_model_fields: bool = False\n original_timesteps: list | np.ndarray | None = None\n num_ensemble_workers: int | None = None\n rho_nwp_models: np.ndarray | None = None\n domain_mask: np.ndarray | None = None\n filter_kwargs: dict | None = None\n noise_kwargs: dict | None = None\n velocity_perturbation_kwargs: dict | None = None\n climatology_kwargs: dict | None = None\n mask_kwargs: dict | None = None\n\n\n@dataclass\nclass StepsBlendingState:\n # Radar and noise states\n precip_cascades: np.ndarray | None = None\n precip_nowcast_cascades: np.ndarray | None = None\n precip_noise_input: np.ndarray | None = None\n precip_noise_cascades: np.ndarray | None = None\n precip_mean_noise: np.ndarray | None = None\n precip_std_noise: np.ndarray | None = None\n\n # Extrapolation states\n mean_extrapolation: np.ndarray | None = None\n std_extrapolation: np.ndarray | None = None\n mean_nowcast: np.ndarray | None = None\n std_nowcast: np.ndarray | None = None\n mean_nowcast_timestep: np.ndarray | None = None\n std_nowcast_timestep: np.ndarray | None = None\n rho_extrap_cascade_prev: np.ndarray | None = None\n rho_extrap_cascade: np.ndarray | None = None\n precip_cascades_prev_subtimestep: np.ndarray | None = None\n cascade_noise_prev_subtimestep: np.ndarray | None = None\n precip_extrapolated_after_decomp: np.ndarray | None = None\n noise_extrapolated_after_decomp: np.ndarray | None = None\n precip_extrapolated_probability_matching: np.ndarray | None = None\n\n # NWP model states\n precip_models_cascades: np.ndarray | None = None\n precip_models_cascades_timestep: np.ndarray | None = None\n precip_models_timestep: np.ndarray | None = None\n mean_models_timestep: np.ndarray | None = None\n std_models_timestep: np.ndarray | None = None\n velocity_models_timestep: np.ndarray | None = None\n\n # Mapping from NWP members to ensemble members\n mapping_list_NWP_member_to_ensemble_member: np.ndarray | None = None\n\n # Random states for precipitation, motion and probmatching\n randgen_precip: list[np.random.RandomState] | None = None\n randgen_motion: list[np.random.RandomState] | None = None\n randgen_probmatching: list[np.random.RandomState] | None = None\n\n # Variables for final forecast computation\n previous_displacement: list[Any] | None = None\n previous_displacement_noise_cascade: list[Any] | None = None\n previous_displacement_prob_matching: list[Any] | None = None\n rho_final_blended_forecast: np.ndarray | None = None\n final_blended_forecast_means: np.ndarray | None = None\n final_blended_forecast_stds: np.ndarray | None = None\n final_blended_forecast_means_mod_only: np.ndarray | None = None\n final_blended_forecast_stds_mod_only: np.ndarray | None = None\n final_blended_forecast_cascades: np.ndarray | None = None\n final_blended_forecast_cascades_mod_only: np.ndarray | None = None\n final_blended_forecast_recomposed: np.ndarray | None = None\n final_blended_forecast_recomposed_mod_only: np.ndarray | None = None\n\n # Final outputs\n final_blended_forecast: np.ndarray | None = None\n final_blended_forecast_non_perturbed: np.ndarray | None = None\n weights: np.ndarray | None = None\n weights_model_only: np.ndarray | None = None\n\n # Timing and indexing\n time_prev_timestep: list[float] | None = None\n leadtime_since_start_forecast: list[float] | None = None\n subtimesteps: list[float] | None = None\n is_nowcast_time_step: bool | None = None\n subtimestep_index: int | None = None\n\n # Weights used for blending\n weights: np.ndarray | None = None\n weights_model_only: np.ndarray | None = None\n\n # This is stores here as well because this is changed during the forecast loop and thus no longer part of the config\n extrapolation_kwargs: dict[str, Any] = field(default_factory=dict)\n\n\nclass StepsBlendingNowcaster:\n def __init__(\n self,\n precip,\n precip_nowcast,\n precip_models,\n velocity,\n velocity_models,\n time_steps,\n issue_time,\n steps_blending_config: StepsBlendingConfig,\n ):\n \"\"\"Initializes the StepsBlendingNowcaster with inputs and configurations.\"\"\"\n # Store inputs\n self.__precip = precip\n self.__precip_nowcast = precip_nowcast\n self.__precip_models = precip_models\n self.__velocity = velocity\n self.__velocity_models = velocity_models\n self.__timesteps = time_steps\n self.__issuetime = issue_time\n\n self.__config = steps_blending_config\n\n # Initialize Params and State\n self.__params = StepsBlendingParams()\n self.__state = StepsBlendingState()\n\n # Additional variables for time measurement\n self.__start_time_init = None\n self.__init_time = None\n self.__mainloop_time = None\n\n def compute_forecast(self):\n \"\"\"\n Generate a blended nowcast ensemble by using the Short-Term Ensemble\n Prediction System (STEPS) method.\n\n Parameters\n ----------\n precip: array-like\n Array of shape (ar_order+1,m,n) containing the input precipitation fields\n ordered by timestamp from oldest to newest. The time steps between the\n inputs are assumed to be regular.\n precip_models: array-like\n Either raw (NWP) model forecast data or decomposed (NWP) model forecast data.\n If you supply decomposed data, it needs to be an array of shape\n (n_models,timesteps+1) containing, per timestep (t=0 to lead time here) and\n per (NWP) model or model ensemble member, a dictionary with a list of cascades\n obtained by calling a method implemented in :py:mod:`pysteps.cascade.decomposition`.\n If you supply the original (NWP) model forecast data, it needs to be an array of shape\n (n_models,timestep+1,m,n) containing precipitation (or other) fields, which will\n then be decomposed in this function.\n\n Depending on your use case it can be advantageous to decompose the model\n forecasts outside beforehand, as this slightly reduces calculation times.\n This is possible with :py:func:`pysteps.blending.utils.decompose_NWP`,\n :py:func:`pysteps.blending.utils.compute_store_nwp_motion`, and\n :py:func:`pysteps.blending.utils.load_NWP`. However, if you have a lot of (NWP) model\n members (e.g. 1 model member per nowcast member), this can lead to excessive memory\n usage.\n\n To further reduce memory usage, both this array and the ``velocity_models`` array\n can be given as float32. They will then be converted to float64 before computations\n to minimize loss in precision.\n\n In case of one (deterministic) model as input, add an extra dimension to make sure\n precip_models is four dimensional prior to calling this function.\n velocity: array-like\n Array of shape (2,m,n) containing the x- and y-components of the advection\n field. The velocities are assumed to represent one time step between the\n inputs. All values are required to be finite.\n velocity_models: array-like\n Array of shape (n_models,timestep,2,m,n) containing the x- and y-components\n of the advection field for the (NWP) model field per forecast lead time.\n All values are required to be finite.\n\n To reduce memory usage, this array\n can be given as float32. They will then be converted to float64 before computations\n to minimize loss in precision.\n time_steps: int or list of floats\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed (relative to the input time step). The elements of\n the list are required to be in ascending order.\n issue_time: datetime\n is issued.\n config: StepsBlendingConfig\n Provides a set of configuration parameters for the nowcast ensemble generation.\n\n Returns\n -------\n out: ndarray\n If return_output is True, a four-dimensional array of shape\n (n_ens_members,num_timesteps,m,n) containing a time series of forecast\n precipitation fields for each ensemble member. Otherwise, a None value\n is returned. The time series starts from t0+timestep, where timestep is\n taken from the input precipitation fields precip. If measure_time is True, the\n return value is a three-element tuple containing the nowcast array, the\n initialization time of the nowcast generator and the time used in the\n main loop (seconds).\n\n See also\n --------\n :py:mod:`pysteps.extrapolation.interface`, :py:mod:`pysteps.cascade.interface`,\n :py:mod:`pysteps.noise.interface`, :py:func:`pysteps.noise.utils.compute_noise_stddev_adjs`\n\n References\n ----------\n :cite:`Seed2003`, :cite:`BPS2004`, :cite:`BPS2006`, :cite:`SPN2013`, :cite:`PCH2019b`\n\n Notes\n -----\n 1. The blending currently does not blend the beta-parameters in the parametric\n noise method. It is recommended to use the non-parameteric noise method.\n\n 2. If blend_nwp_members is True, the BPS2006 method for the weights is\n suboptimal. It is recommended to use the SPN2013 method instead.\n\n 3. Not yet implemented (and neither in the steps nowcasting module): The regression\n of the lag-1 and lag-2 parameters to their climatological values. See also eq.\n 12 - 19 in :cite: `BPS2004`. By doing so, the Phi parameters change over time,\n which enhances the AR process. This can become a future development if this\n turns out to be a warranted functionality.\n \"\"\"\n\n self.__check_inputs()\n self.__print_forecast_info()\n # Measure time for initialization\n if self.__config.measure_time:\n self.__start_time_init = time.time()\n\n # Slice the precipitation field to only use the last ar_order + 1 fields\n self.__precip = self.__precip[-(self.__config.ar_order + 1) :, :, :].copy()\n self.__initialize_nowcast_components()\n self.__prepare_radar_and_NWP_fields()\n\n # Determine if rain is present in both radar and NWP fields\n if self.__params.zero_precip_radar and self.__params.zero_precip_model_fields:\n return self.__zero_precipitation_forecast()\n else:\n # Prepare the data for the zero precipitation radar case and initialize the noise correctly\n if self.__params.zero_precip_radar:\n self.__prepare_nowcast_for_zero_radar()\n else:\n self.__state.precip_noise_input = self.__precip.copy()\n self.__initialize_noise()\n self.__estimate_ar_parameters_radar()\n self.__multiply_precip_cascade_to_match_ensemble_members()\n self.__initialize_random_generators()\n self.__prepare_forecast_loop()\n self.__initialize_noise_cascades()\n if self.__config.measure_time:\n self.__init_time = self.__measure_time(\n \"initialization\", self.__start_time_init\n )\n\n self.__blended_nowcast_main_loop()\n # Stack and return the forecast output\n if self.__config.return_output:\n self.__state.final_blended_forecast = np.stack(\n [\n np.stack(self.__state.final_blended_forecast[j])\n for j in range(self.__config.n_ens_members)\n ]\n )\n if self.__config.measure_time:\n return (\n self.__state.final_blended_forecast,\n self.__init_time,\n self.__mainloop_time,\n )\n else:\n return self.__state.final_blended_forecast\n else:\n return None\n\n def __blended_nowcast_main_loop(self):\n \"\"\"\n Main nowcast loop that iterates through the ensemble members and time steps\n to generate forecasts.\n \"\"\"\n # Isolate the last time slice of observed precipitation\n self.__precip = self.__precip[-1, :, :]\n print(\"Starting blended nowcast computation.\")\n\n if self.__config.measure_time:\n starttime_mainloop = time.time()\n self.__state.extrapolation_kwargs[\"return_displacement\"] = True\n\n self.__state.precip_cascades_prev_subtimestep = deepcopy(\n self.__state.precip_cascades\n )\n self.__state.cascade_noise_prev_subtimestep = deepcopy(\n self.__state.precip_noise_cascades\n )\n\n self.__state.time_prev_timestep = [\n 0.0 for j in range(self.__config.n_ens_members)\n ]\n self.__state.leadtime_since_start_forecast = [\n 0.0 for j in range(self.__config.n_ens_members)\n ]\n\n # iterate each time step\n for t, subtimestep_idx in enumerate(self.__timesteps):\n self.__determine_subtimesteps_and_nowcast_time_step(t, subtimestep_idx)\n if self.__config.measure_time:\n starttime = time.time()\n self.__decompose_nwp_if_needed_and_fill_nans_in_nwp(t)\n self.__find_nowcast_NWP_combination(t)\n self.__determine_skill_for_current_timestep(t)\n # the nowcast iteration for each ensemble member\n final_blended_forecast_all_members_one_timestep = [\n None for _ in range(self.__config.n_ens_members)\n ]\n if self.__config.nowcasting_method == \"external_nowcast\":\n self.__state.mean_nowcast_timestep = self.__state.mean_nowcast[:, :, t]\n self.__state.std_nowcast_timestep = self.__state.std_nowcast[:, :, t]\n\n def worker(j):\n worker_state = copy(self.__state)\n self.__determine_NWP_skill_for_next_timestep(t, j, worker_state)\n self.__determine_weights_per_component(t, worker_state)\n self.__regress_extrapolation_and_noise_cascades(j, worker_state, t)\n self.__perturb_blend_and_advect_extrapolation_and_noise_to_current_timestep(\n t, j, worker_state\n )\n # 8.5 Blend the cascades\n final_blended_forecast_single_member = []\n for t_sub in self.__state.subtimesteps:\n # TODO: does it make sense to use sub time steps - check if it works?\n if t_sub > 0:\n self.__blend_cascades(t_sub, j, worker_state)\n self.__recompose_cascade_to_rainfall_field(j, worker_state)\n final_blended_forecast_single_member = (\n self.__post_process_output(\n j,\n t_sub,\n final_blended_forecast_single_member,\n worker_state,\n )\n )\n\n final_blended_forecast_all_members_one_timestep[j] = (\n final_blended_forecast_single_member\n )\n\n dask_worker_collection = []\n\n if DASK_IMPORTED and self.__config.n_ens_members > 1:\n for j in range(self.__config.n_ens_members):\n dask_worker_collection.append(dask.delayed(worker)(j))\n dask.compute(\n *dask_worker_collection,\n num_workers=self.__params.num_ensemble_workers,\n )\n else:\n for j in range(self.__config.n_ens_members):\n worker(j)\n\n dask_worker_collection = None\n\n if self.__state.is_nowcast_time_step:\n if self.__config.measure_time:\n _ = self.__measure_time(\"subtimestep\", starttime)\n else:\n print(\"done.\")\n\n if self.__config.callback is not None:\n precip_forecast_final = np.stack(\n final_blended_forecast_all_members_one_timestep\n )\n if precip_forecast_final.shape[1] > 0:\n self.__config.callback(precip_forecast_final.squeeze())\n\n if self.__config.return_output:\n for j in range(self.__config.n_ens_members):\n self.__state.final_blended_forecast[j].extend(\n final_blended_forecast_all_members_one_timestep[j]\n )\n\n final_blended_forecast_all_members_one_timestep = None\n if self.__config.measure_time:\n self.__mainloop_time = time.time() - starttime_mainloop\n\n def __check_inputs(self):\n \"\"\"\n Validates the inputs and determines if the user provided raw forecasts or decomposed forecasts.\n \"\"\"\n # Check dimensions of precip\n if self.__precip.ndim != 3:\n raise ValueError(\n \"precip must be a three-dimensional array of shape (ar_order + 1, m, n)\"\n )\n if self.__precip.shape[0] < self.__config.ar_order + 1:\n raise ValueError(\n f\"precip must have at least {self.__config.ar_order + 1} time steps in the first dimension \"\n f\"to match the autoregressive order (ar_order={self.__config.ar_order})\"\n )\n\n # Check when precip_nowcast is provided that nowcasting_method is set to 'external_nowcast'\n # and the other way around.\n if (\n self.__precip_nowcast is not None\n and self.__config.nowcasting_method != \"external_nowcast\"\n ):\n raise KeyError(\n \"if precip_nowcast is not None, nowcasting_method should be set to 'external_nowcast' \"\n )\n if (\n self.__config.nowcasting_method == \"external_nowcast\"\n and self.__precip_nowcast is None\n ):\n raise KeyError(\n \"if nowcasting_method is set to 'external_nowcast', an external precip_nowcast should be provided as variable.\"\n )\n\n # Check dimensions of velocity\n if self.__velocity.ndim != 3:\n raise ValueError(\n \"velocity must be a three-dimensional array of shape (2, m, n)\"\n )\n if self.__velocity_models.ndim != 5:\n raise ValueError(\n \"velocity_models must be a five-dimensional array of shape (n_models, timestep, 2, m, n)\"\n )\n if self.__velocity.shape[0] != 2 or self.__velocity_models.shape[2] != 2:\n raise ValueError(\n \"velocity and velocity_models must have an x- and y-component, check the shape\"\n )\n\n # Check that spatial dimensions match between precip and velocity\n if self.__precip.shape[1:3] != self.__velocity.shape[1:3]:\n raise ValueError(\n f\"Spatial dimensions of precip and velocity do not match: \"\n f\"{self.__precip.shape[1:3]} vs {self.__velocity.shape[1:3]}\"\n )\n # Check if the number of members in the precipitation models and velocity models match\n if self.__precip_models.shape[0] != self.__velocity_models.shape[0]:\n raise ValueError(\n \"The number of members in the precipitation models and velocity models must match\"\n )\n\n if isinstance(self.__timesteps, list):\n self.__params.time_steps_is_list = True\n if not sorted(self.__timesteps) == self.__timesteps:\n raise ValueError(\n \"timesteps is not in ascending order\", self.__timesteps\n )\n if self.__precip_models.shape[1] != math.ceil(self.__timesteps[-1]) + 1:\n raise ValueError(\n \"precip_models does not contain sufficient lead times for this forecast\"\n )\n self.__params.original_timesteps = [0] + list(self.__timesteps)\n self.__timesteps = nowcast_utils.binned_timesteps(\n self.__params.original_timesteps\n )\n else:\n self.__params.time_steps_is_list = False\n if self.__precip_models.shape[1] != self.__timesteps + 1:\n raise ValueError(\n \"precip_models does not contain sufficient lead times for this forecast\"\n )\n self.__timesteps = list(range(self.__timesteps + 1))\n\n precip_nwp_dim = self.__precip_models.ndim\n if precip_nwp_dim == 2:\n if isinstance(self.__precip_models[0][0], dict):\n # It's a 2D array of dictionaries with decomposed cascades\n self.__params.precip_models_provided_is_cascade = True\n else:\n raise ValueError(\n \"When precip_models has ndim == 2, it must contain dictionaries with decomposed cascades.\"\n )\n elif precip_nwp_dim == 4:\n self.__params.precip_models_provided_is_cascade = False\n else:\n raise ValueError(\n \"precip_models must be either a two-dimensional array containing dictionaries with decomposed model fields\"\n \"or a four-dimensional array containing the original (NWP) model forecasts\"\n )\n if self.__precip_nowcast is not None:\n precip_nowcast_dim = self.__precip_nowcast.ndim\n if precip_nowcast_dim != 4:\n raise ValueError(\n \"precip_nowcast must be a four-dimensional array containing the externally calculated nowcast\"\n )\n if self.__config.extrapolation_kwargs is None:\n self.__state.extrapolation_kwargs = dict()\n else:\n self.__state.extrapolation_kwargs = deepcopy(\n self.__config.extrapolation_kwargs\n )\n\n if self.__config.filter_kwargs is None:\n self.__params.filter_kwargs = dict()\n else:\n self.__params.filter_kwargs = deepcopy(self.__config.filter_kwargs)\n\n if self.__config.noise_kwargs is None:\n self.__params.noise_kwargs = {\"win_fun\": \"tukey\"}\n else:\n self.__params.noise_kwargs = deepcopy(self.__config.noise_kwargs)\n\n if self.__config.velocity_perturbation_kwargs is None:\n self.__params.velocity_perturbation_kwargs = dict()\n else:\n self.__params.velocity_perturbation_kwargs = deepcopy(\n self.__config.velocity_perturbation_kwargs\n )\n\n if self.__config.climatology_kwargs is None:\n # Make sure clim_kwargs at least contains the number of models\n self.__params.climatology_kwargs = dict(\n {\"n_models\": self.__precip_models.shape[0]}\n )\n else:\n self.__params.climatology_kwargs = deepcopy(\n self.__config.climatology_kwargs\n )\n\n if self.__config.mask_kwargs is None:\n self.__params.mask_kwargs = dict()\n else:\n self.__params.mask_kwargs = deepcopy(self.__config.mask_kwargs)\n\n self.__params.precip_threshold = self.__config.precip_threshold\n\n if np.any(~np.isfinite(self.__velocity)):\n raise ValueError(\"velocity contains non-finite values\")\n\n if self.__config.mask_method not in [\"obs\", \"incremental\", None]:\n raise ValueError(\n \"unknown mask method %s: must be 'obs', 'incremental' or None\"\n % self.__config.mask_method\n )\n\n if self.__config.conditional and self.__params.precip_threshold is None:\n raise ValueError(\"conditional=True but precip_thr is not set\")\n\n if (\n self.__config.mask_method is not None\n and self.__params.precip_threshold is None\n ):\n raise ValueError(\"mask_method!=None but precip_thr=None\")\n\n if self.__config.noise_stddev_adj not in [\"auto\", \"fixed\", None]:\n raise ValueError(\n \"unknown noise_std_dev_adj method %s: must be 'auto', 'fixed', or None\"\n % self.__config.noise_stddev_adj\n )\n\n if self.__config.kmperpixel is None:\n if self.__config.velocity_perturbation_method is not None:\n raise ValueError(\n \"velocity_perturbation_method is set but kmperpixel=None\"\n )\n if self.__config.mask_method == \"incremental\":\n raise ValueError(\"mask_method='incremental' but kmperpixel=None\")\n\n if self.__config.timestep is None:\n if self.__config.velocity_perturbation_method is not None:\n raise ValueError(\n \"velocity_perturbation_method is set but timestep=None\"\n )\n if self.__config.mask_method == \"incremental\":\n raise ValueError(\"mask_method='incremental' but timestep=None\")\n\n if self.__config.timestep_start_full_nwp_weight is not None:\n if self.__config.timestep_start_full_nwp_weight < 0:\n raise ValueError(\n \"timestep_start_full_nwp_weight cannot be smaller than zero\"\n )\n\n if self.__config.timestep_start_full_nwp_weight is not None:\n if self.__config.timestep_start_full_nwp_weight >= self.__timesteps[-1]:\n raise ValueError(\n \"timestep_start_full_nwp_weight cannot be the same or larger than the total number of timesteps in this forecast\"\n )\n\n def __print_forecast_info(self):\n \"\"\"\n Print information about the forecast setup, including inputs, methods, and parameters.\n \"\"\"\n print(\"STEPS blending\")\n print(\"==============\")\n print(\"\")\n\n print(\"Inputs\")\n print(\"------\")\n print(f\"forecast issue time: {self.__issuetime.isoformat()}\")\n print(\n f\"input dimensions: {self.__precip.shape[1]}x{self.__precip.shape[2]}\"\n )\n if self.__precip_nowcast is not None:\n print(\n f\"input dimensions pre-computed nowcast: {self.__precip_nowcast.shape[2]}x{self.__precip_nowcast.shape[3]}\"\n )\n if self.__config.kmperpixel is not None:\n print(f\"km/pixel: {self.__config.kmperpixel}\")\n if self.__config.timestep is not None:\n print(f\"time step: {self.__config.timestep} minutes\")\n print(\"\")\n\n print(\"NWP and blending inputs\")\n print(\"-----------------------\")\n print(f\"number of (NWP) models: {self.__precip_models.shape[0]}\")\n print(f\"blend (NWP) model members: {self.__config.blend_nwp_members}\")\n print(\n f\"decompose (NWP) models: {'yes' if self.__precip_models.ndim == 4 else 'no'}\"\n )\n print(\"\")\n\n print(\"Methods\")\n print(\"-------\")\n print(f\"extrapolation: {self.__config.extrapolation_method}\")\n print(f\"bandpass filter: {self.__config.bandpass_filter_method}\")\n print(f\"decomposition: {self.__config.decomposition_method}\")\n print(f\"nowcasting algorithm: {self.__config.nowcasting_method}\")\n print(f\"noise generator: {self.__config.noise_method}\")\n print(\n f\"noise adjustment: {'yes' if self.__config.noise_stddev_adj else 'no'}\"\n )\n print(\n f\"velocity perturbator: {self.__config.velocity_perturbation_method}\"\n )\n print(f\"blending weights method: {self.__config.weights_method}\")\n print(\n f\"conditional statistics: {'yes' if self.__config.conditional else 'no'}\"\n )\n print(f\"precip. mask method: {self.__config.mask_method}\")\n print(f\"probability matching: {self.__config.probmatching_method}\")\n print(f\"FFT method: {self.__config.fft_method}\")\n print(f\"domain: {self.__config.domain}\")\n print(\"\")\n\n print(\"Parameters\")\n print(\"----------\")\n if isinstance(self.__timesteps, int):\n print(f\"number of time steps: {self.__timesteps}\")\n else:\n print(f\"time steps: {self.__timesteps}\")\n print(f\"ensemble size: {self.__config.n_ens_members}\")\n print(f\"parallel threads: {self.__config.num_workers}\")\n print(f\"number of cascade levels: {self.__config.n_cascade_levels}\")\n print(f\"order of the AR(p) model: {self.__config.ar_order}\")\n if self.__config.velocity_perturbation_method == \"bps\":\n self.__params.velocity_perturbations_parallel = (\n self.__params.velocity_perturbation_kwargs.get(\n \"p_par\", noise.motion.get_default_params_bps_par()\n )\n )\n self.__params.velocity_perturbations_perpendicular = (\n self.__params.velocity_perturbation_kwargs.get(\n \"p_perp\", noise.motion.get_default_params_bps_perp()\n )\n )\n print(\n f\"vel. pert. parallel: {self.__params.velocity_perturbations_parallel[0]},{self.__params.velocity_perturbations_parallel[1]},{self.__params.velocity_perturbations_parallel[2]}\"\n )\n print(\n f\"vel. pert. perpendicular: {self.__params.velocity_perturbations_perpendicular[0]},{self.__params.velocity_perturbations_perpendicular[1]},{self.__params.velocity_perturbations_perpendicular[2]}\"\n )\n else:\n (\n self.__params.velocity_perturbations_parallel,\n self.__params.velocity_perturbations_perpendicular,\n ) = (None, None)\n\n if self.__config.conditional or self.__config.mask_method is not None:\n print(f\"precip. intensity threshold: {self.__params.precip_threshold}\")\n print(f\"no-rain fraction threshold for radar: {self.__config.norain_threshold}\")\n print(\"\")\n\n def __initialize_nowcast_components(self):\n \"\"\"\n Initialize the FFT, bandpass filters, decomposition methods, and extrapolation method.\n \"\"\"\n # Initialize number of ensemble workers\n self.__params.num_ensemble_workers = min(\n self.__config.n_ens_members, self.__config.num_workers\n )\n\n M, N = self.__precip.shape[1:] # Extract the spatial dimensions (height, width)\n\n # Initialize FFT method\n self.__params.fft = utils.get_method(\n self.__config.fft_method, shape=(M, N), n_threads=self.__config.num_workers\n )\n\n # Initialize the band-pass filter for the cascade decomposition\n filter_method = cascade.get_method(self.__config.bandpass_filter_method)\n self.__params.bandpass_filter = filter_method(\n (M, N),\n self.__config.n_cascade_levels,\n **(self.__params.filter_kwargs or {}),\n )\n\n # Get the decomposition method (e.g., FFT)\n (\n self.__params.decomposition_method,\n self.__params.recomposition_method,\n ) = cascade.get_method(self.__config.decomposition_method)\n\n # Get the extrapolation method (e.g., semilagrangian)\n self.__params.extrapolation_method = extrapolation.get_method(\n self.__config.extrapolation_method\n )\n\n # Generate the mesh grid for spatial coordinates\n x_values, y_values = np.meshgrid(np.arange(N), np.arange(M))\n self.__params.xy_coordinates = np.stack([x_values, y_values])\n\n self.__precip = self.__precip[-(self.__config.ar_order + 1) :, :, :].copy()\n # Determine the domain mask from non-finite values in the precipitation data\n self.__params.domain_mask = np.logical_or.reduce(\n [~np.isfinite(self.__precip[i, :]) for i in range(self.__precip.shape[0])]\n )\n\n print(\"Blended nowcast components initialized successfully.\")\n\n def __prepare_radar_and_NWP_fields(self):\n \"\"\"\n Prepare radar and NWP precipitation fields for nowcasting.\n This includes generating a threshold mask, transforming fields into\n Lagrangian coordinates, cascade decomposing/recomposing, and checking\n for zero-precip areas. The results are stored in class attributes.\n \"\"\"\n # determine the precipitation threshold mask\n if self.__config.conditional:\n # TODO: is this logical_and correct here? Now only those places where precip is in all images is saved?\n self.__params.mask_threshold = np.logical_and.reduce(\n [\n self.__precip[i, :, :] >= self.__params.precip_threshold\n for i in range(self.__precip.shape[0])\n ]\n )\n else:\n self.__params.mask_threshold = None\n\n # we need to know the zerovalue of precip to replace the mask when decomposing after\n # extrapolation\n self.__params.nowcast_zerovalue = np.nanmin(self.__precip_nowcast)\n self.__params.precip_zerovalue = np.nanmin(self.__precip)\n # 1. Start with the radar rainfall fields. We want the fields in a Lagrangian\n # space. Advect the previous precipitation fields to the same position with\n # the most recent one (i.e. transform them into the Lagrangian coordinates).\n self.__state.extrapolation_kwargs[\"xy_coords\"] = self.__params.xy_coordinates\n res = []\n\n def transform_to_lagrangian(precip, i):\n return self.__params.extrapolation_method(\n precip[i, :, :],\n self.__velocity,\n self.__config.ar_order - i,\n \"min\",\n allow_nonfinite_values=True,\n **self.__state.extrapolation_kwargs.copy(),\n )[-1]\n\n if not DASK_IMPORTED:\n # Process each earlier precipitation field directly\n for i in range(self.__config.ar_order):\n self.__precip[i, :, :] = transform_to_lagrangian(self.__precip, i)\n else:\n # Use Dask delayed for parallelization if DASK_IMPORTED is True\n for i in range(self.__config.ar_order):\n res.append(dask.delayed(transform_to_lagrangian)(self.__precip, i))\n num_workers_ = (\n len(res)\n if self.__config.num_workers > len(res)\n else self.__config.num_workers\n )\n self.__precip = np.stack(\n list(dask.compute(*res, num_workers=num_workers_))\n + [self.__precip[-1, :, :]]\n )\n\n # Replace non-finite values with the minimum value for each field\n self.__precip = self.__precip.copy()\n for i in range(self.__precip.shape[0]):\n self.__precip[i, ~np.isfinite(self.__precip[i, :])] = np.nanmin(\n self.__precip[i, :]\n )\n if self.__precip_nowcast is not None:\n self.__precip_nowcast = self.__precip_nowcast.copy()\n for ens_mem in range(self.__precip_nowcast.shape[0]):\n for t in range(self.__precip_nowcast.shape[1]):\n self.__precip_nowcast[\n ens_mem,\n t,\n ~np.isfinite(self.__precip_nowcast[ens_mem, t, :, :]),\n ] = np.nanmin(self.__precip_nowcast[ens_mem, t, :, :])\n\n # Perform the cascade decomposition for the input precip fields and,\n # if necessary, for the (NWP) model fields\n # Compute the cascade decompositions of the input precipitation fields\n precip_forecast_decomp = []\n for i in range(self.__config.ar_order + 1):\n precip_forecast = self.__params.decomposition_method(\n self.__precip[i, :, :],\n self.__params.bandpass_filter,\n mask=self.__params.mask_threshold,\n fft_method=self.__params.fft,\n output_domain=self.__config.domain,\n normalize=True,\n compute_stats=True,\n compact_output=True,\n )\n precip_forecast_decomp.append(precip_forecast)\n\n # Rearrange the cascaded into a four-dimensional array of shape\n # (n_cascade_levels,ar_order+1,m,n) for the autoregressive model\n self.__state.precip_cascades = nowcast_utils.stack_cascades(\n precip_forecast_decomp, self.__config.n_cascade_levels\n )\n\n precip_forecast_decomp = precip_forecast_decomp[-1]\n self.__state.mean_extrapolation = np.array(precip_forecast_decomp[\"means\"])\n self.__state.std_extrapolation = np.array(precip_forecast_decomp[\"stds\"])\n\n # Decompose precomputed nowcasts and rearange them again into the required components\n if self.__precip_nowcast is not None:\n if self.__precip_nowcast.shape[0] == 1:\n results = [self.__decompose_member(self.__precip_nowcast[0])]\n else:\n with ThreadPool(self.__config.num_workers) as pool:\n results = pool.map(\n partial(self.__decompose_member),\n list(self.__precip_nowcast),\n )\n\n self.__state.precip_nowcast_cascades = np.array(\n [result[\"precip_nowcast_decomp\"] for result in results]\n ).swapaxes(1, 2)\n self.__state.mean_nowcast = np.array(\n [result[\"precip_nowcast_means\"] for result in results]\n ).swapaxes(1, 2)\n self.__state.std_nowcast = np.array(\n [result[\"precip_nowcast_stds\"] for result in results]\n ).swapaxes(1, 2)\n\n # If necessary, recompose (NWP) model forecasts\n self.__state.precip_models_cascades = None\n\n if self.__params.precip_models_provided_is_cascade:\n self.__state.precip_models_cascades = self.__precip_models\n # Inline logic of _compute_cascade_recomposition_nwp\n temp_precip_models = []\n for i in range(self.__precip_models.shape[0]):\n precip_model = []\n for time_step in range(self.__precip_models.shape[1]):\n # Use the recomposition method to rebuild the rainfall fields\n recomposed = self.__params.recomposition_method(\n self.__precip_models[i, time_step]\n )\n precip_model.append(recomposed)\n temp_precip_models.append(precip_model)\n\n self.__precip_models = np.stack(temp_precip_models)\n\n # Check for zero input fields in the radar, nowcast and NWP data.\n self.__params.zero_precip_radar = check_norain(\n self.__precip,\n self.__params.precip_threshold,\n self.__config.norain_threshold,\n self.__params.noise_kwargs[\"win_fun\"],\n )\n\n # The norain fraction threshold used for nwp is the default value of 0.0,\n # since nwp does not suffer from clutter.\n self.__params.zero_precip_model_fields = check_norain(\n self.__precip_models,\n self.__params.precip_threshold,\n self.__config.norain_threshold,\n self.__params.noise_kwargs[\"win_fun\"],\n )\n\n def __decompose_member(self, member_field):\n \"\"\"Loop over timesteps for a single ensemble member.\"\"\"\n results_decomp = []\n means = []\n stds = []\n for t in range(member_field.shape[0]): # loop over timesteps\n res = self.__params.decomposition_method(\n field=member_field[t, :, :],\n bp_filter=self.__params.bandpass_filter,\n n_levels=self.__config.n_cascade_levels,\n mask=self.__params.mask_threshold,\n method=\"fft\",\n fft_method=self.__params.fft,\n output_domain=self.__config.domain,\n compute_stats=True,\n normalize=True,\n compact_output=True,\n )\n results_decomp.append(res[\"cascade_levels\"])\n means.append(res[\"means\"])\n stds.append(res[\"stds\"])\n results = {\n \"precip_nowcast_decomp\": results_decomp,\n \"precip_nowcast_means\": means,\n \"precip_nowcast_stds\": stds,\n }\n\n return results\n\n def __zero_precipitation_forecast(self):\n \"\"\"\n Generate a zero-precipitation forecast (filled with the minimum precip value)\n when no precipitation above the threshold is detected. The forecast is\n optionally returned or passed to a callback.\n \"\"\"\n print(\n \"No precipitation above the threshold found in both the radar and NWP fields\"\n )\n print(\"The resulting forecast will contain only zeros\")\n # Create the output list\n precip_forecast = [[] for j in range(self.__config.n_ens_members)]\n\n # Save per time step to ensure the array does not become too large if\n # no return_output is requested and callback is not None.\n for t, subtimestep_idx in enumerate(self.__timesteps):\n # If the timestep is not the first one, we need to provide the zero forecast\n if t > 0:\n # Create an empty np array with shape [n_ens_members, rows, cols]\n # and fill it with the minimum value from precip (corresponding to\n # zero precipitation)\n N, M = self.__precip.shape[1:]\n precip_forecast_workers = np.full(\n (self.__config.n_ens_members, N, M), self.__params.precip_zerovalue\n )\n if subtimestep_idx:\n if self.__config.callback is not None:\n if precip_forecast_workers.shape[1] > 0:\n self.__config.callback(precip_forecast_workers.squeeze())\n if self.__config.return_output:\n for j in range(self.__config.n_ens_members):\n precip_forecast[j].append(precip_forecast_workers[j])\n precip_forecast_workers = None\n\n if self.__config.measure_time:\n zero_precip_time = time.time() - self.__start_time_init\n\n if self.__config.return_output:\n precip_forecast_all_members_all_times = np.stack(\n [\n np.stack(precip_forecast[j])\n for j in range(self.__config.n_ens_members)\n ]\n )\n\n if self.__config.measure_time:\n return (\n precip_forecast_all_members_all_times,\n zero_precip_time,\n zero_precip_time,\n )\n else:\n return precip_forecast_all_members_all_times\n else:\n return None\n\n def __prepare_nowcast_for_zero_radar(self):\n \"\"\"\n Handle the case when radar fields indicate zero precipitation. This method\n updates the cascade with NWP data, uses the NWP velocity field, and\n initializes the noise model based on the time step with the most rain.\n \"\"\"\n # If zero_precip_radar is True, only use the velocity field of the NWP\n # forecast. I.e., velocity (radar) equals velocity_model at the first time\n # step.\n # Use the velocity from velocity_models at time step 0\n self.__velocity = self.__velocity_models[:, 0, :, :, :].astype(\n np.float64, copy=False\n )\n # Take the average over the first axis, which corresponds to n_models\n # (hence, the model average)\n self.__velocity = np.mean(self.__velocity, axis=0)\n\n # Initialize the noise method.\n # If zero_precip_radar is True, initialize noise based on the NWP field time\n # step where the fraction of rainy cells is highest (because other lead times\n # might be zero as well). Else, initialize the noise with the radar\n # rainfall data\n # Initialize noise based on the NWP field time step where the fraction of rainy cells is highest\n if self.__params.precip_threshold is None:\n self.__params.precip_threshold = np.nanmin(self.__precip_models)\n\n max_rain_pixels = -1\n max_rain_pixels_j = -1\n max_rain_pixels_t = -1\n for j in range(self.__precip_models.shape[0]):\n for t in self.__timesteps:\n rain_pixels = self.__precip_models[j][t][\n self.__precip_models[j][t] > self.__params.precip_threshold\n ].size\n if rain_pixels > max_rain_pixels:\n max_rain_pixels = rain_pixels\n max_rain_pixels_j = j\n max_rain_pixels_t = t\n self.__state.precip_noise_input = self.__precip_models[max_rain_pixels_j][\n max_rain_pixels_t\n ]\n self.__state.precip_noise_input = self.__state.precip_noise_input.astype(\n np.float64, copy=False\n )\n\n # If zero_precip_radar, make sure that precip_cascade does not contain\n # only nans or infs. If so, fill it with the zero value.\n if self.__state.precip_models_cascades is not None:\n self.__state.precip_cascades[~np.isfinite(self.__state.precip_cascades)] = (\n np.nanmin(\n self.__state.precip_models_cascades[\n max_rain_pixels_j, max_rain_pixels_t\n ][\"cascade_levels\"]\n )\n )\n else:\n precip_models_cascade_timestep = self.__params.decomposition_method(\n self.__precip_models[max_rain_pixels_j, max_rain_pixels_t, :, :],\n bp_filter=self.__params.bandpass_filter,\n fft_method=self.__params.fft,\n output_domain=self.__config.domain,\n normalize=True,\n compute_stats=True,\n compact_output=True,\n )[\"cascade_levels\"]\n self.__state.precip_cascades[~np.isfinite(self.__state.precip_cascades)] = (\n np.nanmin(precip_models_cascade_timestep)\n )\n\n # Make sure precip_noise_input is three-dimensional\n if len(self.__state.precip_noise_input.shape) != 3:\n self.__state.precip_noise_input = self.__state.precip_noise_input[\n np.newaxis, :, :\n ]\n\n def __initialize_noise(self):\n \"\"\"\n Initialize noise-based perturbations if configured, computing any required\n adjustment coefficients and setting up the perturbation generator.\n \"\"\"\n if self.__config.noise_method is not None:\n # get methods for perturbations\n init_noise, self.__params.noise_generator = noise.get_method(\n self.__config.noise_method\n )\n\n # initialize the perturbation generator for the precipitation field\n self.__params.perturbation_generator = init_noise(\n self.__state.precip_noise_input,\n fft_method=self.__params.fft,\n **self.__params.noise_kwargs,\n )\n\n if self.__config.noise_stddev_adj == \"auto\":\n print(\"Computing noise adjustment coefficients... \", end=\"\", flush=True)\n if self.__config.measure_time:\n starttime = time.time()\n\n precip_forecast_min = np.min(self.__state.precip_noise_input)\n self.__params.noise_std_coeffs = noise.utils.compute_noise_stddev_adjs(\n self.__state.precip_noise_input[-1, :, :],\n self.__params.precip_threshold,\n precip_forecast_min,\n self.__params.bandpass_filter,\n self.__params.decomposition_method,\n self.__params.perturbation_generator,\n self.__params.noise_generator,\n 20,\n conditional=True,\n num_workers=self.__config.num_workers,\n seed=self.__config.seed,\n )\n\n if self.__config.measure_time:\n _ = self.__measure_time(\"Initialize noise\", starttime)\n else:\n print(\"done.\")\n elif self.__config.noise_stddev_adj == \"fixed\":\n f = lambda k: 1.0 / (0.75 + 0.09 * k)\n self.__params.noise_std_coeffs = [\n f(k) for k in range(1, self.__config.n_cascade_levels + 1)\n ]\n else:\n self.__params.noise_std_coeffs = np.ones(self.__config.n_cascade_levels)\n\n if self.__config.noise_stddev_adj is not None:\n print(f\"noise std. dev. coeffs: {self.__params.noise_std_coeffs}\")\n\n else:\n self.__params.perturbation_generator = None\n self.__params.noise_generator = None\n self.__params.noise_std_coeffs = None\n\n def __estimate_ar_parameters_radar(self):\n \"\"\"\n Estimate autoregressive (AR) parameters for the radar rainfall field. If\n precipitation exists, compute temporal auto-correlations; otherwise, use\n predefined climatological values. Adjust coefficients if necessary and\n estimate AR model parameters.\n \"\"\"\n # If there are values in the radar fields, compute the auto-correlations\n GAMMA = np.empty((self.__config.n_cascade_levels, self.__config.ar_order))\n if not self.__params.zero_precip_radar:\n # compute lag-l temporal auto-correlation coefficients for each cascade level\n for i in range(self.__config.n_cascade_levels):\n GAMMA[i, :] = correlation.temporal_autocorrelation(\n self.__state.precip_cascades[i], mask=self.__params.mask_threshold\n )\n\n # Else, use standard values for the auto-correlations\n else:\n # Get the climatological lag-1 and lag-2 auto-correlation values from Table 2\n # in `BPS2004`.\n # Hard coded, change to own (climatological) values when present.\n # TODO: add user warning here so users can be aware of this without reading the code?\n GAMMA = np.array(\n [\n [0.99805, 0.9925, 0.9776, 0.9297, 0.796, 0.482, 0.079, 0.0006],\n [0.9933, 0.9752, 0.923, 0.750, 0.367, 0.069, 0.0018, 0.0014],\n ]\n )\n\n # Check whether the number of cascade_levels is correct\n if GAMMA.shape[1] > self.__config.n_cascade_levels:\n GAMMA = GAMMA[:, 0 : self.__config.n_cascade_levels]\n elif GAMMA.shape[1] < self.__config.n_cascade_levels:\n # Get the number of cascade levels that is missing\n n_extra_lev = self.__config.n_cascade_levels - GAMMA.shape[1]\n # Append the array with correlation values of 10e-4\n GAMMA = np.append(\n GAMMA,\n [np.repeat(0.0006, n_extra_lev), np.repeat(0.0014, n_extra_lev)],\n axis=1,\n )\n\n # Finally base GAMMA.shape[0] on the AR-level\n if self.__config.ar_order == 1:\n GAMMA = GAMMA[0, :]\n if self.__config.ar_order > 2:\n for _ in range(self.__config.ar_order - 2):\n GAMMA = np.vstack((GAMMA, GAMMA[1, :]))\n\n # Finally, transpose GAMMA to ensure that the shape is the same as np.empty((n_cascade_levels, ar_order))\n GAMMA = GAMMA.transpose()\n assert GAMMA.shape == (\n self.__config.n_cascade_levels,\n self.__config.ar_order,\n )\n\n # Print the GAMMA value\n nowcast_utils.print_corrcoefs(GAMMA)\n\n if self.__config.ar_order == 2:\n # adjust the lag-2 correlation coefficient to ensure that the AR(p)\n # process is stationary\n for i in range(self.__config.n_cascade_levels):\n GAMMA[i, 1] = autoregression.adjust_lag2_corrcoef2(\n GAMMA[i, 0], GAMMA[i, 1]\n )\n\n # estimate the parameters of the AR(p) model from the auto-correlation\n # coefficients\n self.__params.PHI = np.empty(\n (self.__config.n_cascade_levels, self.__config.ar_order + 1)\n )\n for i in range(self.__config.n_cascade_levels):\n self.__params.PHI[i, :] = autoregression.estimate_ar_params_yw(GAMMA[i, :])\n\n nowcast_utils.print_ar_params(self.__params.PHI)\n\n def __multiply_precip_cascade_to_match_ensemble_members(self):\n \"\"\"\n Duplicate the last p-1 precipitation cascades across all ensemble members\n for the AR(p) model, ensuring each member has the required input structure.\n \"\"\"\n self.__state.precip_cascades = np.stack(\n [\n [\n self.__state.precip_cascades[i][-self.__config.ar_order :].copy()\n for i in range(self.__config.n_cascade_levels)\n ]\n ]\n * self.__config.n_ens_members\n )\n\n def __initialize_random_generators(self):\n \"\"\"\n Initialize random generators for precipitation noise, probability matching,\n and velocity perturbations. Each ensemble member gets a separate generator,\n ensuring reproducibility and controlled randomness in forecasts.\n \"\"\"\n seed = self.__config.seed\n if self.__config.noise_method is not None:\n self.__state.randgen_precip = []\n for j in range(self.__config.n_ens_members):\n rs = np.random.RandomState(seed)\n self.__state.randgen_precip.append(rs)\n seed = rs.randint(0, high=1e9)\n\n if self.__config.probmatching_method is not None:\n self.__state.randgen_probmatching = []\n for j in range(self.__config.n_ens_members):\n rs = np.random.RandomState(seed)\n self.__state.randgen_probmatching.append(rs)\n seed = rs.randint(0, high=1e9)\n\n if self.__config.velocity_perturbation_method is not None:\n self.__state.randgen_motion = []\n for j in range(self.__config.n_ens_members):\n rs = np.random.RandomState(seed)\n self.__state.randgen_motion.append(rs)\n seed = rs.randint(0, high=1e9)\n\n (\n init_velocity_noise,\n self.__params.generate_velocity_noise,\n ) = noise.get_method(self.__config.velocity_perturbation_method)\n\n # initialize the perturbation generators for the motion field\n self.__params.velocity_perturbations = []\n for j in range(self.__config.n_ens_members):\n kwargs = {\n \"randstate\": self.__state.randgen_motion[j],\n \"p_par\": self.__params.velocity_perturbations_parallel,\n \"p_perp\": self.__params.velocity_perturbations_perpendicular,\n }\n vp_ = init_velocity_noise(\n self.__velocity,\n 1.0 / self.__config.kmperpixel,\n self.__config.timestep,\n **kwargs,\n )\n self.__params.velocity_perturbations.append(vp_)\n else:\n (\n self.__params.velocity_perturbations,\n self.__params.generate_velocity_noise,\n ) = (None, None)\n\n def __prepare_forecast_loop(self):\n \"\"\"\n Initialize variables and structures needed for the forecast loop, including\n displacement tracking, mask parameters, noise handling, FFT objects, and\n extrapolation scaling for nowcasting.\n \"\"\"\n # Empty arrays for the previous displacements and the forecast cascade\n self.__state.previous_displacement = np.stack(\n [None for j in range(self.__config.n_ens_members)]\n )\n self.__state.previous_displacement_noise_cascade = np.stack(\n [None for j in range(self.__config.n_ens_members)]\n )\n self.__state.previous_displacement_prob_matching = np.stack(\n [None for j in range(self.__config.n_ens_members)]\n )\n self.__state.final_blended_forecast = [\n [] for j in range(self.__config.n_ens_members)\n ]\n\n if self.__config.mask_method == \"incremental\":\n # get mask parameters\n self.__params.mask_rim = self.__params.mask_kwargs.get(\"mask_rim\", 10)\n self.__params.max_mask_rim = self.__params.mask_kwargs.get(\n \"max_mask_rim\", 10\n )\n mask_f = self.__params.mask_kwargs.get(\"mask_f\", 1.0)\n # initialize the structuring element\n struct = generate_binary_structure(2, 1)\n # iterate it to expand it nxn\n n = mask_f * self.__config.timestep / self.__config.kmperpixel\n self.__params.struct = iterate_structure(struct, int((n - 1) / 2.0))\n else:\n self.__params.mask_rim, self.__params.struct = None, None\n\n if self.__config.noise_method is None:\n self.__state.final_blended_forecast_non_perturbed = [\n self.__state.precip_cascades[0][i].copy()\n for i in range(self.__config.n_cascade_levels)\n ]\n else:\n self.__state.final_blended_forecast_non_perturbed = None\n\n self.__params.fft_objs = []\n for i in range(self.__config.n_ens_members):\n self.__params.fft_objs.append(\n utils.get_method(\n self.__config.fft_method,\n shape=self.__state.precip_cascades.shape[-2:],\n )\n )\n\n # initizalize the current and previous extrapolation forecast scale for the nowcasting component\n # phi1 / (1 - phi2), see BPS2004\n self.__state.rho_extrap_cascade_prev = np.repeat(\n 1.0, self.__params.PHI.shape[0]\n )\n self.__state.rho_extrap_cascade = self.__params.PHI[:, 0] / (\n 1.0 - self.__params.PHI[:, 1]\n )\n\n def __initialize_noise_cascades(self):\n \"\"\"\n Initialize the noise cascade with identical noise for all AR(n) steps\n We also need to return the mean and standard deviations of the noise\n for the recombination of the noise before advecting it.\n \"\"\"\n self.__state.precip_noise_cascades = np.zeros(\n self.__state.precip_cascades.shape\n )\n self.__state.precip_mean_noise = np.zeros(\n (self.__config.n_ens_members, self.__config.n_cascade_levels)\n )\n self.__state.precip_std_noise = np.zeros(\n (self.__config.n_ens_members, self.__config.n_cascade_levels)\n )\n if self.__config.noise_method:\n for j in range(self.__config.n_ens_members):\n epsilon = self.__params.noise_generator(\n self.__params.perturbation_generator,\n randstate=self.__state.randgen_precip[j],\n fft_method=self.__params.fft_objs[j],\n domain=self.__config.domain,\n )\n epsilon_decomposed = self.__params.decomposition_method(\n epsilon,\n self.__params.bandpass_filter,\n fft_method=self.__params.fft_objs[j],\n input_domain=self.__config.domain,\n output_domain=self.__config.domain,\n compute_stats=True,\n normalize=True,\n compact_output=True,\n )\n self.__state.precip_mean_noise[j] = epsilon_decomposed[\"means\"]\n self.__state.precip_std_noise[j] = epsilon_decomposed[\"stds\"]\n for i in range(self.__config.n_cascade_levels):\n epsilon_temp = epsilon_decomposed[\"cascade_levels\"][i]\n epsilon_temp *= self.__params.noise_std_coeffs[i]\n for n in range(self.__config.ar_order):\n self.__state.precip_noise_cascades[j][i][n] = epsilon_temp\n epsilon_decomposed = None\n epsilon_temp = None\n\n def __determine_subtimesteps_and_nowcast_time_step(self, t, subtimestep_idx):\n \"\"\"\n Determine the current sub-timesteps and check if the current time step\n requires nowcasting. Updates the `is_nowcast_time_step` flag accordingly.\n \"\"\"\n if self.__params.time_steps_is_list:\n self.__state.subtimesteps = [\n self.__params.original_timesteps[t_] for t_ in subtimestep_idx\n ]\n else:\n self.__state.subtimesteps = [t]\n\n if (self.__params.time_steps_is_list and self.__state.subtimesteps) or (\n not self.__params.time_steps_is_list and t > 0\n ):\n self.__state.is_nowcast_time_step = True\n else:\n self.__state.is_nowcast_time_step = False\n\n if self.__state.is_nowcast_time_step:\n print(\n f\"Computing nowcast for time step {t}... \",\n end=\"\",\n flush=True,\n )\n\n def __decompose_nwp_if_needed_and_fill_nans_in_nwp(self, t):\n \"\"\"\n Decompose NWP model precipitation fields if needed, store cascade components,\n and replace any NaN or infinite values with appropriate minimum values.\n \"\"\"\n if self.__state.precip_models_cascades is not None:\n decomp_precip_models = list(self.__state.precip_models_cascades[:, t])\n\n else:\n if self.__precip_models.shape[0] == 1:\n decomp_precip_models = [\n self.__params.decomposition_method(\n self.__precip_models[0, t, :, :],\n bp_filter=self.__params.bandpass_filter,\n fft_method=self.__params.fft,\n output_domain=self.__config.domain,\n normalize=True,\n compute_stats=True,\n compact_output=True,\n )\n ]\n else:\n with ThreadPool(self.__config.num_workers) as pool:\n decomp_precip_models = pool.map(\n partial(\n self.__params.decomposition_method,\n bp_filter=self.__params.bandpass_filter,\n fft_method=self.__params.fft,\n output_domain=self.__config.domain,\n normalize=True,\n compute_stats=True,\n compact_output=True,\n ),\n list(self.__precip_models[:, t, :, :]),\n )\n\n self.__state.precip_models_cascades_timestep = np.array(\n [decomp[\"cascade_levels\"] for decomp in decomp_precip_models]\n )\n self.__state.mean_models_timestep = np.array(\n [decomp[\"means\"] for decomp in decomp_precip_models]\n )\n self.__state.std_models_timestep = np.array(\n [decomp[\"stds\"] for decomp in decomp_precip_models]\n )\n\n # Check if the NWP fields contain nans or infinite numbers. If so,\n # fill these with the minimum value present in precip (corresponding to\n # zero rainfall in the radar observations)\n\n # Ensure that the NWP cascade and fields do no contain any nans or infinite number\n # Fill nans and infinite numbers with the minimum value present in precip\n self.__state.precip_models_timestep = self.__precip_models[:, t, :, :].astype(\n np.float64, copy=False\n ) # (corresponding to zero rainfall in the radar observations)\n min_cascade = np.nanmin(self.__state.precip_cascades)\n min_precip = np.nanmin(self.__precip)\n self.__state.precip_models_cascades_timestep[\n ~np.isfinite(self.__state.precip_models_cascades_timestep)\n ] = min_cascade\n self.__state.precip_models_timestep[\n ~np.isfinite(self.__state.precip_models_timestep)\n ] = min_precip\n # Also set any nans or infs in the mean and sigma of the cascade to\n # respectively 0.0 and 1.0\n self.__state.mean_models_timestep[\n ~np.isfinite(self.__state.mean_models_timestep)\n ] = 0.0\n self.__state.std_models_timestep[\n ~np.isfinite(self.__state.std_models_timestep)\n ] = 0.0\n\n def __find_nowcast_NWP_combination(self, t):\n \"\"\"\n Determine which (NWP) models will be combined with which nowcast ensemble members.\n With the way it is implemented at this moment: n_ens_members of the output equals\n the maximum number of (ensemble) members in the input (either the nowcasts or NWP).\n \"\"\"\n\n self.__state.velocity_models_timestep = self.__velocity_models[\n :, t, :, :, :\n ].astype(np.float64, copy=False)\n # Make sure the number of model members is not larger than or equal to n_ens_members\n n_model_members = self.__state.precip_models_cascades_timestep.shape[0]\n if n_model_members > self.__config.n_ens_members:\n raise ValueError(\n \"The number of NWP model members is larger than the given number of ensemble members. n_model_members <= n_ens_members.\"\n )\n\n # Check if NWP models/members should be used individually, or if all of\n # them are blended together per nowcast ensemble member.\n if self.__config.blend_nwp_members:\n self.__state.mapping_list_NWP_member_to_ensemble_member = None\n\n elif self.__config.nowcasting_method == \"external_nowcast\":\n self.__state.precip_nowcast_timestep = self.__precip_nowcast[\n :, t, :, :\n ].astype(np.float64, copy=False)\n\n n_ens_members_provided = self.__precip_nowcast.shape[0]\n if n_ens_members_provided > self.__config.n_ens_members:\n raise ValueError(\n \"The number of nowcast ensemble members provided is larger than the given number of ensemble members requested. n_ens_members_provided <= n_ens_members.\"\n )\n\n n_ens_members_max = self.__config.n_ens_members\n n_ens_members_min = min(n_ens_members_provided, n_model_members)\n\n # Also make a list of the model index numbers. These indices are needed\n # for indexing the right climatological skill file when pysteps calculates\n # the blended forecast in parallel.\n if n_model_members > 1:\n self.__state.mapping_list_NWP_member_to_ensemble_member = np.arange(\n n_model_members\n )\n else:\n self.__state.mapping_list_NWP_member_to_ensemble_member = [0]\n\n def repeat_precip_to_match_ensemble_size(repeats, model_type):\n if model_type == \"nwp\":\n print(\"Repeating the NWP model for all ensemble members\")\n self.__state.precip_models_cascades_timestep = np.repeat(\n self.__state.precip_models_cascades_timestep,\n repeats,\n axis=0,\n )\n self.__state.mean_models_timestep = np.repeat(\n self.__state.mean_models_timestep, repeats, axis=0\n )\n self.__state.std_models_timestep = np.repeat(\n self.__state.std_models_timestep, repeats, axis=0\n )\n self.__state.velocity_models_timestep = np.repeat(\n self.__state.velocity_models_timestep, repeats, axis=0\n )\n # For the prob. matching\n self.__state.precip_models_timestep = np.repeat(\n self.__state.precip_models_timestep, repeats, axis=0\n )\n # Finally, for the model indices\n self.__state.mapping_list_NWP_member_to_ensemble_member = np.repeat(\n self.__state.mapping_list_NWP_member_to_ensemble_member,\n repeats,\n axis=0,\n )\n if model_type == \"nowcast\":\n print(\"Repeating the nowcast for all ensemble members\")\n self.__state.precip_nowcast_cascades = np.repeat(\n self.__state.precip_nowcast_cascades,\n repeats,\n axis=0,\n )\n self.__precip_nowcast = np.repeat(\n self.__precip_nowcast,\n repeats,\n axis=0,\n )\n self.__state.mean_nowcast = np.repeat(\n self.__state.mean_nowcast, repeats, axis=0\n )\n self.__state.std_nowcast = np.repeat(\n self.__state.std_nowcast, repeats, axis=0\n )\n # For the prob. matching\n self.__state.precip_nowcast_timestep = np.repeat(\n self.__state.precip_nowcast_timestep, repeats, axis=0\n )\n\n # Now, repeat the nowcast ensemble members or the nwp models/members until\n # it has the same amount of members as n_ens_members_max. For instance, if\n # you have 10 ensemble nowcasts members and 3 NWP members, the output will\n # be an ensemble of 10 members. Hence, the three NWP members are blended\n # with the first three members of the nowcast (member one with member one,\n # two with two, etc.), subsequently, the same NWP members are blended with\n # the next three members (NWP member one with member 4, NWP member 2 with\n # member 5, etc.), until 10 is reached.\n if n_ens_members_min != n_ens_members_max:\n if n_model_members == 1:\n repeat_precip_to_match_ensemble_size(n_ens_members_max, \"nwp\")\n if n_ens_members_provided == 1:\n repeat_precip_to_match_ensemble_size(n_ens_members_max, \"nowcast\")\n\n if n_model_members == n_ens_members_min and n_model_members != 1:\n print(\"Repeating the NWP model for all ensemble members\")\n repeats = [\n (n_ens_members_max + i) // n_ens_members_min\n for i in range(n_ens_members_min)\n ]\n repeat_precip_to_match_ensemble_size(repeats, \"nwp\")\n\n if (\n n_ens_members_provided == n_ens_members_min\n and n_ens_members_provided != 1\n ):\n repeat_precip_to_match_ensemble_size(repeats, \"nowcast\")\n\n else:\n # Start with determining the maximum and mimimum number of members/models\n # in both input products\n n_ens_members_max = max(self.__config.n_ens_members, n_model_members)\n n_ens_members_min = min(self.__config.n_ens_members, n_model_members)\n # Also make a list of the model index numbers. These indices are needed\n # for indexing the right climatological skill file when pysteps calculates\n # the blended forecast in parallel.\n if n_model_members > 1:\n self.__state.mapping_list_NWP_member_to_ensemble_member = np.arange(\n n_model_members\n )\n else:\n self.__state.mapping_list_NWP_member_to_ensemble_member = [0]\n\n # Now, repeat the nowcast ensemble members or the nwp models/members until\n # it has the same amount of members as n_ens_members_max. For instance, if\n # you have 10 ensemble nowcasts members and 3 NWP members, the output will\n # be an ensemble of 10 members. Hence, the three NWP members are blended\n # with the first three members of the nowcast (member one with member one,\n # two with two, etc.), subsequently, the same NWP members are blended with\n # the next three members (NWP member one with member 4, NWP member 2 with\n # member 5, etc.), until 10 is reached.\n if n_ens_members_min != n_ens_members_max:\n if n_model_members == 1:\n self.__state.precip_models_cascades_timestep = np.repeat(\n self.__state.precip_models_cascades_timestep,\n n_ens_members_max,\n axis=0,\n )\n self.__state.mean_models_timestep = np.repeat(\n self.__state.mean_models_timestep, n_ens_members_max, axis=0\n )\n self.__state.std_models_timestep = np.repeat(\n self.__state.std_models_timestep, n_ens_members_max, axis=0\n )\n self.__state.velocity_models_timestep = np.repeat(\n self.__state.velocity_models_timestep, n_ens_members_max, axis=0\n )\n # For the prob. matching\n self.__state.precip_models_timestep = np.repeat(\n self.__state.precip_models_timestep, n_ens_members_max, axis=0\n )\n # Finally, for the model indices\n self.__state.mapping_list_NWP_member_to_ensemble_member = np.repeat(\n self.__state.mapping_list_NWP_member_to_ensemble_member,\n n_ens_members_max,\n axis=0,\n )\n\n elif n_model_members == n_ens_members_min:\n repeats = [\n (n_ens_members_max + i) // n_ens_members_min\n for i in range(n_ens_members_min)\n ]\n self.__state.precip_models_cascades_timestep = np.repeat(\n self.__state.precip_models_cascades_timestep,\n repeats,\n axis=0,\n )\n self.__state.mean_models_timestep = np.repeat(\n self.__state.mean_models_timestep, repeats, axis=0\n )\n self.__state.std_models_timestep = np.repeat(\n self.__state.std_models_timestep, repeats, axis=0\n )\n self.__state.velocity_models_timestep = np.repeat(\n self.__state.velocity_models_timestep, repeats, axis=0\n )\n # For the prob. matching\n self.__state.precip_models_timestep = np.repeat(\n self.__state.precip_models_timestep, repeats, axis=0\n )\n # Finally, for the model indices\n self.__state.mapping_list_NWP_member_to_ensemble_member = np.repeat(\n self.__state.mapping_list_NWP_member_to_ensemble_member,\n repeats,\n axis=0,\n )\n\n def __determine_skill_for_current_timestep(self, t):\n \"\"\"\n Compute the skill of NWP model forecasts at t=0 using spatial correlation,\n ensuring skill decreases with increasing scale level. For t>0, update\n extrapolation skill based on lead time.\n \"\"\"\n if t == 0:\n # Calculate the initial skill of the (NWP) model forecasts at t=0.\n self.__params.rho_nwp_models = []\n for model_index in range(\n self.__state.precip_models_cascades_timestep.shape[0]\n ):\n rho_value = blending.skill_scores.spatial_correlation(\n obs=self.__state.precip_cascades[0, :, -1, :, :].copy(),\n mod=self.__state.precip_models_cascades_timestep[\n model_index, :, :, :\n ].copy(),\n domain_mask=self.__params.domain_mask,\n )\n self.__params.rho_nwp_models.append(rho_value)\n self.__params.rho_nwp_models = np.stack(self.__params.rho_nwp_models)\n\n # Ensure that the model skill decreases with increasing scale level.\n for model_index in range(\n self.__state.precip_models_cascades_timestep.shape[0]\n ):\n for i in range(\n 1, self.__state.precip_models_cascades_timestep.shape[1]\n ):\n if (\n self.__params.rho_nwp_models[model_index, i]\n > self.__params.rho_nwp_models[model_index, i - 1]\n ):\n # Set it equal to the previous scale level\n self.__params.rho_nwp_models[model_index, i] = (\n self.__params.rho_nwp_models[model_index, i - 1]\n )\n\n # Save this in the climatological skill file\n blending.clim.save_skill(\n current_skill=self.__params.rho_nwp_models,\n validtime=self.__issuetime,\n outdir_path=self.__config.outdir_path_skill,\n **self.__params.climatology_kwargs,\n )\n if t > 0:\n # Determine the skill of the components for lead time (t0 + t)\n # First for the extrapolation component. Only calculate it when t > 0.\n (\n self.__state.rho_extrap_cascade,\n self.__state.rho_extrap_cascade_prev,\n ) = blending.skill_scores.lt_dependent_cor_extrapolation(\n PHI=self.__params.PHI,\n correlations=self.__state.rho_extrap_cascade,\n correlations_prev=self.__state.rho_extrap_cascade_prev,\n )\n\n def __determine_NWP_skill_for_next_timestep(self, t, j, worker_state):\n \"\"\"\n Compute the skill of NWP model components for the next lead time (t0 + t),\n blending with extrapolation skill if configured. Updates the worker state\n with the final blended skill forecast.\n \"\"\"\n if self.__config.blend_nwp_members:\n rho_nwp_forecast = []\n for model_index in range(self.__params.rho_nwp_models.shape[0]):\n rho_value = blending.skill_scores.lt_dependent_cor_nwp(\n lt=(t * int(self.__config.timestep)),\n correlations=self.__params.rho_nwp_models[model_index],\n outdir_path=self.__config.outdir_path_skill,\n n_model=model_index,\n skill_kwargs=self.__params.climatology_kwargs,\n )\n rho_nwp_forecast.append(rho_value)\n rho_nwp_forecast = np.stack(rho_nwp_forecast)\n # Concatenate rho_extrap_cascade and rho_nwp\n worker_state.rho_final_blended_forecast = np.concatenate(\n (worker_state.rho_extrap_cascade[None, :], rho_nwp_forecast), axis=0\n )\n else:\n # TODO: check if j is the best accessor for this variable\n rho_nwp_forecast = blending.skill_scores.lt_dependent_cor_nwp(\n lt=(t * int(self.__config.timestep)),\n correlations=self.__params.rho_nwp_models[j],\n outdir_path=self.__config.outdir_path_skill,\n n_model=worker_state.mapping_list_NWP_member_to_ensemble_member[j],\n skill_kwargs=self.__params.climatology_kwargs,\n )\n # Concatenate rho_extrap_cascade and rho_nwp\n worker_state.rho_final_blended_forecast = np.concatenate(\n (worker_state.rho_extrap_cascade[None, :], rho_nwp_forecast[None, :]),\n axis=0,\n )\n\n def __determine_weights_per_component(self, t, worker_state):\n \"\"\"\n Compute blending weights for each component based on the selected method\n ('bps' or 'spn'). Weights are determined for both full blending and\n model-only scenarios, accounting for correlations and covariance.\n \"\"\"\n start_smoothing_to_final_weights = False\n if self.__config.timestep_start_full_nwp_weight is not None:\n if t > self.__config.timestep_start_full_nwp_weight:\n start_smoothing_to_final_weights = True\n # Weights following the bps method. These are needed for the velocity\n # weights prior to the advection step. If weights method spn is\n # selected, weights will be overwritten with those weights prior to\n # blending step.\n # weight = [(extr_field, n_model_fields, noise), n_cascade_levels, ...]\n if not start_smoothing_to_final_weights:\n worker_state.weights = calculate_weights_bps(\n worker_state.rho_final_blended_forecast\n )\n else:\n worker_state.weights = calculate_end_weights(\n previous_weights=self.__state.weights,\n timestep=t,\n n_timesteps=self.__timesteps[-1],\n start_full_nwp_weight=self.__config.timestep_start_full_nwp_weight,\n model_only=False,\n )\n\n # The model only weights\n if (\n self.__config.weights_method == \"bps\"\n and not start_smoothing_to_final_weights\n ):\n # Determine the weights of the components without the extrapolation\n # cascade, in case this is no data or outside the mask.\n worker_state.weights_model_only = calculate_weights_bps(\n worker_state.rho_final_blended_forecast[1:, :]\n )\n elif (\n self.__config.weights_method == \"spn\"\n and not start_smoothing_to_final_weights\n ):\n # Only the weights of the components without the extrapolation\n # cascade will be determined here. The full set of weights are\n # determined after the extrapolation step in this method.\n if (\n self.__config.blend_nwp_members\n and worker_state.precip_models_cascades_timestep.shape[0] > 1\n ):\n worker_state.weights_model_only = np.zeros(\n (\n worker_state.precip_models_cascades_timestep.shape[0] + 1,\n self.__config.n_cascade_levels,\n )\n )\n for i in range(self.__config.n_cascade_levels):\n # Determine the normalized covariance matrix (containing)\n # the cross-correlations between the models\n covariance_nwp_models = np.corrcoef(\n np.stack(\n [\n worker_state.precip_models_cascades_timestep[\n n_model, i, :, :\n ].flatten()\n for n_model in range(\n worker_state.precip_models_cascades_timestep.shape[\n 0\n ]\n )\n ]\n )\n )\n # Determine the weights for this cascade level\n worker_state.weights_model_only[:, i] = calculate_weights_spn(\n correlations=worker_state.rho_final_blended_forecast[1:, i],\n covariance=covariance_nwp_models,\n )\n else:\n # Same as correlation and noise is 1 - correlation\n worker_state.weights_model_only = calculate_weights_bps(\n worker_state.rho_final_blended_forecast[1:, :],\n )\n elif start_smoothing_to_final_weights:\n worker_state.weights_model_only = calculate_end_weights(\n previous_weights=self.__state.weights_model_only,\n timestep=t,\n n_timesteps=self.__timesteps[-1],\n start_full_nwp_weight=self.__config.timestep_start_full_nwp_weight,\n model_only=True,\n )\n else:\n raise ValueError(\n \"Unknown weights method %s: must be 'bps' or 'spn'\"\n % self.__config.weights_method\n )\n self.__state.weights = worker_state.weights\n self.__state.weights_model_only = worker_state.weights_model_only\n\n def __regress_extrapolation_and_noise_cascades(self, j, worker_state, t):\n \"\"\"\n Apply autoregressive (AR) updates to the extrapolation and noise cascades\n for the next time step. If noise is enabled, generate and decompose a\n spatially correlated noise field before applying the AR process.\n \"\"\"\n # Determine the epsilon, a cascade of temporally independent\n # but spatially correlated noise\n if self.__config.noise_method is not None:\n # generate noise field\n epsilon = self.__params.noise_generator(\n self.__params.perturbation_generator,\n randstate=worker_state.randgen_precip[j],\n fft_method=self.__params.fft_objs[j],\n domain=self.__config.domain,\n )\n\n # decompose the noise field into a cascade\n epsilon_decomposed = self.__params.decomposition_method(\n epsilon,\n self.__params.bandpass_filter,\n fft_method=self.__params.fft_objs[j],\n input_domain=self.__config.domain,\n output_domain=self.__config.domain,\n compute_stats=True,\n normalize=True,\n compact_output=True,\n )\n else:\n epsilon_decomposed = None\n\n # Regress the extrapolation component to the subsequent time step.\n # Iterate the AR(p) model for each cascade level\n if self.__config.nowcasting_method == \"external_nowcast\":\n for i in range(self.__config.n_cascade_levels):\n # Use a deterministic Externally computed nowcasting model\n worker_state.precip_cascades[j][i] = (\n self.__state.precip_nowcast_cascades[j][i][t]\n )\n\n # Follow the 'standard' STEPS blending approach as described in :cite:`Imhoff2023`\n elif self.__config.nowcasting_method == \"steps\":\n for i in range(self.__config.n_cascade_levels):\n # apply AR(p) process to extrapolation cascade level\n if (\n epsilon_decomposed is not None\n or self.__config.velocity_perturbation_method is not None\n ):\n worker_state.precip_cascades[j][i] = (\n autoregression.iterate_ar_model(\n worker_state.precip_cascades[j][i], self.__params.PHI[i, :]\n )\n )\n # Renormalize the cascade\n worker_state.precip_cascades[j][i][1] /= np.std(\n worker_state.precip_cascades[j][i][1]\n )\n else:\n # use the deterministic AR(p) model computed above if\n # perturbations are disabled\n worker_state.precip_cascades[j][i] = (\n worker_state.final_blended_forecast_non_perturbed[i]\n )\n\n if self.__config.noise_method is not None:\n # Regress the noise component to the subsequent time step\n # iterate the AR(p) model for each cascade level\n for i in range(self.__config.n_cascade_levels):\n # normalize the noise cascade\n if epsilon_decomposed is not None:\n epsilon_temp = epsilon_decomposed[\"cascade_levels\"][i]\n epsilon_temp *= self.__params.noise_std_coeffs[i]\n else:\n epsilon_temp = None\n # apply AR(p) process to noise cascade level\n # (Returns zero noise if epsilon_decomposed is None)\n worker_state.precip_noise_cascades[j][i] = (\n autoregression.iterate_ar_model(\n worker_state.precip_noise_cascades[j][i],\n self.__params.PHI[i, :],\n eps=epsilon_temp,\n )\n )\n\n epsilon_decomposed = None\n epsilon_temp = None\n\n def __perturb_blend_and_advect_extrapolation_and_noise_to_current_timestep(\n self, t, j, worker_state\n ):\n \"\"\"\n Apply perturbations, blend motion fields, and advect extrapolated and noise\n cascades to the current time step (or sub-timesteps). This step ensures\n realistic motion updates in nowcasting.\n \"\"\"\n # Settings and initialize the output\n extrap_kwargs_ = worker_state.extrapolation_kwargs.copy()\n extrap_kwargs_noise = worker_state.extrapolation_kwargs.copy()\n extrap_kwargs_pb = worker_state.extrapolation_kwargs.copy()\n velocity_perturbations_extrapolation = self.__velocity\n # The following should be accessible after this function\n worker_state.precip_extrapolated_decomp = []\n worker_state.noise_extrapolated_decomp = []\n worker_state.precip_extrapolated_probability_matching = []\n\n # Extrapolate per sub time step\n for t_sub in worker_state.subtimesteps:\n if t_sub > 0:\n t_diff_prev_subtimestep_int = t_sub - int(t_sub)\n if t_diff_prev_subtimestep_int > 0.0:\n if self.__config.nowcasting_method == \"steps\":\n precip_forecast_cascade_subtimestep = [\n (1.0 - t_diff_prev_subtimestep_int)\n * worker_state.precip_cascades_prev_subtimestep[j][i][-1, :]\n + t_diff_prev_subtimestep_int\n * worker_state.precip_cascades[j][i][-1, :]\n for i in range(self.__config.n_cascade_levels)\n ]\n if self.__config.noise_method is not None:\n noise_cascade_subtimestep = [\n (1.0 - t_diff_prev_subtimestep_int)\n * worker_state.cascade_noise_prev_subtimestep[j][i][-1, :]\n + t_diff_prev_subtimestep_int\n * worker_state.precip_noise_cascades[j][i][-1, :]\n for i in range(self.__config.n_cascade_levels)\n ]\n\n else:\n if self.__config.nowcasting_method == \"steps\":\n precip_forecast_cascade_subtimestep = [\n worker_state.precip_cascades_prev_subtimestep[j][i][-1, :]\n for i in range(self.__config.n_cascade_levels)\n ]\n if self.__config.noise_method is not None:\n noise_cascade_subtimestep = [\n worker_state.cascade_noise_prev_subtimestep[j][i][-1, :]\n for i in range(self.__config.n_cascade_levels)\n ]\n\n if self.__config.nowcasting_method == \"steps\":\n precip_forecast_cascade_subtimestep = np.stack(\n precip_forecast_cascade_subtimestep\n )\n if self.__config.noise_method is not None:\n noise_cascade_subtimestep = np.stack(noise_cascade_subtimestep)\n\n t_diff_prev_subtimestep = t_sub - worker_state.time_prev_timestep[j]\n worker_state.leadtime_since_start_forecast[j] += t_diff_prev_subtimestep\n\n # compute the perturbed motion field - include the NWP\n # velocities and the weights. Note that we only perturb\n # the extrapolation velocity field, as the NWP velocity\n # field is present per time step\n if self.__config.velocity_perturbation_method is not None:\n velocity_perturbations_extrapolation = (\n self.__velocity\n + self.__params.generate_velocity_noise(\n self.__params.velocity_perturbations[j],\n worker_state.leadtime_since_start_forecast[j]\n * self.__config.timestep,\n )\n )\n\n # Stack the perturbed extrapolation and the NWP velocities\n if self.__config.blend_nwp_members:\n velocity_stack_all = np.concatenate(\n (\n velocity_perturbations_extrapolation[None, :, :, :],\n worker_state.velocity_models_timestep,\n ),\n axis=0,\n )\n else:\n velocity_models = worker_state.velocity_models_timestep[j]\n velocity_stack_all = np.concatenate(\n (\n velocity_perturbations_extrapolation[None, :, :, :],\n velocity_models[None, :, :, :],\n ),\n axis=0,\n )\n velocity_models = None\n\n # Obtain a blended optical flow, using the weights of the\n # second cascade following eq. 24 in BPS2006\n velocity_blended = blending.utils.blend_optical_flows(\n flows=velocity_stack_all,\n weights=worker_state.weights[\n :-1, 1\n ], # [(extr_field, n_model_fields), cascade_level=2]\n )\n\n # Extrapolate both cascades to the next time step\n # First recompose the cascade, advect it and decompose it again\n # This is needed to remove the interpolation artefacts.\n # In addition, the number of extrapolations is greatly reduced\n\n # A. The extrapolation component\n if self.__config.nowcasting_method == \"steps\":\n # First, recompose the cascades into one forecast\n precip_forecast_recomp_subtimestep = (\n blending.utils.recompose_cascade(\n combined_cascade=precip_forecast_cascade_subtimestep,\n combined_mean=worker_state.mean_extrapolation,\n combined_sigma=worker_state.std_extrapolation,\n )\n )\n # Make sure we have values outside the mask\n if self.__params.zero_precip_radar:\n precip_forecast_recomp_subtimestep = np.nan_to_num(\n precip_forecast_recomp_subtimestep,\n copy=True,\n nan=self.__params.precip_zerovalue,\n posinf=self.__params.precip_zerovalue,\n neginf=self.__params.precip_zerovalue,\n )\n # Put back the mask\n precip_forecast_recomp_subtimestep[self.__params.domain_mask] = (\n np.nan\n )\n worker_state.extrapolation_kwargs[\"displacement_prev\"] = (\n worker_state.previous_displacement[j]\n )\n (\n precip_forecast_extrapolated_recomp_subtimestep_temp,\n worker_state.previous_displacement[j],\n ) = self.__params.extrapolation_method(\n precip_forecast_recomp_subtimestep,\n velocity_blended,\n [t_diff_prev_subtimestep],\n allow_nonfinite_values=True,\n **worker_state.extrapolation_kwargs,\n )\n precip_extrapolated_recomp_subtimestep = (\n precip_forecast_extrapolated_recomp_subtimestep_temp[0].copy()\n )\n temp_mask = ~np.isfinite(precip_extrapolated_recomp_subtimestep)\n # Set non-finite values to the zerovalue\n precip_extrapolated_recomp_subtimestep[\n ~np.isfinite(precip_extrapolated_recomp_subtimestep)\n ] = self.__params.precip_zerovalue\n # Decompose the forecast again into multiplicative cascades\n precip_extrapolated_decomp = self.__params.decomposition_method(\n precip_extrapolated_recomp_subtimestep,\n self.__params.bandpass_filter,\n mask=self.__params.mask_threshold,\n fft_method=self.__params.fft,\n output_domain=self.__config.domain,\n normalize=True,\n compute_stats=True,\n compact_output=True,\n )[\"cascade_levels\"]\n # Make sure we have values outside the mask\n if self.__params.zero_precip_radar:\n precip_extrapolated_decomp = np.nan_to_num(\n precip_extrapolated_decomp,\n copy=True,\n nan=np.nanmin(precip_forecast_cascade_subtimestep),\n posinf=np.nanmin(precip_forecast_cascade_subtimestep),\n neginf=np.nanmin(precip_forecast_cascade_subtimestep),\n )\n for i in range(self.__config.n_cascade_levels):\n precip_extrapolated_decomp[i][temp_mask] = np.nan\n\n # Append the results to the output lists\n worker_state.precip_extrapolated_decomp.append(\n precip_extrapolated_decomp.copy()\n )\n\n precip_forecast_cascade_subtimestep = None\n precip_forecast_recomp_subtimestep = None\n precip_forecast_extrapolated_recomp_subtimestep_temp = None\n precip_extrapolated_recomp_subtimestep = None\n precip_extrapolated_decomp = None\n\n # B. The noise component\n if self.__config.noise_method is not None:\n # First, recompose the cascades into one forecast\n noise_cascade_subtimestep_recomp = blending.utils.recompose_cascade(\n combined_cascade=noise_cascade_subtimestep,\n combined_mean=worker_state.precip_mean_noise[j],\n combined_sigma=worker_state.precip_std_noise[j],\n )\n extrap_kwargs_noise[\"displacement_prev\"] = (\n worker_state.previous_displacement_noise_cascade[j]\n )\n extrap_kwargs_noise[\"map_coordinates_mode\"] = \"wrap\"\n (\n noise_extrapolated_recomp_temp,\n worker_state.previous_displacement_noise_cascade[j],\n ) = self.__params.extrapolation_method(\n noise_cascade_subtimestep_recomp,\n velocity_blended,\n [t_diff_prev_subtimestep],\n allow_nonfinite_values=True,\n **extrap_kwargs_noise,\n )\n noise_extrapolated_recomp = noise_extrapolated_recomp_temp[0].copy()\n # Decompose the noise component again into multiplicative cascades\n noise_extrapolated_decomp = self.__params.decomposition_method(\n noise_extrapolated_recomp,\n self.__params.bandpass_filter,\n mask=self.__params.mask_threshold,\n fft_method=self.__params.fft,\n output_domain=self.__config.domain,\n normalize=True,\n compute_stats=True,\n compact_output=True,\n )[\"cascade_levels\"]\n for i in range(self.__config.n_cascade_levels):\n noise_extrapolated_decomp[i] *= self.__params.noise_std_coeffs[\n i\n ]\n\n # Append the results to the output lists\n worker_state.noise_extrapolated_decomp.append(\n noise_extrapolated_decomp.copy()\n )\n\n noise_cascade_subtimestep = None\n noise_cascade_subtimestep_recomp = None\n noise_extrapolated_recomp_temp = None\n noise_extrapolated_recomp = None\n noise_extrapolated_decomp = None\n\n # Finally, also extrapolate the initial radar rainfall field. This will be\n # blended with the rainfall field(s) of the (NWP) model(s) for Lagrangian\n # blended prob. matching min_R = np.min(precip). If we use an external\n # nowcast, this variable will be set later in this function.\n if self.__config.nowcasting_method == \"steps\":\n extrap_kwargs_pb[\"displacement_prev\"] = (\n worker_state.previous_displacement_prob_matching[j]\n )\n # Apply the domain mask to the extrapolation component\n precip_forecast_temp_for_probability_matching = self.__precip.copy()\n precip_forecast_temp_for_probability_matching[\n self.__params.domain_mask\n ] = np.nan\n\n (\n precip_forecast_extrapolated_probability_matching_temp,\n worker_state.previous_displacement_prob_matching[j],\n ) = self.__params.extrapolation_method(\n precip_forecast_temp_for_probability_matching,\n velocity_blended,\n [t_diff_prev_subtimestep],\n allow_nonfinite_values=True,\n **extrap_kwargs_pb,\n )\n\n worker_state.precip_extrapolated_probability_matching.append(\n precip_forecast_extrapolated_probability_matching_temp[0]\n )\n\n worker_state.time_prev_timestep[j] = t_sub\n\n if len(worker_state.precip_extrapolated_decomp) > 0:\n if self.__config.nowcasting_method == \"steps\":\n worker_state.precip_extrapolated_decomp = np.stack(\n worker_state.precip_extrapolated_decomp\n )\n worker_state.precip_extrapolated_probability_matching = np.stack(\n worker_state.precip_extrapolated_probability_matching\n )\n if len(worker_state.noise_extrapolated_decomp) > 0:\n if self.__config.noise_method is not None:\n worker_state.noise_extrapolated_decomp = np.stack(\n worker_state.noise_extrapolated_decomp\n )\n\n # advect the forecast field by one time step if no subtimesteps in the\n # current interval were found\n if not worker_state.subtimesteps:\n t_diff_prev_subtimestep = t + 1 - worker_state.time_prev_timestep[j]\n worker_state.leadtime_since_start_forecast[j] += t_diff_prev_subtimestep\n\n # compute the perturbed motion field - include the NWP\n # velocities and the weights\n if self.__config.velocity_perturbation_method is not None:\n velocity_perturbations_extrapolation = (\n self.__velocity\n + self.__params.generate_velocity_noise(\n self.__params.velocity_perturbations[j],\n worker_state.leadtime_since_start_forecast[j]\n * self.__config.timestep,\n )\n )\n\n # Stack the perturbed extrapolation and the NWP velocities\n if self.__config.blend_nwp_members:\n velocity_stack_all = np.concatenate(\n (\n velocity_perturbations_extrapolation[None, :, :, :],\n worker_state.velocity_models_timestep,\n ),\n axis=0,\n )\n else:\n velocity_models = worker_state.velocity_models_timestep[j]\n velocity_stack_all = np.concatenate(\n (\n velocity_perturbations_extrapolation[None, :, :, :],\n velocity_models[None, :, :, :],\n ),\n axis=0,\n )\n velocity_models = None\n\n # Obtain a blended optical flow, using the weights of the\n # second cascade following eq. 24 in BPS2006\n velocity_blended = blending.utils.blend_optical_flows(\n flows=velocity_stack_all,\n weights=worker_state.weights[\n :-1, 1\n ], # [(extr_field, n_model_fields), cascade_level=2]\n )\n\n # Extrapolate the extrapolation and noise cascade\n extrap_kwargs_[\"displacement_prev\"] = worker_state.previous_displacement[j]\n extrap_kwargs_noise[\"displacement_prev\"] = (\n worker_state.previous_displacement_noise_cascade[j]\n )\n extrap_kwargs_noise[\"map_coordinates_mode\"] = \"wrap\"\n\n # Extrapolate the extrapolation cascade\n if self.__config.nowcasting_method == \"steps\":\n (\n _,\n worker_state.previous_displacement[j],\n ) = self.__params.extrapolation_method(\n None,\n velocity_blended,\n [t_diff_prev_subtimestep],\n allow_nonfinite_values=True,\n **extrap_kwargs_,\n )\n # Extrapolate the noise cascade\n if self.__config.noise_method is not None:\n (\n _,\n worker_state.previous_displacement_noise_cascade[j],\n ) = self.__params.extrapolation_method(\n None,\n velocity_blended,\n [t_diff_prev_subtimestep],\n allow_nonfinite_values=True,\n **extrap_kwargs_noise,\n )\n # Also extrapolate the radar observation, used for the probability\n # matching and post-processing steps\n if self.__config.nowcasting_method == \"steps\":\n extrap_kwargs_pb[\"displacement_prev\"] = (\n worker_state.previous_displacement_prob_matching[j]\n )\n (\n _,\n worker_state.previous_displacement_prob_matching[j],\n ) = self.__params.extrapolation_method(\n None,\n velocity_blended,\n [t_diff_prev_subtimestep],\n allow_nonfinite_values=True,\n **extrap_kwargs_pb,\n )\n\n worker_state.time_prev_timestep[j] = t + 1\n\n # If an external nowcast is provided, precip_extrapolated_decomp and\n # precip_extrapolated_probability_matching have been omitted so far.\n # Fill them in with the external nowcast information now.\n if self.__config.nowcasting_method == \"external_nowcast\":\n for i in range(self.__config.n_cascade_levels):\n precip_extrapolated_decomp = worker_state.precip_cascades[j][i][-1, :]\n\n worker_state.time_prev_timestep[j] = t + 1\n\n worker_state.precip_extrapolated_decomp.append(\n precip_extrapolated_decomp.copy()\n )\n\n # Also update the probability matching fields\n precip_extrapolated = self.__precip_nowcast[j][t][:, :]\n worker_state.precip_extrapolated_probability_matching.append(\n precip_extrapolated.copy()\n )\n\n # Stack it for the output\n worker_state.precip_extrapolated_decomp = np.stack(\n worker_state.precip_extrapolated_decomp\n )[None, :]\n\n worker_state.precip_extrapolated_probability_matching = np.stack(\n worker_state.precip_extrapolated_probability_matching\n ) # [None, :]\n\n worker_state.precip_cascades_prev_subtimestep[j] = worker_state.precip_cascades[\n j\n ]\n worker_state.cascade_noise_prev_subtimestep[j] = (\n worker_state.precip_noise_cascades[j]\n )\n\n def __blend_cascades(self, t_sub, j, worker_state):\n \"\"\"\n Blend extrapolated, NWP model, and noise cascades using predefined weights.\n Computes both full and model-only blends while also blending means and\n standard deviations across scales.\n \"\"\"\n worker_state.subtimestep_index = np.where(\n np.array(worker_state.subtimesteps) == t_sub\n )[0][0]\n # First concatenate the cascades and the means and sigmas\n # precip_models = [n_models,timesteps,n_cascade_levels,m,n]\n if (\n self.__config.blend_nwp_members\n and self.__config.nowcasting_method == \"external_nowcast\"\n ):\n if self.__config.noise_method is None:\n cascade_stack_all_components = np.concatenate(\n (\n worker_state.precip_extrapolated_decomp[\n None, worker_state.subtimestep_index\n ],\n worker_state.precip_models_cascades_timestep,\n ),\n axis=0,\n ) # [(extr_field, n_model_fields), n_cascade_levels, ...]\n else:\n cascade_stack_all_components = np.concatenate(\n (\n worker_state.precip_extrapolated_decomp[\n None, worker_state.subtimestep_index\n ],\n worker_state.precip_models_cascades_timestep,\n worker_state.noise_extrapolated_decomp[\n None, worker_state.subtimestep_index\n ],\n ),\n axis=0,\n ) # [(extr_field, n_model_fields), n_cascade_levels, ...]\n means_stacked = np.concatenate(\n (\n worker_state.mean_nowcast_timestep[None, j, :],\n worker_state.mean_models_timestep,\n ),\n axis=0,\n )\n sigmas_stacked = np.concatenate(\n (\n worker_state.std_nowcast_timestep[None, j, :],\n worker_state.std_models_timestep,\n ),\n axis=0,\n )\n\n elif (\n self.__config.blend_nwp_members\n and self.__config.nowcasting_method == \"steps\"\n ):\n cascade_stack_all_components = np.concatenate(\n (\n worker_state.precip_extrapolated_decomp[\n None, worker_state.subtimestep_index\n ],\n worker_state.precip_models_cascades_timestep,\n worker_state.noise_extrapolated_decomp[\n None, worker_state.subtimestep_index\n ],\n ),\n axis=0,\n ) # [(extr_field, n_model_fields, noise), n_cascade_levels, ...]\n means_stacked = np.concatenate(\n (\n worker_state.mean_extrapolation[None, :],\n worker_state.mean_models_timestep,\n ),\n axis=0,\n )\n sigmas_stacked = np.concatenate(\n (\n worker_state.std_extrapolation[None, :],\n worker_state.std_models_timestep,\n ),\n axis=0,\n )\n\n elif self.__config.nowcasting_method == \"external_nowcast\":\n if self.__config.noise_method is None:\n cascade_stack_all_components = np.concatenate(\n (\n worker_state.precip_extrapolated_decomp[\n None, worker_state.subtimestep_index\n ],\n worker_state.precip_models_cascades_timestep[None, j],\n ),\n axis=0,\n ) # [(extr_field, n_model_fields), n_cascade_levels, ...]\n else:\n cascade_stack_all_components = np.concatenate(\n (\n worker_state.precip_extrapolated_decomp[\n None, worker_state.subtimestep_index\n ],\n worker_state.precip_models_cascades_timestep[None, j],\n worker_state.noise_extrapolated_decomp[\n None, worker_state.subtimestep_index\n ],\n ),\n axis=0,\n ) # [(extr_field, n_model_fields), n_cascade_levels, ...]\n means_stacked = np.concatenate(\n (\n worker_state.mean_nowcast_timestep[None, j, :],\n worker_state.mean_models_timestep[None, j],\n ),\n axis=0,\n )\n sigmas_stacked = np.concatenate(\n (\n worker_state.std_nowcast_timestep[None, j, :],\n worker_state.std_models_timestep[None, j],\n ),\n axis=0,\n )\n\n else:\n cascade_stack_all_components = np.concatenate(\n (\n worker_state.precip_extrapolated_decomp[\n None, worker_state.subtimestep_index\n ],\n worker_state.precip_models_cascades_timestep[None, j],\n worker_state.noise_extrapolated_decomp[\n None, worker_state.subtimestep_index\n ],\n ),\n axis=0,\n ) # [(extr_field, n_model_fields, noise), n_cascade_levels, ...]\n means_stacked = np.concatenate(\n (\n worker_state.mean_extrapolation[None, :],\n worker_state.mean_models_timestep[None, j],\n ),\n axis=0,\n )\n sigmas_stacked = np.concatenate(\n (\n worker_state.std_extrapolation[None, :],\n worker_state.std_models_timestep[None, j],\n ),\n axis=0,\n )\n\n # First determine the blending weights if method is spn. The\n # weights for method bps have already been determined.'\n start_smoothing_to_final_weights = False\n if self.__config.timestep_start_full_nwp_weight is not None:\n if t_sub >= self.__config.timestep_start_full_nwp_weight:\n start_smoothing_to_final_weights = True\n\n if (\n self.__config.weights_method == \"spn\"\n and not start_smoothing_to_final_weights\n ):\n worker_state.weights = np.zeros(\n (\n cascade_stack_all_components.shape[0],\n self.__config.n_cascade_levels,\n )\n )\n for i in range(self.__config.n_cascade_levels):\n # Determine the normalized covariance matrix (containing)\n # the cross-correlations between the models\n cascade_stack_all_components_temp = np.stack(\n [\n cascade_stack_all_components[n_model, i, :, :].flatten()\n for n_model in range(cascade_stack_all_components.shape[0] - 1)\n ]\n ) # -1 to exclude the noise component\n covariance_nwp_models = np.ma.corrcoef(\n np.ma.masked_invalid(cascade_stack_all_components_temp)\n )\n # Determine the weights for this cascade level\n worker_state.weights[:, i] = calculate_weights_spn(\n correlations=worker_state.rho_final_blended_forecast[:, i],\n covariance=covariance_nwp_models,\n )\n\n self.__state.weights = worker_state.weights\n\n # Create weights_with_noise to ensure there is always a 3D weights field, even\n # if self.__config.nowcasting_method is \"external_nowcast\" and n_ens_members is 1.\n worker_state.weights_with_noise = worker_state.weights.copy()\n worker_state.weights_model_only_with_noise = (\n worker_state.weights_model_only.copy()\n )\n if (\n self.__config.nowcasting_method == \"external_nowcast\"\n and self.__config.noise_method is None\n ):\n # First determine the weights without noise\n worker_state.weights = worker_state.weights[:-1, :] / np.sum(\n worker_state.weights[:-1, :], axis=0\n )\n\n worker_state.weights_model_only = worker_state.weights_model_only[\n :-1, :\n ] / np.sum(worker_state.weights_model_only[:-1, :], axis=0)\n # Blend the extrapolation, (NWP) model(s) and noise cascades\n worker_state.final_blended_forecast_cascades = (\n blending.utils.blend_cascades(\n cascades_norm=cascade_stack_all_components,\n weights=worker_state.weights,\n )\n )\n # Also blend the cascade without the extrapolation component\n worker_state.final_blended_forecast_cascades_mod_only = (\n blending.utils.blend_cascades(\n cascades_norm=cascade_stack_all_components[1:, :],\n weights=worker_state.weights_model_only,\n )\n )\n else:\n # Blend the extrapolation, (NWP) model(s) and noise cascades\n worker_state.final_blended_forecast_cascades = (\n blending.utils.blend_cascades(\n cascades_norm=cascade_stack_all_components,\n weights=worker_state.weights_with_noise,\n )\n )\n # Also blend the cascade without the extrapolation component\n worker_state.final_blended_forecast_cascades_mod_only = (\n blending.utils.blend_cascades(\n cascades_norm=cascade_stack_all_components[1:, :],\n weights=worker_state.weights_model_only,\n )\n )\n\n # Blend the means and standard deviations\n # Input is array of shape [number_components, scale_level, ...]\n (\n worker_state.final_blended_forecast_means,\n worker_state.final_blended_forecast_stds,\n ) = blend_means_sigmas(\n means=means_stacked,\n sigmas=sigmas_stacked,\n weights=worker_state.weights_with_noise,\n )\n # Also blend the means and sigmas for the cascade without extrapolation\n (\n worker_state.final_blended_forecast_means_mod_only,\n worker_state.final_blended_forecast_stds_mod_only,\n ) = blend_means_sigmas(\n means=means_stacked[1:, :],\n sigmas=sigmas_stacked[1:, :],\n weights=worker_state.weights_model_only_with_noise,\n )\n\n def __recompose_cascade_to_rainfall_field(self, j, worker_state):\n \"\"\"\n Recompose the blended cascade into a precipitation field using the blended\n means and standard deviations. If using the spectral domain, apply inverse\n FFT for reconstruction.\n \"\"\"\n worker_state.final_blended_forecast_recomposed = (\n blending.utils.recompose_cascade(\n combined_cascade=worker_state.final_blended_forecast_cascades,\n combined_mean=worker_state.final_blended_forecast_means,\n combined_sigma=worker_state.final_blended_forecast_stds,\n )\n )\n # The recomposed cascade without the extrapolation (for NaN filling\n # outside the radar domain)\n worker_state.final_blended_forecast_recomposed_mod_only = (\n blending.utils.recompose_cascade(\n combined_cascade=worker_state.final_blended_forecast_cascades_mod_only,\n combined_mean=worker_state.final_blended_forecast_means_mod_only,\n combined_sigma=worker_state.final_blended_forecast_stds_mod_only,\n )\n )\n if self.__config.domain == \"spectral\":\n # TODO: Check this! (Only tested with domain == 'spatial')\n worker_state.final_blended_forecast_recomposed = self.__params.fft_objs[\n j\n ].irfft2(worker_state.final_blended_forecast_recomposed)\n worker_state.final_blended_forecast_recomposed_mod_only = (\n self.__params.fft_objs[j].irfft2(\n worker_state.final_blended_forecast_recomposed_mod_only\n )\n )\n\n def __post_process_output(\n self, j, t_sub, final_blended_forecast_single_member, worker_state\n ):\n \"\"\"\n Apply post-processing steps to refine the final blended forecast. This\n involves masking, filling missing data with the blended NWP forecast,\n and applying probability matching to ensure consistency.\n\n **Steps:**\n\n 1. **Use Mask and Fill Missing Data:**\n - Areas without reliable radar extrapolation are filled using the\n blended NWP forecast to maintain spatial coherence.\n\n 2. **Lagrangian Blended Probability Matching:**\n - Uses the latest extrapolated radar rainfall field blended with\n the NWP model(s) forecast as a reference.\n - Ensures that the statistical distribution of the final forecast\n remains consistent with the benchmark dataset.\n\n 3. **Blend the Extrapolated Rainfall Field with NWP Forecasts:**\n - The extrapolated rainfall field is used only for post-processing.\n - The forecast is blended using predefined weights at scale level 2.\n - This ensures that both extrapolated and modeled precipitation\n contribute appropriately to the final output.\n\n 4. **Apply Probability Matching:**\n - Adjusts the final precipitation distribution using either empirical\n cumulative distribution functions (CDF) or mean adjustments to\n match the reference dataset.\n\n The final processed forecast is stored in `final_blended_forecast_single_member`.\n \"\"\"\n\n weights_probability_matching = worker_state.weights_with_noise[\n :-1, 1\n ] # Weights without noise, level 2\n weights_probability_matching_normalized = weights_probability_matching / np.sum(\n weights_probability_matching\n )\n\n # And the weights for outside the radar domain\n weights_probability_matching_mod_only = (\n worker_state.weights_model_only_with_noise[:-1, 1]\n ) # Weights without noise, level 2\n weights_probability_matching_normalized_mod_only = (\n weights_probability_matching_mod_only\n / np.sum(weights_probability_matching_mod_only)\n )\n # Stack the fields\n if self.__config.blend_nwp_members:\n precip_forecast_probability_matching_final = np.concatenate(\n (\n worker_state.precip_extrapolated_probability_matching[\n None, worker_state.subtimestep_index\n ],\n worker_state.precip_models_timestep,\n ),\n axis=0,\n )\n else:\n precip_forecast_probability_matching_final = np.concatenate(\n (\n worker_state.precip_extrapolated_probability_matching[\n None, worker_state.subtimestep_index\n ],\n worker_state.precip_models_timestep[None, j],\n ),\n axis=0,\n )\n # Blend it\n precip_forecast_probability_matching_blended = np.sum(\n weights_probability_matching_normalized.reshape(\n weights_probability_matching_normalized.shape[0], 1, 1\n )\n * precip_forecast_probability_matching_final,\n axis=0,\n )\n if self.__config.blend_nwp_members:\n precip_forecast_probability_matching_blended_mod_only = np.sum(\n weights_probability_matching_normalized_mod_only.reshape(\n weights_probability_matching_normalized_mod_only.shape[0],\n 1,\n 1,\n )\n * worker_state.precip_models_timestep,\n axis=0,\n )\n else:\n precip_forecast_probability_matching_blended_mod_only = (\n worker_state.precip_models_timestep[j]\n )\n\n # The extrapolation components are NaN outside the advected\n # radar domain. This results in NaN values in the blended\n # forecast outside the radar domain. Therefore, fill these\n # areas with the \"..._mod_only\" blended forecasts, consisting\n # of the NWP and noise components.\n nan_indices = np.isnan(worker_state.final_blended_forecast_recomposed)\n if self.__config.smooth_radar_mask_range != 0:\n # Compute the smooth dilated mask\n new_mask = blending.utils.compute_smooth_dilated_mask(\n nan_indices,\n max_padding_size_in_px=self.__config.smooth_radar_mask_range,\n )\n\n # Ensure mask values are between 0 and 1\n mask_model = np.clip(new_mask, 0, 1)\n mask_radar = np.clip(1 - new_mask, 0, 1)\n\n # Handle NaNs in precip_forecast_new and precip_forecast_new_mod_only by setting NaNs to 0 in the blending step\n precip_forecast_recomposed_mod_only_no_nan = np.nan_to_num(\n worker_state.final_blended_forecast_recomposed_mod_only, nan=0\n )\n precip_forecast_recomposed_no_nan = np.nan_to_num(\n worker_state.final_blended_forecast_recomposed, nan=0\n )\n\n # Perform the blending of radar and model inside the radar domain using a weighted combination\n worker_state.final_blended_forecast_recomposed = np.nansum(\n [\n mask_model * precip_forecast_recomposed_mod_only_no_nan,\n mask_radar * precip_forecast_recomposed_no_nan,\n ],\n axis=0,\n )\n precip_forecast_probability_matching_blended = np.nansum(\n [\n precip_forecast_probability_matching_blended * mask_radar,\n precip_forecast_probability_matching_blended_mod_only * mask_model,\n ],\n axis=0,\n )\n else:\n worker_state.final_blended_forecast_recomposed[nan_indices] = (\n worker_state.final_blended_forecast_recomposed_mod_only[nan_indices]\n )\n nan_indices = np.isnan(precip_forecast_probability_matching_blended)\n precip_forecast_probability_matching_blended[nan_indices] = (\n precip_forecast_probability_matching_blended_mod_only[nan_indices]\n )\n\n # Finally, fill the remaining nan values, if present, with\n # the minimum value in the forecast\n nan_indices = np.isnan(worker_state.final_blended_forecast_recomposed)\n worker_state.final_blended_forecast_recomposed[nan_indices] = np.nanmin(\n worker_state.final_blended_forecast_recomposed\n )\n nan_indices = np.isnan(precip_forecast_probability_matching_blended)\n precip_forecast_probability_matching_blended[nan_indices] = np.nanmin(\n precip_forecast_probability_matching_blended\n )\n\n # Apply the masking and prob. matching\n precip_field_mask_temp = None\n if self.__config.mask_method is not None:\n # apply the precipitation mask to prevent generation of new\n # precipitation into areas where it was not originally\n # observed\n precip_forecast_min_value = (\n worker_state.final_blended_forecast_recomposed.min()\n )\n if self.__config.mask_method == \"incremental\":\n # The incremental mask is slightly different from the implementation in\n # nowcasts.steps.py, as it is not computed in the Lagrangian space. Instead,\n # we use precip_forecast_probability_matched and let the mask_rim increase with\n # the time step until mask_rim_max. This ensures that for the first t time\n # steps, the buffer mask keeps increasing.\n precip_field_mask = (\n precip_forecast_probability_matching_blended\n >= self.__params.precip_threshold\n )\n\n # Buffer the mask\n # Convert the precipitation field mask into an 8-bit unsigned integer mask\n obs_mask_uint8 = precip_field_mask.astype(\"uint8\")\n\n # Perform an initial binary dilation using the provided structuring element\n dilated_mask = binary_dilation(obs_mask_uint8, self.__params.struct)\n\n # Create a binary structure element for incremental dilations\n struct_element = generate_binary_structure(2, 1)\n\n # Initialize a floating-point mask to accumulate dilations for a smooth transition\n accumulated_mask = dilated_mask.astype(float)\n\n # Iteratively dilate the mask and accumulate the results to create a grayscale rim\n mask_rim_temp = min(\n self.__params.mask_rim + t_sub - 1, self.__params.max_mask_rim\n )\n for _ in range(mask_rim_temp):\n dilated_mask = binary_dilation(dilated_mask, struct_element)\n accumulated_mask += dilated_mask\n\n # Normalize the accumulated mask values between 0 and 1\n precip_field_mask = accumulated_mask / np.max(accumulated_mask)\n # Get the final mask\n worker_state.final_blended_forecast_recomposed = (\n precip_forecast_min_value\n + (\n worker_state.final_blended_forecast_recomposed\n - precip_forecast_min_value\n )\n * precip_field_mask\n )\n precip_field_mask_temp = (\n worker_state.final_blended_forecast_recomposed\n > precip_forecast_min_value\n )\n elif self.__config.mask_method == \"obs\":\n # The mask equals the most recent benchmark\n # rainfall field\n precip_field_mask_temp = (\n precip_forecast_probability_matching_blended\n >= self.__params.precip_threshold\n )\n\n # Set to min value outside of mask\n worker_state.final_blended_forecast_recomposed[~precip_field_mask_temp] = (\n precip_forecast_min_value\n )\n\n # If probmatching_method is not None, resample the distribution from\n # both the extrapolation cascade and the model (NWP) cascade and use\n # that for the probability matching.\n if (\n self.__config.probmatching_method is not None\n and self.__config.resample_distribution\n ):\n arr1 = worker_state.precip_extrapolated_probability_matching[\n worker_state.subtimestep_index\n ]\n arr2 = worker_state.precip_models_timestep[j]\n # resample weights based on cascade level 2.\n # Areas where one of the fields is nan are not included.\n precip_forecast_probability_matching_resampled = (\n probmatching.resample_distributions(\n first_array=arr1,\n second_array=arr2,\n probability_first_array=weights_probability_matching_normalized[0],\n randgen=worker_state.randgen_probmatching[j],\n )\n )\n else:\n precip_forecast_probability_matching_resampled = (\n precip_forecast_probability_matching_blended.copy()\n )\n\n if self.__config.probmatching_method == \"cdf\":\n # nan indices in the extrapolation nowcast\n nan_indices = np.isnan(\n worker_state.precip_extrapolated_probability_matching[\n worker_state.subtimestep_index\n ]\n )\n\n # Adjust the CDF of the forecast to match the resampled distribution combined from\n # extrapolation and model fields.\n # Rainfall outside the pure extrapolation domain is not taken into account.\n\n if np.any(np.isfinite(worker_state.final_blended_forecast_recomposed)):\n worker_state.final_blended_forecast_recomposed = (\n probmatching.nonparam_match_empirical_cdf(\n worker_state.final_blended_forecast_recomposed,\n precip_forecast_probability_matching_resampled,\n nan_indices,\n )\n )\n precip_forecast_probability_matching_resampled = None\n\n elif self.__config.probmatching_method == \"mean\":\n # Use R_pm_blended as benchmark field and\n mean_probabiltity_matching_forecast = np.mean(\n precip_forecast_probability_matching_resampled[\n precip_forecast_probability_matching_resampled\n >= self.__params.precip_threshold\n ]\n )\n no_rain_mask = (\n worker_state.final_blended_forecast_recomposed\n >= self.__params.precip_threshold\n )\n mean_precip_forecast = np.mean(\n worker_state.final_blended_forecast_recomposed[no_rain_mask]\n )\n worker_state.final_blended_forecast_recomposed[no_rain_mask] = (\n worker_state.final_blended_forecast_recomposed[no_rain_mask]\n - mean_precip_forecast\n + mean_probabiltity_matching_forecast\n )\n precip_forecast_probability_matching_resampled = None\n\n final_blended_forecast_single_member.append(\n worker_state.final_blended_forecast_recomposed\n )\n return final_blended_forecast_single_member\n\n def __measure_time(self, label, start_time):\n \"\"\"\n Measure and print the time taken for a specific part of the process.\n\n Parameters:\n - label: A description of the part of the process being measured.\n - start_time: The timestamp when the process started (from time.time()).\n \"\"\"\n if self.__config.measure_time:\n elapsed_time = time.time() - start_time\n print(f\"{label} took {elapsed_time:.2f} seconds.\")\n return elapsed_time\n return None\n\n\ndef forecast(\n precip,\n precip_models,\n velocity,\n velocity_models,\n timesteps,\n timestep,\n issuetime,\n n_ens_members,\n precip_nowcast=None,\n n_cascade_levels=6,\n blend_nwp_members=False,\n precip_thr=None,\n norain_thr=0.0,\n kmperpixel=None,\n extrap_method=\"semilagrangian\",\n decomp_method=\"fft\",\n bandpass_filter_method=\"gaussian\",\n nowcasting_method=\"steps\",\n noise_method=\"nonparametric\",\n noise_stddev_adj=None,\n ar_order=2,\n vel_pert_method=\"bps\",\n weights_method=\"bps\",\n timestep_start_full_nwp_weight=None,\n conditional=False,\n probmatching_method=\"cdf\",\n mask_method=\"incremental\",\n resample_distribution=True,\n smooth_radar_mask_range=0,\n callback=None,\n return_output=True,\n seed=None,\n num_workers=1,\n fft_method=\"numpy\",\n domain=\"spatial\",\n outdir_path_skill=\"./tmp/\",\n extrap_kwargs=None,\n filter_kwargs=None,\n noise_kwargs=None,\n vel_pert_kwargs=None,\n clim_kwargs=None,\n mask_kwargs=None,\n measure_time=False,\n):\n \"\"\"\n Generate a blended nowcast ensemble by using the Short-Term Ensemble\n Prediction System (STEPS) method.\n\n Parameters\n ----------\n precip: array-like\n Array of shape (ar_order+1,m,n) containing the input precipitation fields\n ordered by timestamp from oldest to newest. The time steps between the\n inputs are assumed to be regular.\n precip_models: array-like\n Either raw (NWP) model forecast data or decomposed (NWP) model forecast data.\n If you supply decomposed data, it needs to be an array of shape\n (n_models,timesteps+1) containing, per timestep (t=0 to lead time here) and\n per (NWP) model or model ensemble member, a dictionary with a list of cascades\n obtained by calling a method implemented in :py:mod:`pysteps.cascade.decomposition`.\n If you supply the original (NWP) model forecast data, it needs to be an array of shape\n (n_models,timestep+1,m,n) containing precipitation (or other) fields, which will\n then be decomposed in this function.\n\n Depending on your use case it can be advantageous to decompose the model\n forecasts outside beforehand, as this slightly reduces calculation times.\n This is possible with :py:func:`pysteps.blending.utils.decompose_NWP`,\n :py:func:`pysteps.blending.utils.compute_store_nwp_motion`, and\n :py:func:`pysteps.blending.utils.load_NWP`. However, if you have a lot of (NWP) model\n members (e.g. 1 model member per nowcast member), this can lead to excessive memory\n usage.\n\n To further reduce memory usage, both this array and the ``velocity_models`` array\n can be given as float32. They will then be converted to float64 before computations\n to minimize loss in precision.\n\n In case of one (deterministic) model as input, add an extra dimension to make sure\n precip_models is four dimensional prior to calling this function.\n velocity: array-like\n Array of shape (2,m,n) containing the x- and y-components of the advection\n field. The velocities are assumed to represent one time step between the\n inputs. All values are required to be finite.\n velocity_models: array-like\n Array of shape (n_models,timestep,2,m,n) containing the x- and y-components\n of the advection field for the (NWP) model field per forecast lead time.\n All values are required to be finite. To reduce memory usage, this array can\n be given as float32. They will then be converted to float64 before computations\n to minimize loss in precision.\n timesteps: int or list of floats\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed (relative to the input time step). The elements of\n the list are required to be in ascending order.\n timestep: float\n Time step of the motion vectors (minutes). Required if vel_pert_method is\n not None or mask_method is 'incremental'.\n issuetime: datetime\n is issued.\n n_ens_members: int\n The number of ensemble members to generate. This number should always be\n equal to or larger than the number of NWP ensemble members / number of\n NWP models.\n precip_nowcast: array-like, optional\n Optional input with array of shape (n_ens_members,timestep+1,m,n) containing\n and external nowcast as input to the blending. If precip_nowcast is provided,\n the autoregression step and advection step will be omitted for the\n extrapolation cascade of the blending procedure and instead, precip_nowcast\n will be used as estimate. Defaults to None (which is the standard STEPS)\n method described in :cite:`Imhoff2023`.\n Note that nowcasting_method should be set to 'external_nowcast' if\n precip_nowcast is not None.\n Note that in the current setup, only a deterministic precip_nowcast model can\n be provided and only one ensemble member (without noise generation) is\n returned. This will change soon.\n n_cascade_levels: int, optional\n The number of cascade levels to use. Defaults to 6,\n see issue #385 on GitHub.\n blend_nwp_members: bool\n Check if NWP models/members should be used individually, or if all of\n them are blended together per nowcast ensemble member. Standard set to\n false.\n precip_thr: float, optional\n Specifies the threshold value for minimum observable precipitation\n intensity. Required if mask_method is not None or conditional is True.\n norain_thr: float\n Specifies the threshold value for the fraction of rainy (see above) pixels\n in the radar rainfall field below which we consider there to be no rain.\n Depends on the amount of clutter typically present.\n Standard set to 0.0\n kmperpixel: float, optional\n Spatial resolution of the input data (kilometers/pixel). Required if\n vel_pert_method is not None or mask_method is 'incremental'.\n extrap_method: str, optional\n Name of the extrapolation method to use. See the documentation of\n :py:mod:`pysteps.extrapolation.interface`.\n decomp_method: {'fft'}, optional\n Name of the cascade decomposition method to use. See the documentation\n of :py:mod:`pysteps.cascade.interface`.\n bandpass_filter_method: {'gaussian', 'uniform'}, optional\n Name of the bandpass filter method to use with the cascade decomposition.\n See the documentation of :py:mod:`pysteps.cascade.interface`.\n nowcasting_method: {'steps', 'external_nowcast'},\n Name of the nowcasting method used to generate the nowcasts. If an external\n nowcast is provided, the script will use this as input and bypass the\n autoregression and advection of the extrapolation cascade. Defaults to 'steps',\n which follows the method described in :cite:`Imhoff2023`. Note, if\n nowcasting_method is 'external_nowcast', precip_nowcast cannot be None.\n noise_method: {'parametric','nonparametric','ssft','nested',None}, optional\n Name of the noise generator to use for perturbating the precipitation\n field. See the documentation of :py:mod:`pysteps.noise.interface`. If set to None,\n no noise is generated.\n noise_stddev_adj: {'auto','fixed',None}, optional\n Optional adjustment for the standard deviations of the noise fields added\n to each cascade level. This is done to compensate incorrect std. dev.\n estimates of casace levels due to presence of no-rain areas. 'auto'=use\n the method implemented in :py:func:`pysteps.noise.utils.compute_noise_stddev_adjs`.\n 'fixed'= use the formula given in :cite:`BPS2006` (eq. 6), None=disable\n noise std. dev adjustment.\n ar_order: int, optional\n The order of the autoregressive model to use. Must be >= 1.\n vel_pert_method: {'bps',None}, optional\n Name of the noise generator to use for perturbing the advection field. See\n the documentation of :py:mod:`pysteps.noise.interface`. If set to None, the advection\n field is not perturbed.\n weights_method: {'bps','spn'}, optional\n The calculation method of the blending weights. Options are the method\n by :cite:`BPS2006` and the covariance-based method by :cite:`SPN2013`.\n Defaults to bps.\n timestep_start_full_nwp_weight: int, optional.\n The timestep, which should be smaller than timesteps, at which a linear\n transition takes place from the calculated weights to full (1.0) NWP weight\n (and zero extrapolation and noise weight) to ensure the blending\n procedure becomes equal to the NWP forecast(s) at the last timestep\n of the blending procedure. If not provided, the blending stick to the\n theoretical weights provided by the chosen weights_method for a given\n lead time and skill of each blending component.\n conditional: bool, optional\n If set to True, compute the statistics of the precipitation field\n conditionally by excluding pixels where the values are below the threshold\n precip_thr.\n probmatching_method: {'cdf','mean',None}, optional\n Method for matching the statistics of the forecast field with those of\n the most recently observed one. 'cdf'=map the forecast CDF to the observed\n one, 'mean'=adjust only the conditional mean value of the forecast field\n in precipitation areas, None=no matching applied. Using 'mean' requires\n that mask_method is not None.\n mask_method: {'obs','incremental',None}, optional\n The method to use for masking no precipitation areas in the forecast field.\n The masked pixels are set to the minimum value of the observations.\n 'obs' = apply precip_thr to the most recently observed precipitation intensity\n field, 'incremental' = iteratively buffer the mask with a certain rate\n (currently it is 1 km/min), None=no masking.\n resample_distribution: bool, optional\n Method to resample the distribution from the extrapolation and NWP cascade as input\n for the probability matching. Not resampling these distributions may lead to losing\n some extremes when the weight of both the extrapolation and NWP cascade is similar.\n Defaults to True.\n smooth_radar_mask_range: int, Default is 0.\n Method to smooth the transition between the radar-NWP-noise blend and the NWP-noise\n blend near the edge of the radar domain (radar mask), where the radar data is either\n not present anymore or is not reliable. If set to 0 (grid cells), this generates a\n normal forecast without smoothing. To create a smooth mask, this range should be a\n positive value, representing a buffer band of a number of pixels by which the mask\n is cropped and smoothed. The smooth radar mask removes the hard edges between NWP\n and radar in the final blended product. Typically, a value between 50 and 100 km\n can be used. 80 km generally gives good results.\n callback: function, optional\n Optional function that is called after computation of each time step of\n the nowcast. The function takes one argument: a three-dimensional array\n of shape (n_ens_members,h,w), where h and w are the height and width\n of the input field precip, respectively. This can be used, for instance,\n writing the outputs into files.\n return_output: bool, optional\n Set to False to disable returning the outputs as numpy arrays. This can\n save memory if the intermediate results are written to output files using\n the callback function.\n seed: int, optional\n Optional seed number for the random generators.\n num_workers: int, optional\n The number of workers to use for parallel computation. Applicable if dask\n is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it\n is advisable to disable OpenMP by setting the environment variable\n OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous\n threads.\n fft_method: str, optional\n A string defining the FFT method to use (see FFT methods in\n :py:func:`pysteps.utils.interface.get_method`).\n Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed,\n the recommended method is 'pyfftw'.\n domain: {\"spatial\", \"spectral\"}\n If \"spatial\", all computations are done in the spatial domain (the\n classical STEPS model). If \"spectral\", the AR(2) models and stochastic\n perturbations are applied directly in the spectral domain to reduce\n memory footprint and improve performance :cite:`PCH2019b`.\n outdir_path_skill: string, optional\n Path to folder where the historical skill are stored. Defaults to\n path_workdir from rcparams. If no path is given, './tmp' will be used.\n extrap_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the extrapolation\n method. See the documentation of :py:func:`pysteps.extrapolation.interface`.\n filter_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the filter method.\n See the documentation of :py:mod:`pysteps.cascade.bandpass_filters`.\n noise_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the initializer of\n the noise generator. See the documentation of :py:mod:`pysteps.noise.fftgenerators`.\n vel_pert_kwargs: dict, optional\n Optional dictionary containing keyword arguments 'p_par' and 'p_perp' for\n the initializer of the velocity perturbator. The choice of the optimal\n parameters depends on the domain and the used optical flow method.\n\n Default parameters from :cite:`BPS2006`:\n p_par = [10.88, 0.23, -7.68]\n p_perp = [5.76, 0.31, -2.72]\n\n Parameters fitted to the data (optical flow/domain):\n\n darts/fmi:\n p_par = [13.71259667, 0.15658963, -16.24368207]\n p_perp = [8.26550355, 0.17820458, -9.54107834]\n\n darts/mch:\n p_par = [24.27562298, 0.11297186, -27.30087471]\n p_perp = [-7.80797846e+01, -3.38641048e-02, 7.56715304e+01]\n\n darts/fmi+mch:\n p_par = [16.55447057, 0.14160448, -19.24613059]\n p_perp = [14.75343395, 0.11785398, -16.26151612]\n\n lucaskanade/fmi:\n p_par = [2.20837526, 0.33887032, -2.48995355]\n p_perp = [2.21722634, 0.32359621, -2.57402761]\n\n lucaskanade/mch:\n p_par = [2.56338484, 0.3330941, -2.99714349]\n p_perp = [1.31204508, 0.3578426, -1.02499891]\n\n lucaskanade/fmi+mch:\n p_par = [2.31970635, 0.33734287, -2.64972861]\n p_perp = [1.90769947, 0.33446594, -2.06603662]\n\n vet/fmi:\n p_par = [0.25337388, 0.67542291, 11.04895538]\n p_perp = [0.02432118, 0.99613295, 7.40146505]\n\n vet/mch:\n p_par = [0.5075159, 0.53895212, 7.90331791]\n p_perp = [0.68025501, 0.41761289, 4.73793581]\n\n vet/fmi+mch:\n p_par = [0.29495222, 0.62429207, 8.6804131 ]\n p_perp = [0.23127377, 0.59010281, 5.98180004]\n\n fmi=Finland, mch=Switzerland, fmi+mch=both pooled into the same data set\n\n The above parameters have been fitted by using run_vel_pert_analysis.py\n and fit_vel_pert_params.py located in the scripts directory.\n\n See :py:mod:`pysteps.noise.motion` for additional documentation.\n clim_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the climatological\n skill file. Arguments can consist of: 'outdir_path', 'n_models'\n (the number of NWP models) and 'window_length' (the minimum number of\n days the clim file should have, otherwise the default is used).\n mask_kwargs: dict\n Optional dictionary containing mask keyword arguments 'mask_f',\n 'mask_rim' and 'max_mask_rim', the factor defining the the mask\n increment and the (maximum) rim size, respectively.\n The mask increment is defined as mask_f*timestep/kmperpixel.\n measure_time: bool\n If set to True, measure, print and return the computation time.\n\n Returns\n -------\n out: ndarray\n If return_output is True, a four-dimensional array of shape\n (n_ens_members,num_timesteps,m,n) containing a time series of forecast\n precipitation fields for each ensemble member. Otherwise, a None value\n is returned. The time series starts from t0+timestep, where timestep is\n taken from the input precipitation fields precip. If measure_time is True, the\n return value is a three-element tuple containing the nowcast array, the\n initialization time of the nowcast generator and the time used in the\n main loop (seconds).\n\n See also\n --------\n :py:mod:`pysteps.extrapolation.interface`, :py:mod:`pysteps.cascade.interface`,\n :py:mod:`pysteps.noise.interface`, :py:func:`pysteps.noise.utils.compute_noise_stddev_adjs`\n\n References\n ----------\n :cite:`Seed2003`, :cite:`BPS2004`, :cite:`BPS2006`, :cite:`SPN2013`, :cite:`PCH2019b`\n\n Notes\n -----\n 1. The blending currently does not blend the beta-parameters in the parametric\n noise method. It is recommended to use the non-parameteric noise method.\n\n 2. If blend_nwp_members is True, the BPS2006 method for the weights is\n suboptimal. It is recommended to use the SPN2013 method instead.\n\n 3. Not yet implemented (and neither in the steps nowcasting module): The regression\n of the lag-1 and lag-2 parameters to their climatological values. See also eq.\n 12 - 19 in :cite: `BPS2004`. By doing so, the Phi parameters change over time,\n which enhances the AR process. This can become a future development if this\n turns out to be a warranted functionality.\n \"\"\"\n\n blending_config = StepsBlendingConfig(\n n_ens_members=n_ens_members,\n n_cascade_levels=n_cascade_levels,\n blend_nwp_members=blend_nwp_members,\n precip_threshold=precip_thr,\n norain_threshold=norain_thr,\n kmperpixel=kmperpixel,\n timestep=timestep,\n extrapolation_method=extrap_method,\n decomposition_method=decomp_method,\n bandpass_filter_method=bandpass_filter_method,\n nowcasting_method=nowcasting_method,\n noise_method=noise_method,\n noise_stddev_adj=noise_stddev_adj,\n ar_order=ar_order,\n velocity_perturbation_method=vel_pert_method,\n weights_method=weights_method,\n timestep_start_full_nwp_weight=timestep_start_full_nwp_weight,\n conditional=conditional,\n probmatching_method=probmatching_method,\n mask_method=mask_method,\n resample_distribution=resample_distribution,\n smooth_radar_mask_range=smooth_radar_mask_range,\n seed=seed,\n num_workers=num_workers,\n fft_method=fft_method,\n domain=domain,\n outdir_path_skill=outdir_path_skill,\n extrapolation_kwargs=extrap_kwargs,\n filter_kwargs=filter_kwargs,\n noise_kwargs=noise_kwargs,\n velocity_perturbation_kwargs=vel_pert_kwargs,\n climatology_kwargs=clim_kwargs,\n mask_kwargs=mask_kwargs,\n measure_time=measure_time,\n callback=callback,\n return_output=return_output,\n )\n\n \"\"\"\n With the new refactoring, the blending nowcaster is a class that can be used in multiple ways.\n This method is here to ensure that the class can be used in a similar way as the old function.\n The new refactoring provides more possibilities, eg. when doing multiple forecasts in a row, \n the config does not need to be provided each time\n \"\"\"\n # Create an instance of the new class with all the provided arguments\n blended_nowcaster = StepsBlendingNowcaster(\n precip,\n precip_nowcast,\n precip_models,\n velocity,\n velocity_models,\n timesteps,\n issuetime,\n blending_config,\n )\n\n forecast_steps_nowcast = blended_nowcaster.compute_forecast()\n return forecast_steps_nowcast\n\n\n# TODO: Where does this piece of code best fit: in utils or inside the class?\ndef calculate_ratios(correlations):\n \"\"\"Calculate explained variance ratios from correlation.\n\n Parameters\n ----------\n Array of shape [component, scale_level, ...]\n containing correlation (skills) for each component (NWP and nowcast),\n scale level, and optionally along [y, x] dimensions.\n\n Returns\n -------\n out : numpy array\n An array containing the ratios of explain variance for each\n component, scale level, ...\n \"\"\"\n # correlations: [component, scale, ...]\n square_corrs = np.square(correlations)\n # Calculate the ratio of the explained variance to the unexplained\n # variance of the nowcast and NWP model components\n out = square_corrs / (1 - square_corrs)\n # out: [component, scale, ...]\n return out\n\n\n# TODO: Where does this piece of code best fit: in utils or inside the class?\ndef calculate_weights_bps(correlations):\n \"\"\"Calculate BPS blending weights for STEPS blending from correlation.\n\n Parameters\n ----------\n correlations : array-like\n Array of shape [component, scale_level, ...]\n containing correlation (skills) for each component (NWP and nowcast),\n scale level, and optionally along [y, x] dimensions.\n\n Returns\n -------\n weights : array-like\n Array of shape [component+1, scale_level, ...]\n containing the weights to be used in STEPS blending for\n each original component plus an addtional noise component, scale level,\n and optionally along [y, x] dimensions.\n\n References\n ----------\n :cite:`BPS2006`\n\n Notes\n -----\n The weights in the BPS method can sum op to more than 1.0.\n \"\"\"\n # correlations: [component, scale, ...]\n # Check if the correlations are positive, otherwise rho = 10e-5\n correlations = np.where(correlations < 10e-5, 10e-5, correlations)\n\n # If we merge more than one component with the noise cascade, we follow\n # the weights impolementation in either :cite:`BPS2006` or :cite:`SPN2013`.\n if correlations.shape[0] > 1:\n # Calculate weights for each source\n ratios = calculate_ratios(correlations)\n # ratios: [component, scale, ...]\n total_ratios = np.sum(ratios, axis=0)\n # total_ratios: [scale, ...] - the denominator of eq. 11 & 12 in BPS2006\n weights = correlations * np.sqrt(ratios / total_ratios)\n # weights: [component, scale, ...]\n\n # Calculate the weight of the noise component.\n # Original BPS2006 method in the following two lines (eq. 13)\n total_square_weights = np.sum(np.square(weights), axis=0)\n noise_weight = np.sqrt(1.0 - total_square_weights)\n # Finally, add the noise_weights to the weights variable.\n weights = np.concatenate((weights, noise_weight[None, ...]), axis=0)\n\n # Otherwise, the weight equals the correlation on that scale level and\n # the noise component weight equals 1 - this weight. This only occurs for\n # the weights calculation outside the radar domain where in the case of 1\n # NWP model or ensemble member, no blending of multiple models has to take\n # place\n else:\n noise_weight = 1.0 - correlations\n weights = np.concatenate((correlations, noise_weight), axis=0)\n\n return weights\n\n\n# TODO: Where does this piece of code best fit: in utils or inside the class?\ndef calculate_weights_spn(correlations, covariance):\n \"\"\"Calculate SPN blending weights for STEPS blending from correlation.\n\n Parameters\n ----------\n correlations : array-like\n Array of shape [n_components]\n containing correlation (skills) for each component (NWP models and nowcast).\n covariance : array-like\n Array of shape [n_components, n_components] containing the covariance\n matrix of the models that will be blended. If cov is set to None and\n correlations only contains one model, the weight equals the correlation\n on that scale level and the noise component weight equals 1 - this weight.\n\n Returns\n -------\n weights : array-like\n Array of shape [component+1]\n containing the weights to be used in STEPS blending for each original\n component plus an addtional noise component.\n\n References\n ----------\n :cite:`SPN2013`\n \"\"\"\n # Check if the correlations are positive, otherwise rho = 10e-5\n correlations = np.where(correlations < 10e-5, 10e-5, correlations)\n\n if correlations.shape[0] > 1 and len(covariance) > 1:\n if isinstance(covariance, type(None)):\n raise ValueError(\"cov must contain a covariance matrix\")\n else:\n # Make a numpy array out of cov and get the inverse\n covariance = np.where(covariance == 0.0, 10e-5, covariance)\n # Make sure the determinant of the matrix is not zero, otherwise\n # subtract 10e-5 from the cross-correlations between the models\n if np.linalg.det(covariance) == 0.0:\n covariance = covariance - 10e-5\n # Ensure the correlation of the model with itself is always 1.0\n for i, _ in enumerate(covariance):\n covariance[i][i] = 1.0\n # Use a numpy array instead of a matrix\n cov_matrix = np.array(covariance)\n # Get the inverse of the matrix using scipy's inv function\n cov_matrix_inv = inv(cov_matrix)\n # The component weights are the dot product between cov_matrix_inv and cor_vec\n weights = np.dot(cov_matrix_inv, correlations)\n weights = np.nan_to_num(\n weights, copy=True, nan=10e-5, posinf=10e-5, neginf=10e-5\n )\n weights_dot_correlations = np.dot(weights, correlations)\n # If the dot product of the weights with the correlations is\n # larger than 1.0, we assign a weight of 0.0 to the noise (to make\n # it numerically stable)\n if weights_dot_correlations > 1.0:\n noise_weight = np.array([0])\n # Calculate the noise weight\n else:\n noise_weight = np.sqrt(1.0 - weights_dot_correlations)\n # Convert weights to a 1D array\n weights = np.array(weights).flatten()\n # Ensure noise_weight is a 1D array before concatenation\n noise_weight = np.array(noise_weight).flatten()\n # Finally, add the noise_weights to the weights variable.\n weights = np.concatenate((weights, noise_weight), axis=0)\n\n # Otherwise, the weight equals the correlation on that scale level and\n # the noise component weight equals 1 - this weight. This only occurs for\n # the weights calculation outside the radar domain where in the case of 1\n # NWP model or ensemble member, no blending of multiple models has to take\n # place\n else:\n noise_weight = 1.0 - correlations\n weights = np.concatenate((correlations, noise_weight), axis=0)\n\n # Make sure weights are always a real number\n weights = np.nan_to_num(weights, copy=True, nan=10e-5, posinf=10e-5, neginf=10e-5)\n\n return weights\n\n\n# TODO: Where does this piece of code best fit: in utils or inside the class?\ndef calculate_end_weights(\n previous_weights, timestep, n_timesteps, start_full_nwp_weight, model_only=False\n):\n \"\"\"Calculate the linear transition from the previous weights to the final weights\n (1.0 for NWP and 0.0 for the extrapolation and noise components). This method uses\n the BPS weights determination method to determine the corresponding noise.\n\n Parameters\n ----------\n previous_weights : array-like\n The weights from the previous timestep. This weight will be used to ensure\n a linear transition takes place from the last weights at the timestep of\n start_full_nwp_weight and the final weights (1.0 for NWP and 0.0 for\n the extrapolation and noise components).\n timestep : int\n The timestep or sub timestep for which the weight is calculated. Only\n used when start_full_nwp_weight is not None.\n n_timesteps: int\n The total number of forecast timesteps in the forecast.\n start_full_nwp_weight : int\n The timestep, which should be smaller than timesteps, at which a linear\n transition takes place from the calculated weights to full NWP weight\n (and zero extrapolation and noise weight) to ensure the blending\n procedure becomes equal to the NWP forecast(s) at the last timestep\n of the blending procedure. If not provided, the blending stick to the\n theoretical weights provided by the chosen weights_method for a given\n lead time and skill of each blending component.\n model_only : bool\n If set to True, the weights will only be determined for the model and\n noise components.\n\n Returns\n -------\n weights : array-like\n Array of shape [component+1, scale_level, ...]\n containing the weights to be used in STEPS blending for\n each original component plus an addtional noise component, scale level,\n and optionally along [y, x] dimensions.\n\n References\n ----------\n :cite:`BPS2006`\n\n Notes\n -----\n The weights in the BPS method can sum op to more than 1.0.\n \"\"\"\n weights = previous_weights[:-1, :].copy()\n if not model_only:\n if timestep > start_full_nwp_weight and timestep < n_timesteps:\n weights[0, :] = weights[0, :] - (\n (timestep - start_full_nwp_weight)\n / (n_timesteps - start_full_nwp_weight)\n * weights[0, :]\n )\n weights[1:, :] = (\n 1.0\n / weights[1:, :].shape[0]\n * (\n weights[1:, :]\n + (\n (timestep - start_full_nwp_weight)\n / (n_timesteps - start_full_nwp_weight)\n * (1.0 - weights[1:, :])\n )\n )\n )\n elif timestep > start_full_nwp_weight and timestep == n_timesteps:\n weights[0, :] = 0.0\n # If one model or model member is provided to blend together,\n # the weight equals 1.0, otherwise the sum of the weights\n # equals 1.0.\n weights[1:, :] = 1.0 / weights[1:, :].shape[0]\n\n else:\n if timestep > start_full_nwp_weight and timestep < n_timesteps:\n weights = (\n 1.0\n / weights.shape[0]\n * (\n weights\n + (\n (timestep - start_full_nwp_weight)\n / (n_timesteps - start_full_nwp_weight)\n * (1.0 - weights)\n )\n )\n )\n elif timestep > start_full_nwp_weight and timestep == n_timesteps:\n weights[:] = 1.0 / weights.shape[0]\n\n if weights.shape[0] > 1:\n # Calculate the weight of the noise component.\n # Original BPS2006 method in the following two lines (eq. 13)\n total_square_weights = np.sum(np.square(weights), axis=0)\n noise_weight = np.sqrt(1.0 - total_square_weights)\n # Finally, add the noise_weights to the weights variable.\n weights = np.concatenate((weights, noise_weight[None, ...]), axis=0)\n else:\n noise_weight = 1.0 - weights\n weights = np.concatenate((weights, noise_weight), axis=0)\n\n return weights\n\n\n# TODO: Where does this piece of code best fit: in utils or inside the class?\ndef blend_means_sigmas(means, sigmas, weights):\n \"\"\"Calculate the blended means and sigmas, the normalization parameters\n needed to recompose the cascade. This procedure uses the weights of the\n blending of the normalized cascades and follows eq. 32 and 33 in BPS2004.\n\n Parameters\n ----------\n means : array-like\n Array of shape [number_components, scale_level, ...]\n with the mean for each component (NWP, nowcasts, noise).\n sigmas : array-like\n Array of shape [number_components, scale_level, ...]\n with the standard deviation for each component.\n weights : array-like\n An array of shape [number_components + 1, scale_level, ...]\n containing the weights to be used in this routine\n for each component plus noise, scale level, and optionally [y, x]\n dimensions, obtained by calling either\n :py:func:`pysteps.blending.steps.calculate_weights_bps` or\n :py:func:`pysteps.blending.steps.calculate_weights_spn`.\n\n Returns\n -------\n combined_means : array-like\n An array of shape [scale_level, ...]\n containing per scale level (cascade) the weighted combination of\n means from multiple components (NWP, nowcasts and noise).\n combined_sigmas : array-like\n An array of shape [scale_level, ...]\n similar to combined_means, but containing the standard deviations.\n\n \"\"\"\n # Check if the dimensions are the same\n diff_dims = weights.ndim - means.ndim\n if diff_dims:\n for i in range(diff_dims):\n means = np.expand_dims(means, axis=means.ndim)\n diff_dims = weights.ndim - sigmas.ndim\n if diff_dims:\n for i in range(diff_dims):\n sigmas = np.expand_dims(sigmas, axis=sigmas.ndim)\n # Weight should have one component more (the noise component) than the\n # means and sigmas. Check this\n if (\n weights.shape[0] - means.shape[0] != 1\n or weights.shape[0] - sigmas.shape[0] != 1\n ):\n raise ValueError(\n \"The weights array does not have one (noise) component more than mu and sigma\"\n )\n else:\n # Throw away the last component, which is the noise component\n weights = weights[:-1]\n\n # Combine (blend) the means and sigmas\n combined_means = np.zeros(weights.shape[1])\n combined_sigmas = np.zeros(weights.shape[1])\n total_weight = np.sum((weights), axis=0)\n for i in range(weights.shape[0]):\n combined_means += (weights[i] / total_weight) * means[i]\n combined_sigmas += (weights[i] / total_weight) * sigmas[i]\n\n return combined_means, combined_sigmas\n" }, { "path": "pysteps/blending/utils.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.blending.utils\n======================\n\nModule with common utilities used by the blending methods.\n\n.. autosummary::\n :toctree: ../generated/\n\n stack_cascades\n blend_cascades\n recompose_cascade\n blend_optical_flows\n decompose_NWP\n compute_store_nwp_motion\n load_NWP\n compute_smooth_dilated_mask\n\"\"\"\n\nimport datetime\nimport warnings\nfrom pathlib import Path\n\nimport numpy as np\n\nfrom pysteps.cascade import get_method as cascade_get_method\nfrom pysteps.cascade.bandpass_filters import filter_gaussian\nfrom pysteps.exceptions import MissingOptionalDependency\nfrom pysteps.utils import get_method as utils_get_method\nfrom pysteps.utils.check_norain import check_norain as new_check_norain\n\ntry:\n import netCDF4\n\n NETCDF4_IMPORTED = True\nexcept ImportError:\n NETCDF4_IMPORTED = False\n\ntry:\n import cv2\n\n CV2_IMPORTED = True\nexcept ImportError:\n CV2_IMPORTED = False\n\n\ndef stack_cascades(R_d, donorm=True):\n \"\"\"Stack the given cascades into a larger array.\n\n Parameters\n ----------\n R_d : dict\n Dictionary containing a list of cascades obtained by calling a method\n implemented in pysteps.cascade.decomposition.\n donorm : bool\n If True, normalize the cascade levels before stacking.\n\n Returns\n -------\n out : tuple\n A three-element tuple containing a four-dimensional array of stacked\n cascade levels and arrays of mean values and standard deviations for each\n cascade level.\n \"\"\"\n R_c = []\n mu_c = []\n sigma_c = []\n\n for cascade in R_d:\n R_ = []\n R_i = cascade[\"cascade_levels\"]\n n_levels = R_i.shape[0]\n mu_ = np.asarray(cascade[\"means\"])\n sigma_ = np.asarray(cascade[\"stds\"])\n if donorm:\n for j in range(n_levels):\n R__ = (R_i[j, :, :] - mu_[j]) / sigma_[j]\n R_.append(R__)\n else:\n R_ = R_i\n R_c.append(np.stack(R_))\n mu_c.append(mu_)\n sigma_c.append(sigma_)\n return np.stack(R_c), np.stack(mu_c), np.stack(sigma_c)\n\n\ndef blend_cascades(cascades_norm, weights):\n \"\"\"Calculate blended normalized cascades using STEPS weights following eq.\n 10 in :cite:`BPS2006`.\n\n Parameters\n ----------\n cascades_norm : array-like\n Array of shape [number_components + 1, scale_level, ...]\n with the cascade for each component (NWP, nowcasts, noise) and scale level,\n obtained by calling a method implemented in pysteps.blending.utils.stack_cascades\n\n weights : array-like\n An array of shape [number_components + 1, scale_level, ...]\n containing the weights to be used in this routine\n for each component plus noise, scale level, and optionally [y, x]\n dimensions, obtained by calling a method implemented in\n pysteps.blending.steps.calculate_weights\n\n Returns\n -------\n combined_cascade : array-like\n An array of shape [scale_level, y, x]\n containing per scale level (cascade) the weighted combination of\n cascades from multiple components (NWP, nowcasts and noise) to be used\n in STEPS blending.\n \"\"\"\n # check inputs\n if isinstance(cascades_norm, (list, tuple)):\n cascades_norm = np.stack(cascades_norm)\n\n if isinstance(weights, (list, tuple)):\n weights = np.asarray(weights)\n\n # check weights dimensions match number of sources\n num_sources = cascades_norm.shape[0]\n num_sources_klevels = cascades_norm.shape[1]\n num_weights = weights.shape[0]\n num_weights_klevels = weights.shape[1]\n\n if num_weights != num_sources:\n raise ValueError(\n \"dimension mismatch between cascades and weights.\\n\"\n \"weights dimension must match the number of components in cascades.\\n\"\n f\"number of models={num_sources}, number of weights={num_weights}\"\n )\n if num_weights_klevels != num_sources_klevels:\n raise ValueError(\n \"dimension mismatch between cascades and weights.\\n\"\n \"weights cascade levels dimension must match the number of cascades in cascades_norm.\\n\"\n f\"number of cascade levels={num_sources_klevels}, number of weights={num_weights_klevels}\"\n )\n\n # cascade_norm component, scales, y, x\n # weights component, scales, ....\n # Reshape weights to make the calculation possible with numpy\n all_c_wn = weights.reshape(num_weights, num_weights_klevels, 1, 1) * cascades_norm\n combined_cascade = np.sum(all_c_wn, axis=0)\n # combined_cascade [scale, ...]\n return combined_cascade\n\n\ndef recompose_cascade(combined_cascade, combined_mean, combined_sigma):\n \"\"\"Recompose the cascades into a transformed rain rate field.\n\n\n Parameters\n ----------\n combined_cascade : array-like\n An array of shape [scale_level, y, x]\n containing per scale level (cascade) the weighted combination of\n cascades from multiple components (NWP, nowcasts and noise) to be used\n in STEPS blending.\n combined_mean : array-like\n An array of shape [scale_level, ...]\n similar to combined_cascade, but containing the normalization parameter\n mean.\n combined_sigma : array-like\n An array of shape [scale_level, ...]\n similar to combined_cascade, but containing the normalization parameter\n standard deviation.\n\n Returns\n -------\n out: array-like\n A two-dimensional array containing the recomposed cascade.\n\n \"\"\"\n # Renormalize with the blended sigma and mean values\n renorm = (\n combined_cascade * combined_sigma.reshape(combined_cascade.shape[0], 1, 1)\n ) + combined_mean.reshape(combined_mean.shape[0], 1, 1)\n # print(renorm.shape)\n out = np.sum(renorm, axis=0)\n # print(out.shape)\n return out\n\n\ndef blend_optical_flows(flows, weights):\n \"\"\"Combine advection fields using given weights. Following :cite:`BPS2006`\n the second level of the cascade is used for the weights\n\n Parameters\n ----------\n flows : array-like\n A stack of multiple advenction fields having shape\n (S, 2, m, n), where flows[N, :, :, :] contains the motion vectors\n for source N.\n Advection fields for each source can be obtanined by\n calling any of the methods implemented in\n pysteps.motion and then stack all together\n weights : array-like\n An array of shape [number_sources]\n containing the weights to be used to combine\n the advection fields of each source.\n weights are modified to make their sum equal to one.\n Returns\n -------\n out: ndarray\n Return the blended advection field having shape\n (2, m, n), where out[0, :, :] contains the x-components of\n the blended motion vectors and out[1, :, :] contains the y-components.\n The velocities are in units of pixels / timestep.\n \"\"\"\n\n # check inputs\n if isinstance(flows, (list, tuple)):\n flows = np.stack(flows)\n\n if isinstance(weights, (list, tuple)):\n weights = np.asarray(weights)\n\n # check weights dimensions match number of sources\n num_sources = flows.shape[0]\n num_weights = weights.shape[0]\n\n if num_weights != num_sources:\n raise ValueError(\n \"dimension mismatch between flows and weights.\\n\"\n \"weights dimension must match the number of flows.\\n\"\n f\"number of flows={num_sources}, number of weights={num_weights}\"\n )\n # normalize weigths\n weights = weights / np.sum(weights)\n\n # flows dimension sources, 2, m, n\n # weights dimension sources\n # move source axis to last to allow broadcasting\n # TODO: Check if broadcasting has worked well\n all_c_wn = weights * np.moveaxis(flows, 0, -1)\n # sum uses last axis\n combined_flows = np.sum(all_c_wn, axis=-1)\n # combined_flows [2, m, n]\n return combined_flows\n\n\ndef decompose_NWP(\n R_NWP,\n NWP_model,\n analysis_time,\n timestep,\n valid_times,\n output_path,\n num_cascade_levels=8,\n num_workers=1,\n decomp_method=\"fft\",\n fft_method=\"numpy\",\n domain=\"spatial\",\n normalize=True,\n compute_stats=True,\n compact_output=True,\n):\n \"\"\"Decomposes the NWP forecast data into cascades and saves it in\n a netCDF file\n\n Parameters\n ----------\n R_NWP: array-like\n Array of dimension (n_timesteps, x, y) containing the precipitation forecast\n from some NWP model.\n NWP_model: str\n The name of the NWP model\n analysis_time: numpy.datetime64\n The analysis time of the NWP forecast. The analysis time is assumed to be a\n numpy.datetime64 type as imported by the pysteps importer\n timestep: int\n Timestep in minutes between subsequent NWP forecast fields\n valid_times: array_like\n Array containing the valid times of the NWP forecast fields. The times are\n assumed to be numpy.datetime64 types as imported by the pysteps importer.\n output_path: str\n The location where to save the file with the NWP cascade. Defaults to the\n path_workdir specified in the rcparams file.\n num_cascade_levels: int, optional\n The number of frequency bands to use. Must be greater than 2. Defaults to 8.\n num_workers: int, optional\n The number of workers to use for parallel computation. Applicable if dask\n is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it\n is advisable to disable OpenMP by setting the environment variable\n OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous\n threads.\n\n Other Parameters\n ----------------\n decomp_method: str, optional\n A string defining the decomposition method to use. Defaults to \"fft\".\n fft_method: str or tuple, optional\n A string or a (function,kwargs) tuple defining the FFT method to use\n (see :py:func:`pysteps.utils.interface.get_method`).\n Defaults to \"numpy\". This option is not used if input_domain and\n output_domain are both set to \"spectral\".\n domain: {\"spatial\", \"spectral\"}, optional\n If \"spatial\", the output cascade levels are transformed back to the\n spatial domain by using the inverse FFT. If \"spectral\", the cascade is\n kept in the spectral domain. Defaults to \"spatial\".\n normalize: bool, optional\n If True, normalize the cascade levels to zero mean and unit variance.\n Requires that compute_stats is True. Implies that compute_stats is True.\n Defaults to False.\n compute_stats: bool, optional\n If True, the output dictionary contains the keys \"means\" and \"stds\"\n for the mean and standard deviation of each output cascade level.\n Defaults to False.\n compact_output: bool, optional\n Applicable if output_domain is \"spectral\". If set to True, only the\n parts of the Fourier spectrum with non-negligible filter weights are\n stored. Defaults to False.\n\n\n Returns\n -------\n None\n \"\"\"\n\n if not NETCDF4_IMPORTED:\n raise MissingOptionalDependency(\n \"netCDF4 package is required to save the decomposed NWP data, \"\n \"but it is not installed\"\n )\n\n # Make a NetCDF file\n output_date = f\"{analysis_time.astype('datetime64[us]').astype(datetime.datetime):%Y%m%d%H%M%S}\"\n outfn = Path(output_path) / f\"cascade_{NWP_model}_{output_date}.nc\"\n ncf = netCDF4.Dataset(outfn, \"w\", format=\"NETCDF4\")\n\n # Express times relative to the zero time\n zero_time = np.datetime64(\"1970-01-01T00:00:00\", \"ns\")\n valid_times = np.array(valid_times) - zero_time\n analysis_time = analysis_time - zero_time\n\n # Set attributes of decomposition method\n ncf.domain = domain\n ncf.normalized = int(normalize)\n ncf.compact_output = int(compact_output)\n ncf.analysis_time = int(analysis_time)\n ncf.timestep = int(timestep)\n\n # Create dimensions\n ncf.createDimension(\"time\", R_NWP.shape[0])\n ncf.createDimension(\"cascade_levels\", num_cascade_levels)\n ncf.createDimension(\"x\", R_NWP.shape[2])\n ncf.createDimension(\"y\", R_NWP.shape[1])\n\n # Create variables (decomposed cascade, means and standard deviations)\n R_d = ncf.createVariable(\n \"pr_decomposed\",\n np.float32,\n (\"time\", \"cascade_levels\", \"y\", \"x\"),\n zlib=True,\n complevel=4,\n )\n means = ncf.createVariable(\"means\", np.float64, (\"time\", \"cascade_levels\"))\n stds = ncf.createVariable(\"stds\", np.float64, (\"time\", \"cascade_levels\"))\n v_times = ncf.createVariable(\"valid_times\", np.float64, (\"time\",))\n v_times.units = \"nanoseconds since 1970-01-01 00:00:00\"\n\n # The valid times are saved as an array of floats, because netCDF files can't handle datetime types\n v_times[:] = np.array([np.float64(valid_times[i]) for i in range(len(valid_times))])\n\n # Decompose the NWP data\n filter_g = filter_gaussian(R_NWP.shape[1:], num_cascade_levels)\n fft = utils_get_method(fft_method, shape=R_NWP.shape[1:], n_threads=num_workers)\n decomp_method, _ = cascade_get_method(decomp_method)\n\n for i in range(R_NWP.shape[0]):\n R_ = decomp_method(\n field=R_NWP[i, :, :],\n bp_filter=filter_g,\n fft_method=fft,\n input_domain=domain,\n output_domain=domain,\n normalize=normalize,\n compute_stats=compute_stats,\n compact_output=compact_output,\n )\n\n # Save data to netCDF file\n # print(R_[\"cascade_levels\"])\n R_d[i, :, :, :] = R_[\"cascade_levels\"]\n means[i, :] = R_[\"means\"]\n stds[i, :] = R_[\"stds\"]\n\n # Close the file\n ncf.close()\n\n\ndef compute_store_nwp_motion(\n precip_nwp,\n oflow_method,\n analysis_time,\n nwp_model,\n output_path,\n):\n \"\"\"Computes, per forecast lead time, the velocity field of an NWP model field.\n\n Parameters\n ----------\n precip_nwp: array-like\n Array of dimension (n_timesteps, x, y) containing the precipitation forecast\n from some NWP model.\n oflow_method: {'constant', 'darts', 'lucaskanade', 'proesmans', 'vet'}, optional\n An optical flow method from pysteps.motion.get_method.\n analysis_time: numpy.datetime64\n The analysis time of the NWP forecast. The analysis time is assumed to be a\n numpy.datetime64 type as imported by the pysteps importer.\n nwp_model: str\n The name of the NWP model.\n output_path: str, optional\n The location where to save the file with the NWP velocity fields. Defaults\n to the path_workdir specified in the rcparams file.\n\n Returns\n -------\n Nothing\n \"\"\"\n\n # Set the output file\n output_date = f\"{analysis_time.astype('datetime64[us]').astype(datetime.datetime):%Y%m%d%H%M%S}\"\n outfn = Path(output_path) / f\"motion_{nwp_model}_{output_date}.npy\"\n\n # Get the velocity field per time step\n v_nwp = np.zeros((precip_nwp.shape[0], 2, precip_nwp.shape[1], precip_nwp.shape[2]))\n # Loop through the timesteps. We need two images to construct a motion\n # field, so we can start from timestep 1.\n for t in range(1, precip_nwp.shape[0]):\n v_nwp[t] = oflow_method(precip_nwp[t - 1 : t + 1, :, :])\n\n # Make timestep 0 the same as timestep 1.\n v_nwp[0] = v_nwp[1]\n\n assert v_nwp.ndim == 4, \"v_nwp must be a four-dimensional array\"\n\n # Save it as a numpy array\n np.save(outfn, v_nwp)\n\n\ndef load_NWP(input_nc_path_decomp, input_path_velocities, start_time, n_timesteps):\n \"\"\"Loads the decomposed NWP and velocity data from the netCDF files\n\n Parameters\n ----------\n input_nc_path_decomp: str\n Path to the saved netCDF file containing the decomposed NWP data.\n input_path_velocities: str\n Path to the saved numpy binary file containing the estimated velocity\n fields from the NWP data.\n start_time: numpy.datetime64\n The start time of the nowcasting. Assumed to be a numpy.datetime64 type\n n_timesteps: int\n Number of time steps to forecast\n\n Returns\n -------\n R_d: list\n A list of dictionaries with each element in the list corresponding to\n a different time step. Each dictionary has the same structure as the\n output of the decomposition function\n uv: array-like\n Array of shape (timestep,2,m,n) containing the x- and y-components\n of the advection field for the (NWP) model field per forecast lead time.\n \"\"\"\n\n if not NETCDF4_IMPORTED:\n raise MissingOptionalDependency(\n \"netCDF4 package is required to load the decomposed NWP data, \"\n \"but it is not installed\"\n )\n\n # Open the file\n ncf_decomp = netCDF4.Dataset(input_nc_path_decomp, \"r\", format=\"NETCDF4\")\n velocities = np.load(input_path_velocities)\n\n decomp_dict = {\n \"domain\": ncf_decomp.domain,\n \"normalized\": bool(ncf_decomp.normalized),\n \"compact_output\": bool(ncf_decomp.compact_output),\n }\n\n # Convert the start time and the timestep to datetime64 and timedelta64 type\n zero_time = np.datetime64(\"1970-01-01T00:00:00\", \"ns\")\n analysis_time = np.timedelta64(int(ncf_decomp.analysis_time), \"ns\") + zero_time\n\n timestep = ncf_decomp.timestep\n timestep = np.timedelta64(timestep, \"m\")\n\n valid_times = ncf_decomp.variables[\"valid_times\"][:]\n valid_times = np.array(\n [np.timedelta64(int(valid_times[i]), \"ns\") for i in range(len(valid_times))]\n )\n valid_times = valid_times + zero_time\n\n # Find the indices corresponding with the required start and end time\n start_i = (start_time - analysis_time) // timestep\n assert analysis_time + start_i * timestep == start_time\n end_i = start_i + n_timesteps + 1\n\n # Check if the requested end time (the forecast horizon) is in the stored data.\n # If not, raise an error\n if end_i > ncf_decomp.variables[\"pr_decomposed\"].shape[0]:\n raise IndexError(\n \"The requested forecast horizon is outside the stored NWP forecast horizon. Either request a shorter forecast horizon or store a longer NWP forecast horizon\"\n )\n\n # Add the valid times to the output\n decomp_dict[\"valid_times\"] = valid_times[start_i:end_i]\n\n # Slice the velocity fields with the start and end indices\n uv = velocities[start_i:end_i, :, :, :]\n\n # Initialise the list of dictionaries which will serve as the output (cf: the STEPS function)\n R_d = list()\n\n pr_decomposed = ncf_decomp.variables[\"pr_decomposed\"][start_i:end_i, :, :, :]\n means = ncf_decomp.variables[\"means\"][start_i:end_i, :]\n stds = ncf_decomp.variables[\"stds\"][start_i:end_i, :]\n\n for i in range(n_timesteps + 1):\n decomp_dict[\"cascade_levels\"] = np.ma.filled(\n pr_decomposed[i], fill_value=np.nan\n )\n decomp_dict[\"means\"] = np.ma.filled(means[i], fill_value=np.nan)\n decomp_dict[\"stds\"] = np.ma.filled(stds[i], fill_value=np.nan)\n\n R_d.append(decomp_dict.copy())\n\n ncf_decomp.close()\n return R_d, uv\n\n\ndef check_norain(precip_arr, precip_thr=None, norain_thr=0.0):\n \"\"\"\n DEPRECATED use :py:mod:`pysteps.utils.check_norain.check_norain` in stead\n Parameters\n ----------\n precip_arr: array-like\n Array containing the input precipitation field\n precip_thr: float, optional\n Specifies the threshold value for minimum observable precipitation intensity. If None, the\n minimum value over the domain is taken.\n norain_thr: float, optional\n Specifies the threshold value for the fraction of rainy pixels in precip_arr below which we consider there to be\n no rain. Standard set to 0.0\n Returns\n -------\n norain: bool\n Returns whether the fraction of rainy pixels is below the norain_thr threshold.\n\n \"\"\"\n warnings.warn(\n \"pysteps.blending.utils.check_norain has been deprecated, use pysteps.utils.check_norain.check_norain instead\"\n )\n return new_check_norain(precip_arr, precip_thr, norain_thr, None)\n\n\ndef compute_smooth_dilated_mask(\n original_mask,\n max_padding_size_in_px=0,\n gaussian_kernel_size=9,\n inverted=False,\n non_linear_growth_kernel_sizes=False,\n):\n \"\"\"\n Compute a smooth dilated mask using Gaussian blur and dilation with varying kernel sizes.\n\n Parameters\n ----------\n original_mask : array_like\n Two-dimensional boolean array containing the input mask.\n max_padding_size_in_px : int\n The maximum size of the padding in pixels. Default is 100.\n gaussian_kernel_size : int, optional\n Size of the Gaussian kernel to use for blurring, this should be an uneven number. This option ensures\n that the nan-fields are large enough to start the smoothing. Without it, the method will also be applied\n to local nan-values in the radar domain. Default is 9, which is generally a recommended number to work\n with.\n inverted : bool, optional\n Typically, the smoothed mask works from the outside of the radar domain inward, using the\n max_padding_size_in_px. If set to True, it works from the edge of the radar domain outward\n (generally not recommended). Default is False.\n non_linear_growth_kernel_sizes : bool, optional\n If True, use non-linear growth for kernel sizes. Default is False.\n\n Returns\n -------\n final_mask : array_like\n The smooth dilated mask normalized to the range [0,1].\n \"\"\"\n if not CV2_IMPORTED:\n raise MissingOptionalDependency(\n \"CV2 package is required to transform the mask into a smoot mask.\"\n \" Please install it using `pip install opencv-python`.\"\n )\n\n if max_padding_size_in_px < 0:\n raise ValueError(\"max_padding_size_in_px must be greater than or equal to 0.\")\n\n # Check if gaussian_kernel_size is an uneven number\n assert gaussian_kernel_size % 2\n\n # Convert the original mask to uint8 numpy array and invert if needed\n array_2d = np.array(original_mask, dtype=np.uint8)\n if inverted:\n array_2d = np.bitwise_not(array_2d)\n\n # Rescale the 2D array values to 0-255 (black or white)\n rescaled_array = array_2d * 255\n\n # Apply Gaussian blur to the rescaled array\n blurred_image = cv2.GaussianBlur(\n rescaled_array, (gaussian_kernel_size, gaussian_kernel_size), 0\n )\n\n # Apply binary threshold to negate the blurring effect\n _, binary_image = cv2.threshold(blurred_image, 128, 255, cv2.THRESH_BINARY)\n\n # Define kernel sizes\n if non_linear_growth_kernel_sizes:\n lin_space = np.linspace(0, np.sqrt(max_padding_size_in_px), 10)\n non_lin_space = np.power(lin_space, 2)\n kernel_sizes = list(set(non_lin_space.astype(np.uint8)))\n else:\n kernel_sizes = np.linspace(0, max_padding_size_in_px, 10, dtype=np.uint8)\n\n # Process each kernel size\n final_mask = np.zeros_like(binary_image, dtype=np.float64)\n for kernel_size in kernel_sizes:\n if kernel_size == 0:\n dilated_image = binary_image\n else:\n kernel = cv2.getStructuringElement(\n cv2.MORPH_ELLIPSE, (kernel_size, kernel_size)\n )\n dilated_image = cv2.dilate(binary_image, kernel)\n\n # Convert the dilated image to a binary array\n _, binary_array = cv2.threshold(dilated_image, 128, 1, cv2.THRESH_BINARY)\n final_mask += binary_array\n\n final_mask = final_mask / final_mask.max()\n\n return final_mask\n" }, { "path": "pysteps/cascade/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nMethods for constructing bandpass filters and decomposing 2d precipitation\nfields into different spatial scales.\n\"\"\"\n\nfrom .interface import get_method\n" }, { "path": "pysteps/cascade/bandpass_filters.py", "content": "\"\"\"\npysteps.cascade.bandpass_filters\n================================\n\nBandpass filters for separating different spatial scales from two-dimensional\nimages in the frequency domain.\n\nThe methods in this module implement the following interface::\n\n filter_xxx(shape, n, optional arguments)\n\nwhere shape is the shape of the input field, respectively, and n is the number\nof frequency bands to use.\n\nThe output of each filter function is a dictionary containing the following\nkey-value pairs:\n\n.. tabularcolumns:: |p{1.8cm}|L|\n\n+-----------------+-----------------------------------------------------------+\n| Key | Value |\n+=================+===========================================================+\n| weights_1d | 2d array of shape (n, r) containing 1d filter weights for |\n| | each frequency band k=1,2,...,n |\n+-----------------+-----------------------------------------------------------+\n| weights_2d | 3d array of shape (n, M, int(N/2)+1) containing the 2d |\n| | filter weights for each frequency band k=1,2,...,n |\n+-----------------+-----------------------------------------------------------+\n| central_freqs | 1d array of shape n containing the central frequencies of |\n| | the filters |\n+-----------------+-----------------------------------------------------------+\n| shape | the shape of the input field in the spatial domain |\n+-----------------+-----------------------------------------------------------+\n\nwhere r = int(max(N, M)/2)+1\n\nBy default, the filter weights are normalized so that for any Fourier\nwavenumber they sum to one.\n\nAvailable filters\n-----------------\n\n.. autosummary::\n :toctree: ../generated/\n\n filter_uniform\n filter_gaussian\n\"\"\"\n\nimport numpy as np\n\n\ndef filter_uniform(shape, n):\n \"\"\"\n A dummy filter with one frequency band covering the whole domain. The\n weights are set to one.\n\n Parameters\n ----------\n shape: int or tuple\n The dimensions (height, width) of the input field. If shape is an int,\n the domain is assumed to have square shape.\n n: int\n Not used. Needed for compatibility with the filter interface.\n\n Returns\n -------\n out: dict\n A dictionary containing the filter.\n \"\"\"\n del n # Unused\n\n out = {}\n\n try:\n height, width = shape\n except TypeError:\n height, width = (shape, shape)\n\n r_max = int(max(width, height) / 2) + 1\n\n out[\"weights_1d\"] = np.ones((1, r_max))\n out[\"weights_2d\"] = np.ones((1, height, int(width / 2) + 1))\n out[\"central_freqs\"] = None\n out[\"central_wavenumbers\"] = None\n out[\"shape\"] = shape\n\n return out\n\n\ndef filter_gaussian(\n shape,\n n,\n gauss_scale=0.5,\n d=1.0,\n normalize=True,\n return_weight_funcs=False,\n include_mean=True,\n):\n \"\"\"\n Implements a set of Gaussian bandpass filters in logarithmic frequency\n scale.\n\n Parameters\n ----------\n shape: int or tuple\n The dimensions (height, width) of the input field. If shape is an int,\n the domain is assumed to have square shape.\n n: int\n The number of frequency bands to use. Must be greater than 2.\n gauss_scale: float\n Optional scaling parameter. Proportional to the standard deviation of\n the Gaussian weight functions.\n d: scalar, optional\n Sample spacing (inverse of the sampling rate). Defaults to 1.\n normalize: bool\n If True, normalize the weights so that for any given wavenumber\n they sum to one.\n return_weight_funcs: bool\n If True, add callable weight functions to the output dictionary with\n the key 'weight_funcs'.\n include_mean: bool\n If True, include the first Fourier wavenumber (corresponding to the\n field mean) to the first filter.\n\n Returns\n -------\n out: dict\n A dictionary containing the bandpass filters corresponding to the\n specified frequency bands.\n\n References\n ----------\n :cite:`PCH2018`\n\n \"\"\"\n if n < 3:\n raise ValueError(\"n must be greater than 2\")\n\n try:\n height, width = shape\n except TypeError:\n height, width = (shape, shape)\n\n max_length = max(width, height)\n\n rx = np.s_[: int(width / 2) + 1]\n\n if (height % 2) == 1:\n ry = np.s_[-int(height / 2) : int(height / 2) + 1]\n else:\n ry = np.s_[-int(height / 2) : int(height / 2)]\n\n y_grid, x_grid = np.ogrid[ry, rx]\n dy = int(height / 2) if height % 2 == 0 else int(height / 2) + 1\n\n r_2d = np.roll(np.sqrt(x_grid * x_grid + y_grid * y_grid), dy, axis=0)\n\n r_max = int(max_length / 2) + 1\n r_1d = np.arange(r_max)\n\n wfs, central_wavenumbers = _gaussweights_1d(\n max_length,\n n,\n gauss_scale=gauss_scale,\n )\n\n weights_1d = np.empty((n, r_max))\n weights_2d = np.empty((n, height, int(width / 2) + 1))\n\n for i, wf in enumerate(wfs):\n weights_1d[i, :] = wf(r_1d)\n weights_2d[i, :, :] = wf(r_2d)\n\n if normalize:\n weights_1d_sum = np.sum(weights_1d, axis=0)\n weights_2d_sum = np.sum(weights_2d, axis=0)\n for k in range(weights_2d.shape[0]):\n weights_1d[k, :] /= weights_1d_sum\n weights_2d[k, :, :] /= weights_2d_sum\n\n for i in range(len(wfs)):\n if i == 0 and include_mean:\n weights_1d[i, 0] = 1.0\n weights_2d[i, 0, 0] = 1.0\n else:\n weights_1d[i, 0] = 0.0\n weights_2d[i, 0, 0] = 0.0\n\n out = {\"weights_1d\": weights_1d, \"weights_2d\": weights_2d}\n out[\"shape\"] = shape\n\n central_wavenumbers = np.array(central_wavenumbers)\n out[\"central_wavenumbers\"] = central_wavenumbers\n\n # Compute frequencies\n central_freqs = 1.0 * central_wavenumbers / max_length\n central_freqs[0] = 1.0 / max_length\n central_freqs[-1] = 0.5 # Nyquist freq\n central_freqs = 1.0 * d * central_freqs\n out[\"central_freqs\"] = central_freqs\n\n if return_weight_funcs:\n out[\"weight_funcs\"] = wfs\n\n return out\n\n\ndef _gaussweights_1d(l, n, gauss_scale=0.5):\n q = pow(0.5 * l, 1.0 / n)\n r = [(pow(q, k - 1), pow(q, k)) for k in range(1, n + 1)]\n r = [0.5 * (r_[0] + r_[1]) for r_ in r]\n\n def log_e(x):\n if len(np.shape(x)) > 0:\n res = np.empty(x.shape)\n res[x == 0] = 0.0\n res[x > 0] = np.log(x[x > 0]) / np.log(q)\n else:\n if x == 0.0:\n res = 0.0\n else:\n res = np.log(x) / np.log(q)\n\n return res\n\n class GaussFunc:\n def __init__(self, c, s):\n self.c = c\n self.s = s\n\n def __call__(self, x):\n x = log_e(x) - self.c\n return np.exp(-(x**2.0) / (2.0 * self.s**2.0))\n\n weight_funcs = []\n central_wavenumbers = []\n\n for i, ri in enumerate(r):\n rc = log_e(ri)\n weight_funcs.append(GaussFunc(rc, gauss_scale))\n central_wavenumbers.append(ri)\n\n return weight_funcs, central_wavenumbers\n" }, { "path": "pysteps/cascade/decomposition.py", "content": "\"\"\"\npysteps.cascade.decomposition\n=============================\n\nMethods for decomposing two-dimensional fields into multiple spatial scales and\nrecomposing the individual scales to obtain the original field.\n\nThe methods in this module implement the following interface::\n\n decomposition_xxx(field, bp_filter, **kwargs)\n recompose_xxx(decomp, **kwargs)\n\nwhere field is the input field and bp_filter is a dictionary returned by a\nfilter method implemented in :py:mod:`pysteps.cascade.bandpass_filters`. The\ndecomp argument is a decomposition obtained by calling decomposition_xxx.\nOptional parameters can be passed in the keyword arguments. The output of each\nmethod is a dictionary with the following key-value pairs:\n\n+-------------------+----------------------------------------------------------+\n| Key | Value |\n+===================+==========================================================+\n| cascade_levels | three-dimensional array of shape (k,m,n), where k is the |\n| | number of cascade levels and the input fields have shape |\n| | (m,n) |\n| | if domain is \"spectral\" and compact output is requested |\n| | (see the table below), cascade_levels contains a list of |\n| | one-dimensional arrays |\n+-------------------+----------------------------------------------------------+\n| domain | domain of the cascade decomposition: \"spatial\" or |\n| | \"spectral\" |\n+-------------------+----------------------------------------------------------+\n| normalized | are the cascade levels normalized: True or False |\n+-------------------+----------------------------------------------------------+\n\nThe following key-value pairs are optional. They are included in the output if\n``kwargs`` contains the \"compute_stats\" key with value set to True:\n\n+-------------------+----------------------------------------------------------+\n| Key | Value |\n+===================+==========================================================+\n| means | list of mean values for each cascade level |\n+-------------------+----------------------------------------------------------+\n| stds | list of standard deviations for each cascade level |\n+-------------------+----------------------------------------------------------+\n\nThe following key-value pairs are included in the output if ``kwargs`` contains\nthe key \"output_domain\" with value set to \"spectral\":\n\n+-------------------+----------------------------------------------------------+\n| Key | Value |\n+===================+==========================================================+\n| compact_output | True or False. If set to True, only the parts of the |\n| | Fourier spectrum with non-negligible filter weights are |\n| | stored. |\n+-------------------+----------------------------------------------------------+\n| weight_masks | Applicable if compact_output is True. Contains a list of |\n| | masks, where a True value indicates that the |\n| | corresponding Fourier wavenumber is included in the |\n| | decomposition |\n+-------------------+----------------------------------------------------------+\n\n\nAvailable methods\n-----------------\n\n.. autosummary::\n :toctree: ../generated/\n\n decomposition_fft\n recompose_fft\n\"\"\"\n\nimport numpy as np\nfrom pysteps import utils\n\n\ndef decomposition_fft(field, bp_filter, **kwargs):\n \"\"\"\n Decompose a two-dimensional input field into multiple spatial scales by\n using the Fast Fourier Transform (FFT) and a set of bandpass filters.\n\n Parameters\n ----------\n field: array_like\n Two-dimensional array containing the input field. All values are\n required to be finite.\n bp_filter: dict\n A filter returned by a method implemented in\n :py:mod:`pysteps.cascade.bandpass_filters`.\n\n Other Parameters\n ----------------\n fft_method: str or tuple\n A string or a (function,kwargs) tuple defining the FFT method to use\n (see :py:func:`pysteps.utils.interface.get_method`).\n Defaults to \"numpy\". This option is not used if input_domain and\n output_domain are both set to \"spectral\".\n normalize: bool\n If True, normalize the cascade levels to zero mean and unit variance.\n Requires that compute_stats is True. Implies that compute_stats is True.\n Defaults to False.\n mask: array_like\n Optional mask to use for computing the statistics for the cascade\n levels. Pixels with mask==False are excluded from the computations.\n This option is not used if output domain is \"spectral\".\n input_domain: {\"spatial\", \"spectral\"}\n The domain of the input field. If \"spectral\", the input is assumed to\n be in the spectral domain. Defaults to \"spatial\".\n output_domain: {\"spatial\", \"spectral\"}\n If \"spatial\", the output cascade levels are transformed back to the\n spatial domain by using the inverse FFT. If \"spectral\", the cascade is\n kept in the spectral domain. Defaults to \"spatial\".\n compute_stats: bool\n If True, the output dictionary contains the keys \"means\" and \"stds\"\n for the mean and standard deviation of each output cascade level.\n Defaults to False.\n compact_output: bool\n Applicable if output_domain is \"spectral\". If set to True, only the\n parts of the Fourier spectrum with non-negligible filter weights are\n stored. Defaults to False.\n subtract_mean: bool\n If set to True, subtract the mean value before the decomposition and\n store it to the output dictionary. Applicable if input_domain is\n \"spatial\". Defaults to False.\n\n Returns\n -------\n out: ndarray\n A dictionary described in the module documentation.\n The number of cascade levels is determined from the filter\n (see :py:mod:`pysteps.cascade.bandpass_filters`).\n\n \"\"\"\n fft = kwargs.get(\"fft_method\", \"numpy\")\n if isinstance(fft, str):\n fft = utils.get_method(fft, shape=field.shape)\n normalize = kwargs.get(\"normalize\", False)\n mask = kwargs.get(\"mask\", None)\n input_domain = kwargs.get(\"input_domain\", \"spatial\")\n output_domain = kwargs.get(\"output_domain\", \"spatial\")\n compute_stats = kwargs.get(\"compute_stats\", True)\n compact_output = kwargs.get(\"compact_output\", False)\n subtract_mean = kwargs.get(\"subtract_mean\", False)\n\n if normalize and not compute_stats:\n compute_stats = True\n\n if len(field.shape) != 2:\n raise ValueError(\"The input is not two-dimensional array\")\n\n if mask is not None and mask.shape != field.shape:\n raise ValueError(\n \"Dimension mismatch between field and mask:\"\n + \"field.shape=\"\n + str(field.shape)\n + \",mask.shape\"\n + str(mask.shape)\n )\n\n if field.shape[0] != bp_filter[\"weights_2d\"].shape[1]:\n raise ValueError(\n \"dimension mismatch between field and bp_filter: \"\n + \"field.shape[0]=%d , \" % field.shape[0]\n + \"bp_filter['weights_2d'].shape[1]\"\n \"=%d\" % bp_filter[\"weights_2d\"].shape[1]\n )\n\n if (\n input_domain == \"spatial\"\n and int(field.shape[1] / 2) + 1 != bp_filter[\"weights_2d\"].shape[2]\n ):\n raise ValueError(\n \"Dimension mismatch between field and bp_filter: \"\n \"int(field.shape[1]/2)+1=%d , \" % (int(field.shape[1] / 2) + 1)\n + \"bp_filter['weights_2d'].shape[2]\"\n \"=%d\" % bp_filter[\"weights_2d\"].shape[2]\n )\n\n if (\n input_domain == \"spectral\"\n and field.shape[1] != bp_filter[\"weights_2d\"].shape[2]\n ):\n raise ValueError(\n \"Dimension mismatch between field and bp_filter: \"\n \"field.shape[1]=%d , \" % (field.shape[1] + 1)\n + \"bp_filter['weights_2d'].shape[2]\"\n \"=%d\" % bp_filter[\"weights_2d\"].shape[2]\n )\n\n if output_domain != \"spectral\":\n compact_output = False\n\n if np.any(~np.isfinite(field)):\n raise ValueError(\"field contains non-finite values\")\n\n result = {}\n means = []\n stds = []\n\n if subtract_mean and input_domain == \"spatial\":\n field_mean = np.mean(field)\n field = field - field_mean\n result[\"field_mean\"] = field_mean\n\n if input_domain == \"spatial\":\n field_fft = fft.rfft2(field)\n else:\n field_fft = field\n if output_domain == \"spectral\" and compact_output:\n weight_masks = []\n field_decomp = []\n\n for k in range(len(bp_filter[\"weights_1d\"])):\n field_ = field_fft * bp_filter[\"weights_2d\"][k, :, :]\n\n if output_domain == \"spatial\" or (compute_stats and mask is not None):\n field__ = fft.irfft2(field_)\n else:\n field__ = field_\n\n if compute_stats:\n if output_domain == \"spatial\" or (compute_stats and mask is not None):\n if mask is not None:\n masked_field = field__[mask]\n else:\n masked_field = field__\n mean = np.mean(masked_field)\n std = np.std(masked_field)\n else:\n mean = utils.spectral.mean(field_, bp_filter[\"shape\"])\n std = utils.spectral.std(field_, bp_filter[\"shape\"])\n\n means.append(mean)\n stds.append(std)\n\n if output_domain == \"spatial\":\n field_ = field__\n if normalize:\n field_ = (field_ - mean) / std\n if output_domain == \"spectral\" and compact_output:\n weight_mask = bp_filter[\"weights_2d\"][k, :, :] > 1e-12\n field_ = field_[weight_mask]\n weight_masks.append(weight_mask)\n field_decomp.append(field_)\n\n result[\"domain\"] = output_domain\n result[\"normalized\"] = normalize\n result[\"compact_output\"] = compact_output\n\n if output_domain == \"spatial\" or not compact_output:\n field_decomp = np.stack(field_decomp)\n\n result[\"cascade_levels\"] = field_decomp\n if output_domain == \"spectral\" and compact_output:\n result[\"weight_masks\"] = np.stack(weight_masks)\n\n if compute_stats:\n result[\"means\"] = means\n result[\"stds\"] = stds\n\n return result\n\n\ndef recompose_fft(decomp, **kwargs):\n \"\"\"\n Recompose a cascade obtained with decomposition_fft by inverting the\n normalization and summing the cascade levels.\n\n Parameters\n ----------\n decomp: dict\n A cascade decomposition returned by decomposition_fft.\n\n Returns\n -------\n out: numpy.ndarray\n A two-dimensional array containing the recomposed cascade.\n \"\"\"\n levels = decomp[\"cascade_levels\"]\n if decomp[\"normalized\"]:\n mu = decomp[\"means\"]\n sigma = decomp[\"stds\"]\n\n if not decomp[\"normalized\"] and not (\n decomp[\"domain\"] == \"spectral\" and decomp[\"compact_output\"]\n ):\n result = np.sum(levels, axis=0)\n else:\n if decomp[\"compact_output\"]:\n weight_masks = decomp[\"weight_masks\"]\n result = np.zeros(weight_masks.shape[1:], dtype=complex)\n\n for i in range(len(levels)):\n if decomp[\"normalized\"]:\n result[weight_masks[i]] += levels[i] * sigma[i] + mu[i]\n else:\n result[weight_masks[i]] += levels[i]\n else:\n result = [levels[i] * sigma[i] + mu[i] for i in range(len(levels))]\n result = np.sum(np.stack(result), axis=0)\n\n if \"field_mean\" in decomp:\n result += decomp[\"field_mean\"]\n\n return result\n" }, { "path": "pysteps/cascade/interface.py", "content": "\"\"\"\npysteps.cascade.interface\n=========================\n\nInterface for the cascade module.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nfrom pysteps.cascade import decomposition, bandpass_filters\n\n_cascade_methods = dict()\n_cascade_methods[\"fft\"] = (decomposition.decomposition_fft, decomposition.recompose_fft)\n_cascade_methods[\"gaussian\"] = bandpass_filters.filter_gaussian\n_cascade_methods[\"uniform\"] = bandpass_filters.filter_uniform\n\n\ndef get_method(name):\n \"\"\"\n Return a callable function for the bandpass filter or cascade decomposition\n method corresponding to the given name. For the latter, two functions are\n returned: the first is for the decomposing and the second is for recomposing\n the cascade.\n\n Filter methods:\n\n +-------------------+------------------------------------------------------+\n | Name | Description |\n +===================+======================================================+\n | gaussian | implementation of bandpass filter using Gaussian |\n | | weights |\n +-------------------+------------------------------------------------------+\n | uniform | implementation of a filter where all weights are set |\n | | to one |\n +-------------------+------------------------------------------------------+\n\n Decomposition/recomposition methods:\n\n +-------------------+------------------------------------------------------+\n | Name | Description |\n +===================+======================================================+\n | fft | decomposition into multiple spatial scales based on |\n | | the fast Fourier Transform (FFT) and a set of |\n | | bandpass filters |\n +-------------------+------------------------------------------------------+\n\n \"\"\"\n\n if isinstance(name, str):\n name = name.lower()\n else:\n raise TypeError(\n \"Only strings supported for the method's names.\\n\"\n + \"Available names:\"\n + str(list(_cascade_methods.keys()))\n ) from None\n try:\n return _cascade_methods[name]\n except KeyError:\n raise ValueError(\n \"Unknown method {}\\n\".format(name)\n + \"The available methods are:\"\n + str(list(_cascade_methods.keys()))\n ) from None\n" }, { "path": "pysteps/datasets.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.datasets\n================\n\nUtilities to download the pysteps data and to create a default pysteps configuration\nfile pointing to that data.\n\n.. autosummary::\n :toctree: ../generated/\n\n download_pysteps_data\n create_default_pystepsrc\n info\n load_dataset\n\"\"\"\n\nimport gzip\nimport json\nimport os\nimport shutil\nimport sys\nimport time\nfrom datetime import datetime, timedelta\nfrom logging.handlers import RotatingFileHandler\nfrom tempfile import NamedTemporaryFile, TemporaryDirectory\nfrom urllib import request\nfrom urllib.error import HTTPError\nfrom zipfile import ZipFile\n\nfrom jsmin import jsmin\n\nimport pysteps\nfrom pysteps import io\nfrom pysteps.exceptions import DirectoryNotEmpty\nfrom pysteps.utils import conversion\n\n# \"event name\" , \"%Y%m%d%H%M\"\n_precip_events = {\n \"fmi\": \"201609281445\",\n \"fmi2\": \"201705091045\",\n \"mch\": \"201505151545\",\n \"mch2\": \"201607112045\",\n \"mch3\": \"201701310945\",\n \"opera\": \"201808241800\",\n \"knmi\": \"201008260000\",\n \"bom\": \"201806161000\",\n \"mrms\": \"201906100000\",\n}\n\n_data_sources = {\n \"fmi\": \"Finish Meteorological Institute\",\n \"mch\": \"MeteoSwiss\",\n \"bom\": \"Australian Bureau of Meteorology\",\n \"knmi\": \"Royal Netherlands Meteorological Institute\",\n \"opera\": \"OPERA\",\n \"mrms\": \"NSSL's Multi-Radar/Multi-Sensor System\",\n}\n\n\n# Include this function here to avoid a dependency on pysteps.__init__.py\ndef _decode_filesystem_path(path):\n if not isinstance(path, str):\n return path.decode(sys.getfilesystemencoding())\n else:\n return path\n\n\ndef info():\n \"\"\"\n Describe the available datasets in the pysteps example data.\n\n >>> from pysteps import datasets\n >>> datasets.info()\n \"\"\"\n print(\"\\nAvailable datasets:\\n\")\n\n print(f\"{'Case':<8} {'Event date':<22} {'Source':<45}\\n\")\n\n for case_name, case_date in _precip_events.items():\n _source = \"\".join([i for i in case_name if not i.isdigit()])\n _source = _data_sources[_source]\n\n _case_date = datetime.strptime(_precip_events[case_name], \"%Y%m%d%H%M\")\n _case_date = datetime.strftime(_case_date, \"%Y-%m-%d %H:%M UTC\")\n\n print(f\"{case_name:<8} {_case_date:<22} {_source:<45}\")\n\n\nclass ShowProgress(object):\n \"\"\"\n Class used to report the download progress.\n\n Usage::\n\n >>> from urllib import request\n >>> pbar = ShowProgress()\n >>> request.urlretrieve(\"http://python.org/\", \"/tmp/index.html\", pbar)\n >>> pbar.end()\n \"\"\"\n\n def __init__(self, bar_length=20):\n self.prev_msg_width = 0\n self.init_time = None\n self.total_size = None\n self._progress_bar_length = bar_length\n\n def _clear_line(self):\n sys.stdout.write(\"\\b\" * self.prev_msg_width)\n sys.stdout.write(\"\\r\")\n\n def _print(self, msg):\n self.prev_msg_width = len(msg)\n sys.stdout.write(msg)\n\n def __call__(self, count, block_size, total_size, exact=True):\n self._clear_line()\n\n downloaded_size = count * block_size / (1024**2)\n\n if self.total_size is None and total_size > 0:\n self.total_size = total_size / (1024**2)\n\n if count == 0:\n self.init_time = time.time()\n progress_msg = \"\"\n else:\n if self.total_size is not None:\n progress = count * block_size / total_size\n block = int(round(self._progress_bar_length * progress))\n\n elapsed_time = time.time() - self.init_time\n eta = (elapsed_time / progress - elapsed_time) / 60\n\n bar_str = \"#\" * block + \"-\" * (self._progress_bar_length - block)\n\n if exact:\n downloaded_msg = (\n f\"({downloaded_size:.1f} Mb / {self.total_size:.1f} Mb)\"\n )\n else:\n downloaded_msg = (\n f\"(~{downloaded_size:.0f} Mb/ {self.total_size:.0f} Mb)\"\n )\n\n progress_msg = (\n f\"Progress: [{bar_str}]\"\n + downloaded_msg\n + f\" - Time left: {int(eta):d}:{int(eta * 60)} [m:s]\"\n )\n\n else:\n progress_msg = (\n f\"Progress: ({downloaded_size:.1f} Mb)\" f\" - Time left: unknown\"\n )\n\n self._print(progress_msg)\n\n @staticmethod\n def end(message=\"Download complete\"):\n sys.stdout.write(\"\\n\" + message + \"\\n\")\n\n\ndef download_mrms_data(dir_path, initial_date, final_date, timestep=2, nodelay=False):\n \"\"\"\n Download a small dataset with 6 hours of the NSSL's Multi-Radar/Multi-Sensor\n System ([MRMS](https://www.nssl.noaa.gov/projects/mrms/)) precipitation\n product (grib format).\n\n All the available files in the archive in the indicated time period\n (`initial_date` to `final_date`) are downloaded.\n By default, the timestep between files downloaded is 2 min.\n If the `timestep` is exactly divisible by 2 min, the immediately lower\n multiple is used. For example, if `timestep=5min`, the value is lowered to\n 4 min.\n\n Note\n ----\n To reduce the load on the archive's server, an internal delay of 5 seconds\n every 30 files downloaded is implemented.\n This delay can be disabled by setting `nodelay=True`.\n\n\n Parameters\n ----------\n dir_path: str\n Path to directory where the MRMS data is be placed.\n If None, the default location defined in the pystepsrc file is used.\n The files are archived following the folder structure defined in\n the pystepsrc file.\n If the directory exists existing MRMS files may be overwritten.\n initial_date: datetime\n Beginning of the date period.\n final_date: datetime\n End of the date period.\n timestep: int or timedelta\n Timestep between downloaded files in minutes.\n nodelay: bool\n Do not implement a 5-seconds delay every 30 files downloaded.\n \"\"\"\n\n if dir_path is None:\n data_source = pysteps.rcparams.data_sources[\"mrms\"]\n dir_path = data_source[\"root_path\"]\n\n if not isinstance(timestep, (int, timedelta)):\n raise TypeError(\n \"'timestep' must be an integer or a timedelta object.\"\n f\"Received: {type(timestep)}\"\n )\n\n if isinstance(timestep, int):\n timestep = timedelta(seconds=timestep * 60)\n\n if timestep.total_seconds() < 120:\n raise ValueError(\n \"The time step should be greater than 2 minutes.\"\n f\"Received: {timestep.total_seconds()}\"\n )\n\n _remainder = timestep % timedelta(seconds=120)\n timestep -= _remainder\n\n if not os.path.isdir(dir_path):\n os.makedirs(dir_path)\n\n if nodelay:\n\n def delay(_counter):\n return 0\n\n else:\n\n def delay(_counter):\n if _counter >= 30:\n _counter = 0\n time.sleep(5)\n return _counter\n\n archive_url = \"https://mtarchive.geol.iastate.edu\"\n print(f\"Downloading MRMS data from {archive_url}\")\n\n current_date = initial_date\n\n counter = 0\n while current_date <= final_date:\n counter = delay(counter)\n\n sub_dir = os.path.join(dir_path, datetime.strftime(current_date, \"%Y/%m/%d\"))\n\n if not os.path.isdir(sub_dir):\n os.makedirs(sub_dir)\n\n # Generate files URL from https://mtarchive.geol.iastate.edu\n dest_file_name = datetime.strftime(\n current_date, \"PrecipRate_00.00_%Y%m%d-%H%M%S.grib2\"\n )\n\n rel_url_fmt = (\n \"/%Y/%m/%d\"\n \"/mrms/ncep/PrecipRate\"\n \"/PrecipRate_00.00_%Y%m%d-%H%M%S.grib2.gz\"\n )\n\n file_url = archive_url + datetime.strftime(current_date, rel_url_fmt)\n\n try:\n print(f\"Downloading {file_url} \", end=\"\")\n tmp_file_name, _ = request.urlretrieve(file_url)\n print(\"DONE\")\n\n dest_file_path = os.path.join(sub_dir, dest_file_name)\n\n # Uncompress the data\n with gzip.open(tmp_file_name, \"rb\") as f_in:\n with open(dest_file_path, \"wb\") as f_out:\n shutil.copyfileobj(f_in, f_out)\n\n current_date = current_date + timedelta(seconds=60 * 2)\n counter += 1\n\n except HTTPError as err:\n print(err)\n\n\ndef download_pysteps_data(dir_path, force=True):\n \"\"\"\n Download pysteps data from github.\n\n Parameters\n ----------\n dir_path: str\n Path to directory where the psyteps data will be placed.\n force: bool\n If the destination directory exits and force=False, a DirectoryNotEmpty\n exception if raised.\n If force=True, the data will we downloaded in the destination directory and may\n override existing files.\n \"\"\"\n\n # Check if directory exists but is not empty\n if os.path.exists(dir_path) and os.path.isdir(dir_path):\n if os.listdir(dir_path) and not force:\n raise DirectoryNotEmpty(\n dir_path + \"is not empty.\\n\"\n \"Set force=True force the extraction of the files.\"\n )\n else:\n os.makedirs(dir_path)\n\n # NOTE:\n # The http response from github can either contain Content-Length (size of the file)\n # or use chunked Transfer-Encoding.\n # If Transfer-Encoding is chunked, then the Content-Length is not available since\n # the content is dynamically generated and we can't know the length a priori easily.\n pbar = ShowProgress()\n print(\"Downloading pysteps-data from github.\")\n tmp_file_name, _ = request.urlretrieve(\n \"https://github.com/pySTEPS/pysteps-data/archive/master.zip\",\n reporthook=pbar,\n )\n pbar.end(message=\"Download complete\\n\")\n\n with ZipFile(tmp_file_name, \"r\") as zip_obj:\n tmp_dir = TemporaryDirectory()\n\n # Extract all the contents of zip file in the temp directory\n common_path = os.path.commonprefix(zip_obj.namelist())\n\n zip_obj.extractall(tmp_dir.name)\n\n shutil.copytree(\n os.path.join(tmp_dir.name, common_path), dir_path, dirs_exist_ok=True\n )\n\n\ndef create_default_pystepsrc(\n pysteps_data_dir, config_dir=None, file_name=\"pystepsrc\", dryrun=False\n):\n \"\"\"\n Create a default configuration file pointing to the pysteps data directory.\n\n If the configuration file already exists, it will backup the existing file by\n appending the extensions '.1', '.2', up to '.5.' to the filename.\n A maximum of 5 files are kept. .2, up to app.log.5.\n\n File rotation is implemented for the backup files.\n For example, if the default configuration filename is 'pystepsrc' and the files\n pystepsrc, pystepsrc.1, pystepsrc.2, etc. exist, they are renamed to respectively\n pystepsrc.1, pystepsrc.2, pystepsrc.2, etc. Finally, after the existing files are\n backed up, the new configuration file is written.\n\n Parameters\n ----------\n pysteps_data_dir: str\n Path to the directory with the pysteps data.\n config_dir: str\n Destination directory for the configuration file.\n Default values: $HOME/.pysteps (unix and Mac OS X)\n or $USERPROFILE/pysteps (windows).\n The directory is created if it does not exists.\n file_name: str\n Configuration file name. `pystepsrc` by default.\n dryrun: bool\n Do not create the parameter file, nor create backups of existing files.\n No changes are made in the file system. It just returns the file path.\n\n Returns\n -------\n dest_path: str\n Configuration file path.\n \"\"\"\n\n pysteps_lib_root = os.path.dirname(_decode_filesystem_path(pysteps.__file__))\n\n # Load the library built-in configuration file\n with open(os.path.join(pysteps_lib_root, \"pystepsrc\"), \"r\") as f:\n rcparams_json = json.loads(jsmin(f.read()))\n\n for key, value in rcparams_json[\"data_sources\"].items():\n value[\"root_path\"] = os.path.abspath(\n os.path.join(pysteps_data_dir, value[\"root_path\"])\n )\n\n if config_dir is None:\n home_dir = os.path.expanduser(\"~\")\n if os.name == \"nt\":\n subdir = \"pysteps\"\n else:\n subdir = \".pysteps\"\n config_dir = os.path.join(home_dir, subdir)\n\n dest_path = os.path.join(config_dir, file_name)\n\n if not dryrun:\n if not os.path.isdir(config_dir):\n os.makedirs(config_dir)\n\n # Backup existing configuration files if it exists and rotate previous backups\n if os.path.isfile(dest_path):\n RotatingFileHandler(dest_path, backupCount=6).doRollover()\n\n with open(dest_path, \"w\") as f:\n json.dump(rcparams_json, f, indent=4)\n\n return os.path.normpath(dest_path)\n\n\ndef load_dataset(case=\"fmi\", frames=14):\n \"\"\"\n Load a sequence of radar composites from the pysteps example data.\n\n To print the available datasets run\n\n >>> from pysteps import datasets\n >>> datasets.info()\n\n This function load by default 14 composites, corresponding to a 1h and 10min\n time window.\n For example, the first two composites can be used to obtain the motion field of\n the precipitation pattern, while the remaining twelve composites can be used to\n evaluate the quality of our forecast.\n\n Calling this function requires the pysteps-data installed, otherwise an exception\n is raised. To install the pysteps example data check the `example_data` section.\n\n Parameters\n ----------\n case: str\n Case to load.\n frames: int\n Number composites (radar images).\n Max allowed value: 24 (35 for MRMS product)\n Default: 14\n\n Returns\n -------\n rainrate: array-like\n Precipitation data in mm/h. Dimensions: [time, lat, lon]\n metadata: dict\n The metadata observations attributes.\n timestep: number\n Time interval between composites in minutes.\n \"\"\"\n\n case = case.lower()\n\n if case == \"mrms\":\n max_frames = 36\n else:\n max_frames = 24\n if frames > max_frames:\n raise ValueError(\n f\"The number of frames should be smaller than {max_frames + 1}\"\n )\n\n case_date = datetime.strptime(_precip_events[case], \"%Y%m%d%H%M\")\n\n source = \"\".join([i for i in case if not i.isdigit()])\n data_source = pysteps.rcparams.data_sources[source]\n\n # Find the input files from the archive\n file_names = io.archive.find_by_date(\n case_date,\n data_source[\"root_path\"],\n data_source[\"path_fmt\"],\n data_source[\"fn_pattern\"],\n data_source[\"fn_ext\"],\n data_source[\"timestep\"],\n num_prev_files=0,\n num_next_files=frames - 1,\n )\n\n if None in file_names[0]:\n raise FileNotFoundError(f\"Error loading {case} case. Some files are missing.\")\n\n # Read the radar composites\n importer = io.get_method(data_source[\"importer\"], \"importer\")\n importer_kwargs = data_source[\"importer_kwargs\"]\n reflectivity, _, metadata = io.read_timeseries(\n file_names, importer, **importer_kwargs\n )\n\n # Convert to rain rate\n precip, metadata = conversion.to_rainrate(reflectivity, metadata)\n\n return precip, metadata, data_source[\"timestep\"]\n" }, { "path": "pysteps/decorators.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.decorators\n==================\n\nDecorators used to define reusable building blocks that can change or extend\nthe behavior of some functions in pysteps.\n\n.. autosummary::\n :toctree: ../generated/\n\n postprocess_import\n check_input_frames\n prepare_interpolator\n memoize\n\"\"\"\n\nimport inspect\nimport uuid\nimport warnings\nfrom collections import defaultdict\nfrom functools import wraps\n\nimport numpy as np\n\n\ndef _add_extra_kwrds_to_docstrings(target_func, extra_kwargs_doc_text):\n \"\"\"\n Update the functions docstrings by replacing the `{extra_kwargs_doc}` occurences in\n the docstring by the `extra_kwargs_doc_text` value.\n \"\"\"\n # Clean up indentation from docstrings for the\n # docstrings to be merged correctly.\n extra_kwargs_doc = inspect.cleandoc(extra_kwargs_doc_text)\n target_func.__doc__ = inspect.cleandoc(target_func.__doc__)\n\n # Add extra kwargs docstrings\n target_func.__doc__ = target_func.__doc__.format_map(\n defaultdict(str, extra_kwargs_doc=extra_kwargs_doc)\n )\n return target_func\n\n\ndef postprocess_import(fillna=np.nan, dtype=\"double\"):\n \"\"\"\n Postprocess the imported precipitation data.\n Operations:\n - Allow type casting (dtype keyword)\n - Set invalid or missing data to predefined value (fillna keyword)\n This decorator replaces the text \"{extra_kwargs}\" in the function's\n docstring with the documentation of the keywords used in the postprocessing.\n The additional docstrings are added as \"Other Parameters\" in the importer function.\n\n Parameters\n ----------\n dtype: str\n Default data type for precipitation. Double precision by default.\n fillna: float or np.nan\n Default value used to represent the missing data (\"No Coverage\").\n By default, np.nan is used.\n If the importer returns a MaskedArray, all the masked values are set to the\n fillna value. If a numpy array is returned, all the invalid values (nan and inf)\n are set to the fillna value.\n\n \"\"\"\n\n def _postprocess_import(importer):\n @wraps(importer)\n def _import_with_postprocessing(*args, **kwargs):\n precip, *other_args = importer(*args, **kwargs)\n\n _dtype = kwargs.get(\"dtype\", dtype)\n\n accepted_precisions = [\"float32\", \"float64\", \"single\", \"double\"]\n if _dtype not in accepted_precisions:\n raise ValueError(\n \"The selected precision does not correspond to a valid value.\"\n \"The accepted values are: \" + str(accepted_precisions)\n )\n\n if isinstance(precip, np.ma.MaskedArray):\n invalid_mask = np.ma.getmaskarray(precip)\n precip.data[invalid_mask] = fillna\n else:\n # If plain numpy arrays are used, the importers should indicate\n # the invalid values with np.nan.\n _fillna = kwargs.get(\"fillna\", fillna)\n if _fillna is not np.nan:\n mask = ~np.isfinite(precip)\n precip[mask] = _fillna\n\n return (precip.astype(_dtype),) + tuple(other_args)\n\n extra_kwargs_doc = \"\"\"\n Other Parameters\n ----------------\n dtype: str\n Data-type to which the array is cast.\n Valid values: \"float32\", \"float64\", \"single\", and \"double\".\n fillna: float or np.nan\n Value used to represent the missing data (\"No Coverage\").\n By default, np.nan is used.\n \"\"\"\n\n _add_extra_kwrds_to_docstrings(_import_with_postprocessing, extra_kwargs_doc)\n\n return _import_with_postprocessing\n\n return _postprocess_import\n\n\ndef check_input_frames(\n minimum_input_frames=2, maximum_input_frames=np.inf, just_ndim=False\n):\n \"\"\"\n Check that the input_images used as inputs in the optical-flow\n methods have the correct shape (t, x, y ).\n \"\"\"\n\n def _check_input_frames(motion_method_func):\n @wraps(motion_method_func)\n def new_function(*args, **kwargs):\n \"\"\"\n Return new function with the checks prepended to the\n target motion_method_func function.\n \"\"\"\n\n input_images = args[0]\n if input_images.ndim != 3:\n raise ValueError(\n \"input_images dimension mismatch.\\n\"\n f\"input_images.shape: {str(input_images.shape)}\\n\"\n \"(t, x, y ) dimensions expected\"\n )\n\n if not just_ndim:\n num_of_frames = input_images.shape[0]\n\n if minimum_input_frames < num_of_frames > maximum_input_frames:\n raise ValueError(\n f\"input_images frames {num_of_frames} mismatch.\\n\"\n f\"Minimum frames: {minimum_input_frames}\\n\"\n f\"Maximum frames: {maximum_input_frames}\\n\"\n )\n\n return motion_method_func(*args, **kwargs)\n\n return new_function\n\n return _check_input_frames\n\n\ndef prepare_interpolator(nchunks=4):\n \"\"\"\n Check that all the inputs have the correct shape, and that all values are\n finite. It also split the destination grid in `nchunks` parts, and process each\n part independently.\n \"\"\"\n\n def _preamble_interpolation(interpolator):\n @wraps(interpolator)\n def _interpolator_with_preamble(xy_coord, values, xgrid, ygrid, **kwargs):\n nonlocal nchunks # https://stackoverflow.com/questions/5630409/\n\n values = values.copy()\n xy_coord = xy_coord.copy()\n\n input_ndims = values.ndim\n input_nvars = 1 if input_ndims == 1 else values.shape[1]\n input_nsamples = values.shape[0]\n\n coord_ndims = xy_coord.ndim\n coord_nsamples = xy_coord.shape[0]\n\n grid_shape = (ygrid.size, xgrid.size)\n\n if np.any(~np.isfinite(values)):\n raise ValueError(\"argument 'values' contains non-finite values\")\n if np.any(~np.isfinite(xy_coord)):\n raise ValueError(\"argument 'xy_coord' contains non-finite values\")\n\n if input_ndims > 2:\n raise ValueError(\n \"argument 'values' must have 1 (n) or 2 dimensions (n, m), \"\n f\"but it has {input_ndims}\"\n )\n if not coord_ndims == 2:\n raise ValueError(\n \"argument 'xy_coord' must have 2 dimensions (n, 2), \"\n f\"but it has {coord_ndims}\"\n )\n\n if not input_nsamples == coord_nsamples:\n raise ValueError(\n \"the number of samples in argument 'values' does not match the \"\n f\"number of coordinates {input_nsamples}!={coord_nsamples}\"\n )\n\n # only one sample, return uniform output\n if input_nsamples == 1:\n output_array = np.ones((input_nvars,) + grid_shape)\n for n, v in enumerate(values[0, ...]):\n output_array[n, ...] *= v\n return output_array.squeeze()\n\n # all equal elements, return uniform output\n if values.max() == values.min():\n return np.ones((input_nvars,) + grid_shape) * values.ravel()[0]\n\n # split grid in n chunks\n nchunks = int(kwargs.get(\"nchunks\", nchunks) ** 0.5)\n if nchunks > 1:\n subxgrids = np.array_split(xgrid, nchunks)\n subxgrids = [x for x in subxgrids if x.size > 0]\n subygrids = np.array_split(ygrid, nchunks)\n subygrids = [y for y in subygrids if y.size > 0]\n\n # generate a unique identifier to be used for caching\n # intermediate results\n kwargs[\"hkey\"] = uuid.uuid1().int\n else:\n subxgrids = [xgrid]\n subygrids = [ygrid]\n\n interpolated = np.zeros((input_nvars,) + grid_shape)\n indx = 0\n for subxgrid in subxgrids:\n deltax = subxgrid.size\n indy = 0\n for subygrid in subygrids:\n deltay = subygrid.size\n interpolated[:, indy : (indy + deltay), indx : (indx + deltax)] = (\n interpolator(xy_coord, values, subxgrid, subygrid, **kwargs)\n )\n indy += deltay\n indx += deltax\n\n return interpolated.squeeze()\n\n extra_kwargs_doc = \"\"\"\n nchunks: int, optional\n Split and process the destination grid in nchunks.\n Useful for large grids to limit the memory footprint.\n \"\"\"\n\n _add_extra_kwrds_to_docstrings(_interpolator_with_preamble, extra_kwargs_doc)\n\n return _interpolator_with_preamble\n\n return _preamble_interpolation\n\n\ndef memoize(maxsize=10):\n \"\"\"\n Add a Least Recently Used (LRU) cache to any function.\n Caching is purely based on the optional keyword argument 'hkey', which needs\n to be a hashable.\n\n Parameters\n ----------\n maxsize: int, optional\n The maximum number of elements stored in the LRU cache.\n \"\"\"\n\n def _memoize(func):\n cache = dict()\n hkeys = []\n\n @wraps(func)\n def _func_with_cache(*args, **kwargs):\n hkey = kwargs.pop(\"hkey\", None)\n if hkey in cache:\n return cache[hkey]\n result = func(*args, **kwargs)\n if hkey is not None:\n cache[hkey] = result\n hkeys.append(hkey)\n if len(hkeys) > maxsize:\n cache.pop(hkeys.pop(0))\n\n return result\n\n return _func_with_cache\n\n return _memoize\n\n\ndef deprecate_args(old_new_args, deprecation_release):\n \"\"\"\n Support deprecated argument names while issuing deprecation warnings.\n\n Parameters\n ----------\n old_new_args: dict[str, str]\n Mapping from old to new argument names.\n deprecation_release: str\n Specify which future release will convert this warning into an error.\n \"\"\"\n\n def _deprecate(func):\n @wraps(func)\n def wrapper(*args, **kwargs):\n kwargs_names = list(kwargs.keys())\n for key_old in kwargs_names:\n if key_old in old_new_args:\n key_new = old_new_args[key_old]\n kwargs[key_new] = kwargs.pop(key_old)\n warnings.warn(\n f\"Argument '{key_old}' has been renamed to '{key_new}'. \"\n f\"This will raise a TypeError in pysteps {deprecation_release}.\",\n FutureWarning,\n )\n return func(*args, **kwargs)\n\n return wrapper\n\n return _deprecate\n" }, { "path": "pysteps/downscaling/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implementations of deterministic and ensemble downscaling methods.\"\"\"\n\nfrom pysteps.downscaling.interface import get_method\n" }, { "path": "pysteps/downscaling/interface.py", "content": "\"\"\"\npysteps.downscaling.interface\n=============================\n\nInterface for the downscaling module. It returns a callable function for computing\ndownscaling.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nfrom pysteps.downscaling import rainfarm\n\n_downscale_methods = dict()\n_downscale_methods[\"rainfarm\"] = rainfarm.downscale\n\n\ndef get_method(name):\n \"\"\"\n Return a callable function for computing downscaling.\n\n Description:\n Return a callable function for computing deterministic or ensemble\n precipitation downscaling.\n\n Implemented methods:\n\n +-----------------+-------------------------------------------------------+\n | Name | Description |\n +=================+=======================================================+\n | rainfarm | the rainfall downscaling by a filtered autoregressive |\n | | model (RainFARM) method developed in |\n | | :cite:`Rebora2006` |\n +-----------------+-------------------------------------------------------+\n \"\"\"\n if isinstance(name, str):\n name = name.lower()\n else:\n raise TypeError(\n \"Only strings supported for the method's names.\\n\"\n + \"Available names:\"\n + str(list(_downscale_methods.keys()))\n ) from None\n\n try:\n return _downscale_methods[name]\n except KeyError:\n raise ValueError(\n \"Unknown downscaling method {}\\n\".format(name)\n + \"The available methods are:\"\n + str(list(_downscale_methods.keys()))\n ) from None\n" }, { "path": "pysteps/downscaling/rainfarm.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.downscaling.rainfarm\n============================\n\nImplementation of the RainFARM stochastic downscaling method as described in\n:cite:`Rebora2006` and :cite:`DOnofrio2014`.\n\nRainFARM is a downscaling algorithm for rainfall fields developed by Rebora et\nal. (2006). The method can represent the realistic small-scale variability of the\ndownscaled precipitation field by means of Gaussian random fields.\n\n\n.. autosummary::\n :toctree: ../generated/\n\n downscale\n\"\"\"\n\nimport warnings\n\nimport numpy as np\nfrom scipy.signal import convolve\nfrom pysteps.utils.spectral import rapsd\nfrom pysteps.utils.dimension import aggregate_fields\n\n\ndef _gaussianize(precip):\n \"\"\"\n Gaussianize field using rank ordering as in :cite:`DOnofrio2014`.\n \"\"\"\n m, n = np.shape(precip)\n nn = m * n\n ii = np.argsort(precip.reshape(nn))\n precip_gaussianize = np.zeros(nn)\n precip_gaussianize[ii] = sorted(np.random.normal(0, 1, nn))\n precip_gaussianize = precip_gaussianize.reshape(m, n)\n sd = np.std(precip_gaussianize)\n if sd == 0:\n sd = 1\n return precip_gaussianize / sd\n\n\ndef _compute_freq_array(array, ds_factor=1):\n \"\"\"\n Compute the frequency array following a given downscaling factor.\n \"\"\"\n freq_i = np.fft.fftfreq(array.shape[0] * ds_factor, d=1 / ds_factor)\n freq_j = np.fft.fftfreq(array.shape[1] * ds_factor, d=1 / ds_factor)\n freq_sqr = freq_i[:, None] ** 2 + freq_j[None, :] ** 2\n return np.sqrt(freq_sqr)\n\n\ndef _log_slope(log_k, log_power_spectrum):\n \"\"\"\n Calculate the log-slope of the power spectrum given an array of logarithmic wavenumbers\n and an array of logarithmic power spectrum values.\n \"\"\"\n lk_min = log_k.min()\n lk_max = log_k.max()\n lk_range = lk_max - lk_min\n lk_min += (1 / 6) * lk_range\n lk_max -= (1 / 6) * lk_range\n selected = (lk_min <= log_k) & (log_k <= lk_max)\n lk_sel = log_k[selected]\n ps_sel = log_power_spectrum[selected]\n alpha = np.polyfit(lk_sel, ps_sel, 1)[0]\n alpha = -alpha\n return alpha\n\n\ndef _estimate_alpha(array, k):\n \"\"\"\n Estimate the alpha parameter using the power spectrum of the input array.\n \"\"\"\n fp = np.fft.fft2(array)\n fp_abs = abs(fp)\n log_power_spectrum = np.log(fp_abs**2)\n valid = (k != 0) & np.isfinite(log_power_spectrum)\n alpha = _log_slope(np.log(k[valid]), log_power_spectrum[valid])\n return alpha\n\n\ndef _compute_noise_field(freq_array_highres, alpha):\n \"\"\"\n Compute a field of correlated noise field using the given frequency array and alpha\n value.\n \"\"\"\n white_noise_field = np.random.rand(*freq_array_highres.shape)\n white_noise_field_complex = np.exp(complex(0, 1) * 2 * np.pi * white_noise_field)\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\")\n noise_field_complex = white_noise_field_complex * np.sqrt(\n freq_array_highres**-alpha\n )\n noise_field_complex[0, 0] = 0\n return np.fft.ifft2(noise_field_complex).real\n\n\ndef _apply_spectral_fusion(\n array_low, array_high, freq_array_low, freq_array_high, ds_factor\n):\n \"\"\"\n Apply spectral fusion to merge two arrays in the frequency domain.\n \"\"\"\n\n # Validate inputs\n if array_low.shape != freq_array_low.shape:\n raise ValueError(\"Shape of array_low must match shape of freq_array_low.\")\n if array_high.shape != freq_array_high.shape:\n raise ValueError(\"Shape of array_high must match shape of freq_array_high.\")\n\n nax, _ = np.shape(array_low)\n nx, _ = np.shape(array_high)\n k0 = nax // 2\n\n # Calculate power spectral density at specific frequency\n def compute_psd(array, fft_size):\n return rapsd(array, fft_method=np.fft)[k0 - 1] * fft_size**2\n\n psd_low = compute_psd(array_low, nax)\n psd_high = compute_psd(array_high, nx)\n\n # Normalize high-resolution array\n normalization_factor = np.sqrt(psd_low / psd_high)\n array_high *= normalization_factor\n\n # Perform FFT on both arrays\n fft_low = np.fft.fft2(array_low)\n fft_high = np.fft.fft2(array_high)\n\n # Initialize the merged FFT array with low-resolution data\n fft_merged = np.zeros_like(fft_high, dtype=np.complex128)\n fft_merged[0:k0, 0:k0] = fft_low[0:k0, 0:k0]\n fft_merged[nx - k0 : nx, 0:k0] = fft_low[k0 : 2 * k0, 0:k0]\n fft_merged[0:k0, nx - k0 : nx] = fft_low[0:k0, k0 : 2 * k0]\n fft_merged[nx - k0 : nx, nx - k0 : nx] = fft_low[k0 : 2 * k0, k0 : 2 * k0]\n\n fft_merged[k0, 0] = np.conj(fft_merged[nx - k0, 0])\n fft_merged[0, k0] = np.conj(fft_merged[0, nx - k0])\n\n # Compute frequency arrays\n freq_i = np.fft.fftfreq(nx, d=1 / ds_factor)\n freq_i = np.tile(freq_i, (nx, 1))\n freq_j = freq_i.T\n\n # Compute frequency domain adjustment\n ddx = np.pi * (1 / nax - 1 / nx) / np.abs(freq_i[0, 1] - freq_i[0, 0])\n freq_squared_high = freq_array_high**2\n freq_squared_low_center = freq_array_low[k0, k0] ** 2\n\n # Fuse in the frequency domain\n mask_high = freq_squared_high > freq_squared_low_center\n mask_low = ~mask_high\n fft_merged = fft_high * mask_high + fft_merged * mask_low * np.exp(\n -1j * ddx * freq_i - 1j * ddx * freq_j\n )\n\n # Inverse FFT to obtain the merged array in the spatial domain\n merged = np.real(np.fft.ifftn(fft_merged)) / fft_merged.size\n\n return merged\n\n\ndef _compute_kernel_radius(ds_factor):\n return int(round(ds_factor / np.sqrt(np.pi)))\n\n\ndef _make_tophat_kernel(ds_factor):\n \"\"\"Compute 2d uniform (tophat) kernel\"\"\"\n radius = _compute_kernel_radius(ds_factor)\n mx, my = np.mgrid[-radius : radius + 0.01, -radius : radius + 0.01]\n tophat = ((mx**2 + my**2) <= radius**2).astype(float)\n return tophat / tophat.sum()\n\n\ndef _make_gaussian_kernel(ds_factor):\n \"\"\"\n Compute 2d gaussian kernel\n ref: https://github.com/scipy/scipy/blob/de80faf9d3480b9dbb9b888568b64499e0e70c19/scipy/ndimage/_filters.py#L179\n the smoothing sigma has width half a large pixel\n \"\"\"\n radius = _compute_kernel_radius(ds_factor)\n sigma = ds_factor / 2\n sigma2 = sigma * sigma\n x = np.arange(-radius, radius + 1)\n kern1d = np.exp(-0.5 / sigma2 * x**2)\n kern2d = np.outer(kern1d, kern1d)\n return kern2d / kern2d.sum()\n\n\ndef _balanced_spatial_average(array, kernel):\n \"\"\"\n Compute the balanced spatial average of an array using a given kernel while handling\n missing or invalid values.\n \"\"\"\n array = array.copy()\n mask_valid = np.isfinite(array)\n array[~mask_valid] = 0.0\n array_conv = convolve(array, kernel, mode=\"same\")\n array_conv /= convolve(mask_valid, kernel, mode=\"same\")\n array_conv[~mask_valid] = np.nan\n return array_conv\n\n\n_make_kernel = dict()\n_make_kernel[\"gaussian\"] = _make_gaussian_kernel\n_make_kernel[\"tophat\"] = _make_tophat_kernel\n_make_kernel[\"uniform\"] = _make_tophat_kernel\n\n\ndef downscale(\n precip,\n ds_factor,\n alpha=None,\n threshold=None,\n return_alpha=False,\n kernel_type=None,\n spectral_fusion=False,\n):\n \"\"\"\n Downscale a rainfall field by increasing its spatial resolution by a positive\n integer factor.\n\n Parameters\n ----------\n precip: array_like\n Array of shape (m, n) containing the input field.\n The input is expected to contain rain rate values.\n All values are required to be finite.\n alpha: float, optional\n Spectral slope. If None, the slope is estimated from\n the input array.\n ds_factor: positive int\n Downscaling factor, it specifies by how many times\n to increase the initial grid resolution.\n threshold: float, optional\n Set all values lower than the threshold to zero.\n return_alpha: bool, optional\n Whether to return the estimated spectral slope ``alpha``.\n kernel_type: {None, \"gaussian\", \"uniform\", \"tophat\"}\n The name of the smoothing operator. If None no smoothing is applied.\n spectral_fusion: bool, optional\n Whether to apply spectral merging as in :cite:`DOnofrio2014`.\n\n Returns\n -------\n precip_highres: ndarray\n Array of shape (m * ds_factor, n * ds_factor) containing\n the downscaled field.\n alpha: float\n Returned only when ``return_alpha=True``.\n\n Notes\n -----\n Currently, the pysteps implementation of RainFARM only covers spatial downscaling.\n That is, it can improve the spatial resolution of a rainfall field. However, unlike\n the original algorithm from Rebora et al. (2006), it cannot downscale the temporal\n dimension. It implements spectral merging from D'Onofrio et al. (2014).\n\n References\n ----------\n :cite:`Rebora2006`\n :cite:`DOnofrio2014`\n\n \"\"\"\n\n # Validate inputs\n if not np.isfinite(precip).all():\n raise ValueError(\"All values in 'precip' must be finite.\")\n if not isinstance(ds_factor, int) or ds_factor <= 0:\n raise ValueError(\"'ds_factor' must be a positive integer.\")\n\n # Preprocess the input field if spectral fusion is enabled\n precip_transformed = _gaussianize(precip) if spectral_fusion else precip\n\n # Compute frequency arrays for the original and high-resolution fields\n freq_array = _compute_freq_array(precip_transformed)\n freq_array_highres = _compute_freq_array(precip_transformed, ds_factor)\n\n # Estimate spectral slope alpha if not provided\n if alpha is None:\n alpha = _estimate_alpha(precip_transformed, freq_array)\n\n # Generate noise field\n noise_field = _compute_noise_field(freq_array_highres, alpha)\n\n # Apply spectral fusion if enabled\n if spectral_fusion:\n noise_field /= noise_field.shape[0] ** 2\n noise_field = np.exp(noise_field)\n noise_field = _apply_spectral_fusion(\n precip_transformed, noise_field, freq_array, freq_array_highres, ds_factor\n )\n\n # Normalize and exponentiate the noise field\n noise_field /= noise_field.std()\n noise_field = np.exp(noise_field)\n\n # Aggregate the noise field to low resolution\n noise_lowres = aggregate_fields(noise_field, ds_factor, axis=(0, 1))\n\n # Expand input and noise fields to high resolution\n precip_expanded = np.kron(precip, np.ones((ds_factor, ds_factor)))\n noise_lowres_expanded = np.kron(noise_lowres, np.ones((ds_factor, ds_factor)))\n\n # Apply smoothing if a kernel type is provided\n if kernel_type:\n if kernel_type not in _make_kernel:\n raise ValueError(\n f\"kernel type '{kernel_type}' is invalid, available kernels: {list(_make_kernel)}\"\n )\n kernel = _make_kernel[kernel_type](ds_factor)\n precip_expanded = _balanced_spatial_average(precip_expanded, kernel)\n noise_lowres_expanded = _balanced_spatial_average(noise_lowres_expanded, kernel)\n\n # Normalize the high-res precipitation field by the low-res noise field\n norm_k0 = precip_expanded / noise_lowres_expanded\n precip_highres = noise_field * norm_k0\n\n # Apply thresholding if specified\n if threshold is not None:\n precip_highres[precip_highres < threshold] = 0\n\n # Return the downscaled field and optionally the spectral slope alpha\n if return_alpha:\n return precip_highres, alpha\n\n return precip_highres\n" }, { "path": "pysteps/exceptions.py", "content": "# -*- coding: utf-8 -*-\n\n# Custom pySteps exceptions\n\n\nclass MissingOptionalDependency(Exception):\n \"\"\"Raised when an optional dependency is needed but not found.\"\"\"\n\n pass\n\n\nclass DirectoryNotEmpty(Exception):\n \"\"\"Raised when the destination directory in a file copy operation is not empty.\"\"\"\n\n pass\n\n\nclass DataModelError(Exception):\n \"\"\"Raised when a file is not compilant with the Data Information Model.\"\"\"\n\n pass\n" }, { "path": "pysteps/extrapolation/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\n Methods for advection-based extrapolation of precipitation fields.\nCurrently the module contains an implementation of the\nsemi-Lagrangian method described in :cite:`GZ2002` and the\neulerian persistence.\"\"\"\n\nfrom pysteps.extrapolation.interface import get_method\n" }, { "path": "pysteps/extrapolation/interface.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.extrapolation.interface\n===============================\n\nThe functions in the extrapolation module implement the following interface::\n\n ``extrapolate(extrap, precip, velocity, timesteps, outval=np.nan, **keywords)``\n\nwhere *extrap* is an extrapolator object returned by the initialize function,\n*precip* is a (m,n) array with input precipitation field to be advected and\n*velocity* is a (2,m,n) array containing the x- and y-components of\nthe m x n advection field.\ntimesteps is an integer or list specifying the time steps to extrapolate. If\nan integer is given, a range of uniformly spaced steps 1,2,...,timesteps is\ncreated. If a list is given, it is assumed to represent a sequence of\nmonotonously increasing time steps. One time unit is assumed to represent the\ntime step of the advection field.\nThe optional argument *outval* specifies the value for pixels advected\nfrom outside the domain.\nOptional keyword arguments that are specific to a given extrapolation\nmethod are passed as a dictionary.\n\nThe output of each method is an array that contains the time series of\nextrapolated fields of shape (num_timesteps, m, n).\n\n.. currentmodule:: pysteps.extrapolation.interface\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n eulerian_persistence\n\"\"\"\n\nimport numpy as np\n\nfrom pysteps.extrapolation import semilagrangian\n\n\ndef eulerian_persistence(precip, velocity, timesteps, outval=np.nan, **kwargs):\n \"\"\"\n A dummy extrapolation method to apply Eulerian persistence to a\n two-dimensional precipitation field. The method returns the a sequence\n of the same initial field with no extrapolation applied (i.e. Eulerian\n persistence).\n\n Parameters\n ----------\n precip : array-like\n Array of shape (m,n) containing the input precipitation field. All\n values are required to be finite.\n velocity : array-like\n Not used by the method.\n timesteps : int or list of floats\n Number of time steps or a list of time steps.\n outval : float, optional\n Not used by the method.\n\n Other Parameters\n ----------------\n return_displacement : bool\n If True, return the total advection velocity (displacement) between the\n initial input field and the advected one integrated along\n the trajectory. Default : False\n\n Returns\n -------\n out : array or tuple\n If return_displacement=False, return a sequence of the same initial field\n of shape (num_timesteps,m,n). Otherwise, return a tuple containing the\n replicated fields and a (2,m,n) array of zeros.\n\n References\n ----------\n :cite:`GZ2002`\n\n \"\"\"\n del velocity, outval # Unused by _eulerian_persistence\n\n if isinstance(timesteps, int):\n num_timesteps = timesteps\n else:\n num_timesteps = len(timesteps)\n\n return_displacement = kwargs.get(\"return_displacement\", False)\n\n extrapolated_precip = np.repeat(precip[np.newaxis, :, :], num_timesteps, axis=0)\n\n if not return_displacement:\n return extrapolated_precip\n else:\n return extrapolated_precip, np.zeros((2,) + extrapolated_precip.shape)\n\n\ndef _do_nothing(precip, velocity, timesteps, outval=np.nan, **kwargs):\n \"\"\"Return None.\"\"\"\n del precip, velocity, timesteps, outval, kwargs # Unused\n return None\n\n\ndef _return_none(**kwargs):\n del kwargs # Not used\n return None\n\n\n_extrapolation_methods = dict()\n_extrapolation_methods[\"eulerian\"] = eulerian_persistence\n_extrapolation_methods[\"semilagrangian\"] = semilagrangian.extrapolate\n_extrapolation_methods[None] = _do_nothing\n_extrapolation_methods[\"none\"] = _do_nothing\n\n\ndef get_method(name):\n \"\"\"\n Return two-element tuple for the extrapolation method corresponding to\n the given name. The elements of the tuple are callable functions for the\n initializer of the extrapolator and the extrapolation method, respectively.\n The available options are:\\n\n\n +-----------------+--------------------------------------------------------+\n | Name | Description |\n +=================+========================================================+\n | None | returns None |\n +-----------------+--------------------------------------------------------+\n | eulerian | this methods does not apply any advection to the input |\n | | precipitation field (Eulerian persistence) |\n +-----------------+--------------------------------------------------------+\n | semilagrangian | implementation of the semi-Lagrangian method described |\n | | in :cite:`GZ2002` |\n +-----------------+--------------------------------------------------------+\n\n \"\"\"\n if isinstance(name, str):\n name = name.lower()\n\n try:\n return _extrapolation_methods[name]\n\n except KeyError:\n raise ValueError(\n \"Unknown method {}\\n\".format(name)\n + \"The available methods are:\"\n + str(list(_extrapolation_methods.keys()))\n ) from None\n" }, { "path": "pysteps/extrapolation/semilagrangian.py", "content": "\"\"\"\npysteps.extrapolation.semilagrangian\n====================================\n\nImplementation of the semi-Lagrangian method described in :cite:`GZ2002`.\n\n.. autosummary::\n :toctree: ../generated/\n\n extrapolate\n\n\"\"\"\n\nimport time\nimport warnings\n\nimport numpy as np\nfrom scipy.ndimage import map_coordinates\n\n\ndef extrapolate(\n precip,\n velocity,\n timesteps,\n outval=np.nan,\n xy_coords=None,\n allow_nonfinite_values=False,\n vel_timestep=1,\n **kwargs,\n):\n \"\"\"Apply semi-Lagrangian backward extrapolation to a two-dimensional\n precipitation field.\n\n Parameters\n ----------\n precip: array-like or None\n Array of shape (m,n) containing the input precipitation field. All\n values are required to be finite by default. If set to None, only the\n displacement field is returned without interpolating the inputs. This\n requires that return_displacement is set to True.\n velocity: array-like\n Array of shape (2,m,n) containing the x- and y-components of the m*n\n advection field. All values are required to be finite by default.\n timesteps: int or list of floats\n If timesteps is integer, it specifies the number of time steps to\n extrapolate. If a list is given, each element is the desired\n extrapolation time step from the current time. The elements of the list\n are required to be in ascending order.\n outval: float, optional\n Optional argument for specifying the value for pixels advected from\n outside the domain. If outval is set to 'min', the value is taken as\n the minimum value of precip.\n Default: np.nan\n xy_coords: ndarray, optional\n Array with the coordinates of the grid dimension (2, m, n ).\n\n * xy_coords[0]: x coordinates\n * xy_coords[1]: y coordinates\n\n By default, the *xy_coords* are computed for each extrapolation.\n allow_nonfinite_values: bool, optional\n If True, allow non-finite values in the precipitation and advection\n fields. This option is useful if the input fields contain a radar mask\n (i.e. pixels with no observations are set to nan).\n\n Other Parameters\n ----------------\n displacement_prev: array-like\n Optional initial displacement vector field of shape (2,m,n) for the\n extrapolation.\n Default: None\n n_iter: int\n Number of inner iterations in the semi-Lagrangian scheme. If n_iter > 0,\n the integration is done using the midpoint rule. Otherwise, the advection\n vectors are taken from the starting point of each interval.\n Default: 1\n return_displacement: bool\n If True, return the displacement between the initial input field and\n the one obtained by integrating along the advection field.\n Default: False\n vel_timestep: float\n The time step of the velocity field. It is assumed to have the same\n unit as the timesteps argument. Applicable if timesteps is a list.\n Default: 1.\n interp_order: int\n The order of interpolation to use. Default: 1 (linear). Setting this\n to 0 (nearest neighbor) gives the best computational performance but\n may produce visible artefacts. Setting this to 3 (cubic) gives the best\n ability to reproduce small-scale variability but may significantly\n increase the computation time.\n\n Returns\n -------\n out: array or tuple\n If return_displacement=False, return a time series extrapolated fields\n of shape (num_timesteps,m,n). Otherwise, return a tuple containing the\n extrapolated fields and the integrated trajectory (displacement) along\n the advection field.\n\n\n References\n ----------\n :cite:`GZ2002`\n \"\"\"\n\n if precip is not None and precip.ndim != 2:\n raise ValueError(\"precip must be a two-dimensional array\")\n\n if velocity.ndim != 3:\n raise ValueError(\"velocity must be a three-dimensional array\")\n\n if not allow_nonfinite_values:\n if precip is not None and np.any(~np.isfinite(precip)):\n raise ValueError(\"precip contains non-finite values\")\n\n if np.any(~np.isfinite(velocity)):\n raise ValueError(\"velocity contains non-finite values\")\n\n if precip is not None and np.all(~np.isfinite(precip)):\n raise ValueError(\"precip contains only non-finite values\")\n\n if np.all(~np.isfinite(velocity)):\n raise ValueError(\"velocity contains only non-finite values\")\n\n if isinstance(timesteps, list) and not sorted(timesteps) == timesteps:\n raise ValueError(\"timesteps is not in ascending order\")\n\n # defaults\n verbose = kwargs.get(\"verbose\", False)\n displacement_prev = kwargs.get(\"displacement_prev\", None)\n n_iter = kwargs.get(\"n_iter\", 1)\n return_displacement = kwargs.get(\"return_displacement\", False)\n interp_order = kwargs.get(\"interp_order\", 1)\n map_coordinates_mode = kwargs.get(\"map_coordinates_mode\", \"constant\")\n\n if precip is None and not return_displacement:\n raise ValueError(\"precip is None but return_displacement is False\")\n\n if \"D_prev\" in kwargs.keys():\n warnings.warn(\n \"deprecated argument D_prev is ignored, use displacement_prev instead\",\n )\n\n # if interp_order > 1, apply separate masking to preserve nan and\n # non-precipitation values\n if precip is not None and interp_order > 1:\n minval = np.nanmin(precip)\n mask_min = (precip > minval).astype(float)\n if allow_nonfinite_values:\n mask_finite = np.isfinite(precip)\n precip = precip.copy()\n precip[~mask_finite] = 0.0\n mask_finite = mask_finite.astype(float)\n else:\n mask_finite = np.ones(precip.shape)\n\n prefilter = True if interp_order > 1 else False\n\n if isinstance(timesteps, int):\n timesteps = np.arange(1, timesteps + 1)\n vel_timestep = 1.0\n elif np.any(np.diff(timesteps) <= 0.0):\n raise ValueError(\"the given timestep sequence is not monotonously increasing\")\n\n timestep_diff = np.hstack([[timesteps[0]], np.diff(timesteps)])\n\n if verbose:\n print(\"Computing the advection with the semi-lagrangian scheme.\")\n t0 = time.time()\n\n if precip is not None and outval == \"min\":\n outval = np.nanmin(precip)\n\n if xy_coords is None:\n x_values, y_values = np.meshgrid(\n np.arange(velocity.shape[2]), np.arange(velocity.shape[1]), copy=False\n )\n\n xy_coords = np.stack([x_values, y_values])\n\n def interpolate_motion(displacement, velocity_inc, td):\n coords_warped = xy_coords + displacement\n coords_warped = [coords_warped[1, :, :], coords_warped[0, :, :]]\n\n velocity_inc_x = map_coordinates(\n velocity[0, :, :], coords_warped, mode=\"nearest\", order=1, prefilter=False\n )\n velocity_inc_y = map_coordinates(\n velocity[1, :, :], coords_warped, mode=\"nearest\", order=1, prefilter=False\n )\n\n velocity_inc[0, :, :] = velocity_inc_x\n velocity_inc[1, :, :] = velocity_inc_y\n\n if n_iter > 1:\n velocity_inc /= n_iter\n\n velocity_inc *= td / vel_timestep\n\n precip_extrap = []\n if displacement_prev is None:\n displacement = np.zeros((2, velocity.shape[1], velocity.shape[2]))\n velocity_inc = velocity.copy() * timestep_diff[0] / vel_timestep\n else:\n displacement = displacement_prev.copy()\n velocity_inc = np.empty(velocity.shape)\n interpolate_motion(displacement, velocity_inc, timestep_diff[0])\n\n for ti, td in enumerate(timestep_diff):\n if n_iter > 0:\n for k in range(n_iter):\n interpolate_motion(displacement - velocity_inc / 2.0, velocity_inc, td)\n displacement -= velocity_inc\n interpolate_motion(displacement, velocity_inc, td)\n else:\n if ti > 0 or displacement_prev is not None:\n interpolate_motion(displacement, velocity_inc, td)\n\n displacement -= velocity_inc\n\n coords_warped = xy_coords + displacement\n coords_warped = [coords_warped[1, :, :], coords_warped[0, :, :]]\n\n if precip is not None:\n precip_warped = map_coordinates(\n precip,\n coords_warped,\n mode=map_coordinates_mode,\n cval=outval,\n order=interp_order,\n prefilter=prefilter,\n )\n\n if interp_order > 1:\n mask_warped = map_coordinates(\n mask_min,\n coords_warped,\n mode=map_coordinates_mode,\n cval=0,\n order=1,\n prefilter=False,\n )\n precip_warped[mask_warped < 0.5] = minval\n\n mask_warped = map_coordinates(\n mask_finite,\n coords_warped,\n mode=map_coordinates_mode,\n cval=0,\n order=1,\n prefilter=False,\n )\n precip_warped[mask_warped < 0.5] = np.nan\n\n precip_extrap.append(np.reshape(precip_warped, precip.shape))\n\n if verbose:\n print(\"--- %s seconds ---\" % (time.time() - t0))\n\n if precip is not None:\n if not return_displacement:\n return np.stack(precip_extrap)\n else:\n return np.stack(precip_extrap), displacement\n else:\n return None, displacement\n" }, { "path": "pysteps/feature/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implementations of feature detection methods.\"\"\"\n\nfrom pysteps.feature.interface import get_method\n" }, { "path": "pysteps/feature/blob.py", "content": "\"\"\"\npysteps.feature.blob\n====================\n\nBlob detection methods.\n\n.. autosummary::\n :toctree: ../generated/\n\n detection\n\"\"\"\n\nimport numpy as np\n\nfrom pysteps.exceptions import MissingOptionalDependency\n\nfrom scipy.ndimage import gaussian_laplace\n\ntry:\n from skimage import feature\n\n SKIMAGE_IMPORTED = True\nexcept ImportError:\n SKIMAGE_IMPORTED = False\n\n\ndef detection(\n input_image,\n max_num_features=None,\n method=\"log\",\n threshold=0.5,\n min_sigma=3,\n max_sigma=20,\n overlap=0.5,\n return_sigmas=False,\n **kwargs,\n):\n \"\"\"\n .. _`feature.blob_*`:\\\n https://scikit-image.org/docs/dev/auto_examples/features_detection/plot_blob.html\n\n Interface to the `feature.blob_*`_ methods implemented in scikit-image. A\n blob is defined as a scale-space maximum of a Gaussian-filtered image.\n\n .. _ndarray:\\\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\n\n Parameters\n ----------\n input_image: array_like\n Array of shape (m, n) containing the input image. Nan values are ignored.\n max_num_features : int, optional\n The maximum number of blobs to detect. Set to None for no restriction.\n If specified, the most significant blobs are chosen based on their\n intensities in the corresponding Laplacian of Gaussian (LoG)-filtered\n images.\n method: {'log', 'dog', 'doh'}, optional\n The method to use: 'log' = Laplacian of Gaussian, 'dog' = Difference of\n Gaussian, 'doh' = Determinant of Hessian.\n threshold: float, optional\n Detection threshold.\n min_sigma: float, optional\n The minimum standard deviation for the Gaussian kernel.\n max_sigma: float, optional\n The maximum standard deviation for the Gaussian kernel.\n overlap: float, optional\n A value between 0 and 1. If the area of two blobs overlaps by a fraction\n greater than the value for overlap, the smaller blob is eliminated.\n return_sigmas: bool, optional\n If True, return the standard deviations of the Gaussian kernels\n corresponding to the detected blobs.\n\n Returns\n -------\n points: ndarray_\n Array of shape (p, 2) or (p, 3) indicating the pixel coordinates of *p*\n detected blobs. If return_sigmas is True, the third column contains\n the standard deviations of the Gaussian kernels corresponding to the\n blobs.\n \"\"\"\n if method not in [\"log\", \"dog\", \"doh\"]:\n raise ValueError(\"unknown method %s, must be 'log', 'dog' or 'doh'\" % method)\n\n if not SKIMAGE_IMPORTED:\n raise MissingOptionalDependency(\n \"skimage is required for the blob_detection routine but it is not installed\"\n )\n\n if method == \"log\":\n detector = feature.blob_log\n elif method == \"dog\":\n detector = feature.blob_dog\n else:\n detector = feature.blob_doh\n\n blobs = detector(\n input_image,\n min_sigma=min_sigma,\n max_sigma=max_sigma,\n threshold=threshold,\n overlap=overlap,\n **kwargs,\n )\n\n if max_num_features is not None and blobs.shape[0] > max_num_features:\n blob_intensities = []\n for i in range(blobs.shape[0]):\n gl_image = -gaussian_laplace(input_image, blobs[i, 2]) * blobs[i, 2] ** 2\n blob_intensities.append(gl_image[int(blobs[i, 0]), int(blobs[i, 1])])\n idx = np.argsort(blob_intensities)[::-1]\n blobs = blobs[idx[:max_num_features], :]\n\n if not return_sigmas:\n return np.column_stack([blobs[:, 1], blobs[:, 0]])\n else:\n return np.column_stack([blobs[:, 1], blobs[:, 0], blobs[:, 2]])\n" }, { "path": "pysteps/feature/interface.py", "content": "\"\"\"\npysteps.feature.interface\n=========================\n\nInterface for the feature detection module. It returns a callable function for\ndetecting features from two-dimensional images.\n\nThe feature detectors implement the following interface:\n\n ``detection(input_image, **keywords)``\n\nThe input is a two-dimensional image. Additional arguments to the specific\nmethod can be given via ``**keywords``. The output is an array of shape (n, m),\nwhere each row corresponds to one of the n features. The first two columns\ncontain the coordinates (x, y) of the features, and additional information can\nbe specified in the remaining columns.\n\nAll implemented methods support the following keyword arguments:\n\n+------------------+-----------------------------------------------------+\n| Key | Value |\n+==================+=====================================================+\n| max_num_features | maximum number of features to detect |\n+------------------+-----------------------------------------------------+\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nfrom pysteps.feature import blob\nfrom pysteps.feature import tstorm\nfrom pysteps.feature import shitomasi\n\n_detection_methods = dict()\n_detection_methods[\"blob\"] = blob.detection\n_detection_methods[\"tstorm\"] = tstorm.detection\n_detection_methods[\"shitomasi\"] = shitomasi.detection\n\n\ndef get_method(name):\n \"\"\"\n Return a callable function for feature detection.\n\n Implemented methods:\n\n +-----------------+-------------------------------------------------------+\n | Name | Description |\n +=================+=======================================================+\n | blob | blob detection in scale space |\n +-----------------+-------------------------------------------------------+\n | tstorm | Thunderstorm cell detection |\n +-----------------+-------------------------------------------------------+\n | shitomasi | Shi-Tomasi corner detection |\n +-----------------+-------------------------------------------------------+\n \"\"\"\n if isinstance(name, str):\n name = name.lower()\n else:\n raise TypeError(\n \"Only strings supported for the method's names.\\n\"\n + \"Available names:\"\n + str(list(_detection_methods.keys()))\n ) from None\n\n try:\n return _detection_methods[name]\n except KeyError:\n raise ValueError(\n \"Unknown detection method {}\\n\".format(name)\n + \"The available methods are:\"\n + str(list(_detection_methods.keys()))\n ) from None\n" }, { "path": "pysteps/feature/shitomasi.py", "content": "\"\"\"\npysteps.feature.shitomasi\n=========================\n\nShi-Tomasi features detection method to detect corners in an image.\n\n.. autosummary::\n :toctree: ../generated/\n\n detection\n\"\"\"\n\nimport numpy as np\nfrom numpy.ma.core import MaskedArray\n\nfrom pysteps.exceptions import MissingOptionalDependency\n\ntry:\n import cv2\n\n CV2_IMPORTED = True\nexcept ImportError:\n CV2_IMPORTED = False\n\n\ndef detection(\n input_image,\n max_corners=1000,\n max_num_features=None,\n quality_level=0.01,\n min_distance=10,\n block_size=5,\n buffer_mask=5,\n use_harris=False,\n k=0.04,\n verbose=False,\n **kwargs,\n):\n \"\"\"\n .. _`Shi-Tomasi`:\\\n https://docs.opencv.org/3.4.1/dd/d1a/group__imgproc__feature.html#ga1d6bb77486c8f92d79c8793ad995d541\n\n Interface to the OpenCV `Shi-Tomasi`_ features detection method to detect\n corners in an image.\n\n Corners are used for local tracking methods.\n\n .. _MaskedArray:\\\n https://docs.scipy.org/doc/numpy/reference/maskedarray.baseclass.html#numpy.ma.MaskedArray\n\n .. _ndarray:\\\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\n\n .. _`Harris detector`:\\\n https://docs.opencv.org/3.4.1/dd/d1a/group__imgproc__feature.html#gac1fc3598018010880e370e2f709b4345\n\n .. _cornerMinEigenVal:\\\n https://docs.opencv.org/3.4.1/dd/d1a/group__imgproc__feature.html#ga3dbce297c1feb859ee36707e1003e0a8\n\n Parameters\n ----------\n input_image: ndarray_ or MaskedArray_\n Array of shape (m, n) containing the input image.\n\n In case of ndarray_, invalid values (Nans or infs) are masked,\n otherwise the mask of the MaskedArray_ is used. Such mask defines a\n region where features are not detected.\n\n The fill value for the masked pixels is taken as the minimum of all\n valid pixels.\n max_corners: int, optional\n The ``maxCorners`` parameter in the `Shi-Tomasi`_ corner detection\n method.\n It represents the maximum number of points to be tracked (corners).\n If set to zero, all detected corners are used.\n max_num_features: int, optional\n If specified, this argument is substituted for max_corners. Set to None\n for no restriction. Added for compatibility with the feature detector\n interface.\n quality_level: float, optional\n The ``qualityLevel`` parameter in the `Shi-Tomasi`_ corner detection\n method.\n It represents the minimal accepted quality for the image corners.\n min_distance: int, optional\n The ``minDistance`` parameter in the `Shi-Tomasi`_ corner detection\n method.\n It represents minimum possible Euclidean distance in pixels between\n corners.\n block_size: int, optional\n The ``blockSize`` parameter in the `Shi-Tomasi`_ corner detection\n method.\n It represents the window size in pixels used for computing a derivative\n covariation matrix over each pixel neighbourhood.\n use_harris: bool, optional\n Whether to use a `Harris detector`_ or cornerMinEigenVal_.\n k: float, optional\n Free parameter of the Harris detector.\n buffer_mask: int, optional\n A mask buffer width in pixels. This extends the input mask (if any)\n to limit edge effects.\n verbose: bool, optional\n Print the number of features detected.\n\n Returns\n -------\n points: ndarray_\n Array of shape (p, 2) indicating the pixel coordinates of *p* detected\n corners.\n\n References\n ----------\n Jianbo Shi and Carlo Tomasi. Good features to track. In Computer Vision and\n Pattern Recognition, 1994. Proceedings CVPR'94., 1994 IEEE Computer Society\n Conference on, pages 593–600. IEEE, 1994.\n \"\"\"\n if not CV2_IMPORTED:\n raise MissingOptionalDependency(\n \"opencv package is required for the goodFeaturesToTrack() \"\n \"routine but it is not installed\"\n )\n\n input_image = input_image.copy()\n\n if input_image.ndim != 2:\n raise ValueError(\"input_image must be a two-dimensional array\")\n\n # Check if a MaskedArray is used. If not, mask the ndarray\n if not isinstance(input_image, MaskedArray):\n input_image = np.ma.masked_invalid(input_image)\n\n np.ma.set_fill_value(input_image, input_image.min())\n\n # buffer the quality mask to ensure that no vectors are computed nearby\n # the edges of the radar mask\n mask = np.ma.getmaskarray(input_image).astype(\"uint8\")\n if buffer_mask > 0:\n mask = cv2.dilate(\n mask, np.ones((int(buffer_mask), int(buffer_mask)), np.uint8), 1\n )\n input_image[mask] = np.ma.masked\n\n # scale image between 0 and 255\n im_min = input_image.min()\n im_max = input_image.max()\n if im_max - im_min > 1e-8:\n input_image = (input_image.filled() - im_min) / (im_max - im_min) * 255\n else:\n input_image = input_image.filled() - im_min\n\n # convert to 8-bit\n input_image = np.ndarray.astype(input_image, \"uint8\")\n mask = ~mask & 1\n\n params = dict(\n maxCorners=max_num_features if max_num_features is not None else max_corners,\n qualityLevel=quality_level,\n minDistance=min_distance,\n blockSize=block_size,\n useHarrisDetector=use_harris,\n k=k,\n )\n points = cv2.goodFeaturesToTrack(input_image, mask=mask, **params)\n if points is None:\n points = np.empty(shape=(0, 2))\n else:\n points = points[:, 0, :]\n\n if verbose:\n print(f\"--- {points.shape[0]} good features to track detected ---\")\n\n return points\n" }, { "path": "pysteps/feature/tstorm.py", "content": "\"\"\"\npysteps.feature.tstorm\n======================\n\nThunderstorm cell detection module, part of Thunderstorm Detection and Tracking (DATing)\nThis module was implemented following the procedures used in the TRT Thunderstorms\nRadar Tracking algorithm (:cite:`TRT2004`) used operationally at MeteoSwiss.\nFull documentation is published in :cite:`Feldmann2021`.\nModifications include advecting the identified thunderstorms with the optical flow\nobtained from pysteps, as well as additional options in the thresholding.\n\nReferences\n...............\n:cite:`TRT2004`\n:cite:`Feldmann2021`\n\n@author: mfeldman\n\n.. autosummary::\n :toctree: ../generated/\n\n detection\n breakup\n longdistance\n get_profile\n\"\"\"\n\nimport numpy as np\nimport scipy.ndimage as ndi\n\nfrom pysteps.exceptions import MissingOptionalDependency\n\ntry:\n import skimage\n\n SKIMAGE_IMPORTED = True\nexcept ImportError:\n SKIMAGE_IMPORTED = False\nif SKIMAGE_IMPORTED:\n import skimage.measure as skime\n import skimage.morphology as skim\n import skimage.segmentation as skis\ntry:\n import pandas as pd\n\n PANDAS_IMPORTED = True\nexcept ImportError:\n PANDAS_IMPORTED = False\n\n\ndef detection(\n input_image,\n max_num_features=None,\n minref=35,\n maxref=48,\n mindiff=6,\n minsize=50,\n minmax=41,\n mindis=10,\n output_feat=False,\n output_splits_merges=False,\n time=\"000000000\",\n):\n \"\"\"\n This function detects thunderstorms using a multi-threshold approach. It is\n recommended to use a 2-D Cartesian maximum reflectivity composite, however the\n function will process any 2-D array.\n The thunderstorm cell detection requires both scikit-image and pandas.\n\n Parameters\n ----------\n input_image: array-like\n Array of shape (m,n) containing input image, usually maximum reflectivity in\n dBZ with a resolution of 1 km. Nan values are ignored.\n max_num_features : int, optional\n The maximum number of cells to detect. Set to None for no restriction.\n If specified, the most significant cells are chosen based on their area.\n minref: float, optional\n Lower threshold for object detection. Lower values will be set to NaN.\n The default is 35 dBZ.\n maxref: float, optional\n Upper threshold for object detection. Higher values will be set to this value.\n The default is 48 dBZ.\n mindiff: float, optional\n Minimal difference between two identified maxima within same area to split area\n into two objects. The default is 6 dBZ.\n minsize: float, optional\n Minimal area for possible detected object. The default is 50 pixels.\n minmax: float, optional\n Minimum value of maximum in identified objects. Objects with a maximum lower\n than this will be discarded. The default is 41 dBZ.\n mindis: float, optional\n Minimum distance between two maxima of identified objects. Objects with a\n smaller distance will be merged. The default is 10 km.\n output_feat: bool, optional\n Set to True to return only the cell coordinates.\n output_split_merge: bool, optional\n Set to True to return additional columns in the dataframe for describing the\n splitting and merging of cells. Note that columns are initialized with None,\n and the information needs to be analyzed while tracking.\n time: string, optional\n Date and time as string. Used to label time in the resulting dataframe.\n The default is '000000000'.\n\n Returns\n -------\n cells_id: pandas dataframe\n Pandas dataframe containing all detected cells and their respective properties\n corresponding to the input image.\n Columns of dataframe: ID - cell ID, time - time stamp, x - array of all\n x-coordinates of cell, y - array of all y-coordinates of cell, cen_x -\n x-coordinate of cell centroid, cen_y - y-coordinate of cell centroid, max_ref -\n maximum (reflectivity) value of cell, cont - cell contours\n labels: array-like\n Array of shape (m,n), grid of labelled cells.\n \"\"\"\n if not SKIMAGE_IMPORTED:\n raise MissingOptionalDependency(\n \"skimage is required for thunderstorm DATing \" \"but it is not installed\"\n )\n if not PANDAS_IMPORTED:\n raise MissingOptionalDependency(\n \"pandas is required for thunderstorm DATing \" \"but it is not installed\"\n )\n filt_image = np.zeros(input_image.shape)\n filt_image[input_image >= minref] = input_image[input_image >= minref]\n filt_image[input_image > maxref] = maxref\n max_image = np.zeros(filt_image.shape)\n max_image[filt_image == maxref] = 1\n labels, n_groups = ndi.label(max_image)\n for n in range(1, n_groups + 1):\n indx, indy = np.where(labels == n)\n if len(indx) > 3:\n max_image[indx[0], indy[0]] = 2\n filt_image[max_image == 2] = maxref + 1\n binary = np.zeros(filt_image.shape)\n binary[filt_image > 0] = 1\n labels, n_groups = ndi.label(binary)\n for n in range(1, n_groups + 1):\n ind = np.where(labels == n)\n size = len(ind[0])\n maxval = np.nanmax(input_image[ind])\n if size < minsize: # removing too small areas\n binary[labels == n] = 0\n labels[labels == n] = 0\n if maxval < minmax: # removing areas with too low max value\n binary[labels == n] = 0\n labels[labels == n] = 0\n filt_image = filt_image * binary\n if mindis % 2 == 0:\n elem = mindis - 1\n else:\n elem = mindis\n struct = np.ones([elem, elem])\n if np.nanmax(filt_image.flatten()) < minref:\n maxima = np.zeros(filt_image.shape)\n else:\n maxima = skim.h_maxima(filt_image, h=mindiff, footprint=struct)\n loc_max = np.where(maxima > 0)\n\n loc_max = longdistance(loc_max, mindis)\n i_cell = labels[loc_max]\n n_cell = np.unique(labels)[1:]\n for n in n_cell:\n if n not in i_cell:\n binary[labels == n] = 0\n labels[labels == n] = 0\n\n maxima_dis = np.zeros(maxima.shape)\n maxima_dis[loc_max] = 1\n\n areas, lines = breakup(input_image, np.nanmin(input_image.flatten()), maxima_dis)\n\n cells_id, labels = get_profile(\n areas,\n binary,\n input_image,\n loc_max,\n time,\n minref,\n output_splits_merges=output_splits_merges,\n )\n\n if max_num_features is not None:\n idx = np.argsort(cells_id.area.to_numpy())[::-1]\n\n if not output_feat:\n if max_num_features is None:\n return cells_id, labels\n else:\n for i in idx[max_num_features:]:\n labels[labels == cells_id.ID[i]] = 0\n return cells_id.loc[idx[:max_num_features]], labels\n if output_feat:\n out = np.column_stack([np.array(cells_id.cen_x), np.array(cells_id.cen_y)])\n if max_num_features is not None:\n out = out[idx[:max_num_features], :]\n\n return out\n\n\ndef breakup(ref, minval, maxima):\n \"\"\"\n This function segments the entire 2-D array into areas belonging to each identified\n maximum according to a watershed algorithm.\n \"\"\"\n ref_t = np.zeros(ref.shape)\n ref_t[:] = minval\n ref_t[ref > minval] = ref[ref > minval]\n markers = ndi.label(maxima)[0]\n areas = skis.watershed(-ref_t, markers=markers)\n lines = skis.watershed(-ref_t, markers=markers, watershed_line=True)\n\n return areas, lines\n\n\ndef longdistance(loc_max, mindis):\n \"\"\"\n This function computes the distance between all maxima and rejects maxima that are\n less than a minimum distance apart.\n \"\"\"\n x_max = loc_max[1]\n y_max = loc_max[0]\n n = 0\n while n < len(y_max):\n disx = x_max[n] - x_max\n disy = y_max[n] - y_max\n dis = np.sqrt(disx * disx + disy * disy)\n close = np.where(dis < mindis)[0]\n close = np.delete(close, np.where(close <= n))\n if len(close) > 0:\n x_max = np.delete(x_max, close)\n y_max = np.delete(y_max, close)\n n += 1\n\n new_max = y_max, x_max\n\n return new_max\n\n\ndef get_profile(areas, binary, ref, loc_max, time, minref, output_splits_merges=False):\n \"\"\"\n This function returns the identified cells in a dataframe including their x,y\n locations, location of their maxima, maximum reflectivity and contours.\n Optionally, the dataframe can include columns for storing information regarding\n splitting and merging of cells.\n \"\"\"\n cells = areas * binary\n cell_labels = cells[loc_max]\n labels = np.zeros(cells.shape)\n cells_id = []\n for n, cell_label in enumerate(cell_labels):\n this_id = n + 1\n x = np.where(cells == cell_label)[1]\n y = np.where(cells == cell_label)[0]\n cell_unique = np.zeros(cells.shape)\n cell_unique[cells == cell_label] = 1\n maxref = np.nanmax(ref[y, x])\n contours = skime.find_contours(cell_unique, 0.8)\n cells_id.append(\n {\n \"ID\": this_id,\n \"time\": time,\n \"x\": x,\n \"y\": y,\n \"cen_x\": np.round(np.nanmean(x)).astype(int),\n \"cen_y\": np.round(np.nanmean(y)).astype(int),\n \"max_ref\": maxref,\n \"cont\": contours,\n \"area\": len(x),\n }\n )\n if output_splits_merges:\n cells_id[-1].update(\n {\n \"splitted\": None,\n \"split_IDs\": None,\n \"merged\": None,\n \"merged_IDs\": None,\n \"results_from_split\": None,\n \"will_merge\": None,\n }\n )\n labels[cells == cell_labels[n]] = this_id\n\n columns = [\n \"ID\",\n \"time\",\n \"x\",\n \"y\",\n \"cen_x\",\n \"cen_y\",\n \"max_ref\",\n \"cont\",\n \"area\",\n ]\n if output_splits_merges:\n columns.extend(\n [\n \"splitted\",\n \"split_IDs\",\n \"merged\",\n \"merged_IDs\",\n \"results_from_split\",\n \"will_merge\",\n ]\n )\n cells_id = pd.DataFrame(\n data=cells_id,\n index=range(len(cell_labels)),\n columns=columns,\n )\n if output_splits_merges:\n cells_id[\"split_IDs\"] = cells_id[\"split_IDs\"].astype(\"object\")\n cells_id[\"merged_IDs\"] = cells_id[\"merged_IDs\"].astype(\"object\")\n return cells_id, labels\n" }, { "path": "pysteps/io/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nMethods for browsing data archives, reading 2d precipitation fields and writing\nforecasts into files.\n\"\"\"\n\nfrom .interface import get_method, discover_importers, importers_info\nfrom .archive import *\nfrom .exporters import *\nfrom .importers import *\nfrom .nowcast_importers import *\nfrom .readers import *\n" }, { "path": "pysteps/io/archive.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.io.archive\n==================\n\nUtilities for finding archived files that match the given criteria.\n\n.. autosummary::\n :toctree: ../generated/\n\n find_by_date\n\"\"\"\n\nfrom datetime import datetime, timedelta\nimport fnmatch\nimport os\n\n\ndef find_by_date(\n date,\n root_path,\n path_fmt,\n fn_pattern,\n fn_ext,\n timestep,\n num_prev_files=0,\n num_next_files=0,\n silent=False,\n):\n \"\"\"\n List input files whose timestamp matches the given date.\n\n Parameters\n ----------\n date: datetime.datetime\n The given date.\n root_path: str\n The root path to search the input files.\n path_fmt: str\n Path format. It may consist of directory names separated by '/',\n date/time specifiers beginning with '%' (e.g. %Y/%m/%d) and wildcards\n (?) that match any single character.\n fn_pattern: str\n The name pattern of the input files without extension. The pattern can\n contain time specifiers (e.g. %H, %M and %S).\n fn_ext: str\n Extension of the input files.\n timestep: float\n Time step between consecutive input files (minutes).\n num_prev_files: int\n Optional, number of previous files to find before the given timestamp.\n num_next_files: int\n Optional, number of future files to find after the given timestamp.\n silent: bool\n Optional, whether to suppress all messages from the method.\n\n Returns\n -------\n out: tuple\n If num_prev_files=0 and num_next_files=0, return a pair containing the\n found file name and the corresponding timestamp as a datetime.datetime\n object. Otherwise, return a tuple of two lists, the first one for the\n file names and the second one for the corresponding timestemps. The lists\n are sorted in ascending order with respect to timestamp. A None value is\n assigned if a file name corresponding to a given timestamp is not found.\n\n \"\"\"\n filenames = []\n timestamps = []\n\n for i in range(num_prev_files + num_next_files + 1):\n curdate = (\n date\n + timedelta(minutes=num_next_files * timestep)\n - timedelta(minutes=i * timestep)\n )\n fn = _find_matching_filename(\n curdate, root_path, path_fmt, fn_pattern, fn_ext, silent\n )\n filenames.append(fn)\n\n timestamps.append(curdate)\n\n if all(filename is None for filename in filenames):\n raise IOError(\"no input data found in %s\" % root_path)\n\n if (num_prev_files + num_next_files) > 0:\n return filenames[::-1], timestamps[::-1]\n else:\n return filenames, timestamps\n\n\ndef _find_matching_filename(\n date, root_path, path_fmt, fn_pattern, fn_ext, silent=False\n):\n path = _generate_path(date, root_path, path_fmt)\n\n if os.path.exists(path):\n fn = datetime.strftime(date, fn_pattern) + \".\" + fn_ext\n\n # test for wildcars\n if \"?\" in fn:\n filenames = os.listdir(path)\n if len(filenames) > 0:\n for filename in filenames:\n if fnmatch.fnmatch(filename, fn):\n fn = filename\n break\n\n fn = os.path.join(path, fn)\n\n if os.path.exists(fn):\n return fn\n else:\n if not silent:\n print(f\"file not found: {fn}\")\n return None\n else:\n if not silent:\n print(f\"path not found: {path}\")\n return None\n\n\ndef _generate_path(date, root_path, path_format):\n \"\"\"Generate file path.\"\"\"\n if not isinstance(date, datetime):\n raise TypeError(\"The input 'date' argument must be a datetime object\")\n\n if path_format != \"\":\n sub_path = date.strftime(path_format)\n return os.path.join(root_path, sub_path)\n else:\n return root_path\n" }, { "path": "pysteps/io/exporters.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.io.exporters\n====================\n\nMethods for exporting forecasts of 2d precipitation fields into various file\nformats.\n\nEach exporter method in this module has its own initialization function that\nimplements the following interface::\n\n initialize_forecast_exporter_xxx(outpath, outfnprefix, startdate, timestep,\n n_timesteps, shape, metadata,\n n_ens_members=1,\n incremental=None, **kwargs)\n\nwhere xxx specifies the file format.\n\nThis function creates the output files and writes the metadata. See the\ndocumentation of the initialization methods for the format of the output files\nand their names. The datasets are written by calling\n:py:func:`pysteps.io.exporters.export_forecast_dataset`, and the files are\nclosed by calling :py:func:`pysteps.io.exporters.close_forecast_files`.\n\nThe arguments of initialize_forecast_exporter_xxx are described in the\nfollowing table:\n\n.. tabularcolumns:: |p{2cm}|p{2cm}|L|\n\n+---------------+-------------------+-----------------------------------------+\n| Argument | Type/values | Description |\n+===============+===================+=========================================+\n| outpath | str | output path |\n+---------------+-------------------+-----------------------------------------+\n| outfnprefix | str | prefix of output file names |\n+---------------+-------------------+-----------------------------------------+\n| startdate | datetime.datetime | start date of the forecast |\n+---------------+-------------------+-----------------------------------------+\n| timestep | int | length of the forecast time step |\n| | | (minutes) |\n+---------------+-------------------+-----------------------------------------+\n| n_timesteps | int | number of time steps in the forecast |\n| | | this argument is ignored if |\n| | | incremental is set to 'timestep'. |\n+---------------+-------------------+-----------------------------------------+\n| shape | tuple | two-element tuple defining the shape |\n| | | (height,width) of the forecast grids |\n+---------------+-------------------+-----------------------------------------+\n| metadata | dict | metadata dictionary containing the |\n| | | projection,x1,x2,y1,y2 and unit |\n| | | attributes described in the |\n| | | documentation of pysteps.io.importers |\n+---------------+-------------------+-----------------------------------------+\n| n_ens_members | int | number of ensemble members in the |\n| | | forecast |\n| | | this argument is ignored if incremental |\n| | | is set to 'member' |\n+---------------+-------------------+-----------------------------------------+\n| incremental | {None, 'timestep',| allow incremental writing of datasets |\n| | 'member'} | the available options are: |\n| | | 'timestep' = write a forecast or a |\n| | | forecast ensemble for a given |\n| | | time step |\n| | | 'member' = write a forecast sequence |\n| | | for a given ensemble member |\n+---------------+-------------------+-----------------------------------------+\n\nOptional exporter-specific arguments are passed with ``kwargs``.\nThe return value is a dictionary containing an exporter object.\nThis can be used with :py:func:`pysteps.io.exporters.export_forecast_dataset`\nto write the datasets to the output files.\n\nAvailable Exporters\n-------------------\n\n.. autosummary::\n :toctree: ../generated/\n\n initialize_forecast_exporter_geotiff\n initialize_forecast_exporter_kineros\n initialize_forecast_exporter_netcdf\n\nGeneric functions\n-----------------\n\n.. autosummary::\n :toctree: ../generated/\n\n export_forecast_dataset\n close_forecast_files\n\"\"\"\n\nimport os\nfrom datetime import datetime\n\nimport numpy as np\n\nfrom pysteps.exceptions import MissingOptionalDependency\n\ntry:\n from osgeo import gdal, osr\n\n # Preserve current behavior explicitly (no GDAL exceptions) and avoid the\n # GDAL 4.0 future-warning emitted when neither mode is selected.\n if hasattr(gdal, \"DontUseExceptions\"):\n gdal.DontUseExceptions()\n GDAL_IMPORTED = True\nexcept ImportError:\n GDAL_IMPORTED = False\ntry:\n import netCDF4\n\n NETCDF4_IMPORTED = True\nexcept ImportError:\n NETCDF4_IMPORTED = False\ntry:\n import pyproj\n\n PYPROJ_IMPORTED = True\nexcept ImportError:\n PYPROJ_IMPORTED = False\n\n\ndef initialize_forecast_exporter_geotiff(\n outpath,\n outfnprefix,\n startdate,\n timestep,\n n_timesteps,\n shape,\n metadata,\n n_ens_members=1,\n incremental=None,\n **kwargs,\n):\n \"\"\"\n Initialize a GeoTIFF forecast exporter.\n\n The output files are named as '__.tif', where\n startdate is in YYmmddHHMM format and t is lead time (minutes). GDAL needs\n to be installed to use this exporter.\n\n Parameters\n ----------\n outpath: str\n Output path.\n\n outfnprefix: str\n Prefix for output file names.\n\n startdate: datetime.datetime\n Start date of the forecast.\n\n timestep: int\n Time step of the forecast (minutes).\n\n n_timesteps: int\n Number of time steps in the forecast. This argument is ignored if\n incremental is set to 'timestep'.\n\n shape: tuple of int\n Two-element tuple defining the shape (height,width) of the forecast\n grids.\n\n metadata: dict\n Metadata dictionary containing the projection,x1,x2,y1,y2 and unit\n attributes described in the documentation of\n :py:mod:`pysteps.io.importers`.\n\n n_ens_members: int\n Number of ensemble members in the forecast.\n\n incremental: {None,'timestep'}, optional\n Allow incremental writing of datasets into the GeoTIFF files. Set to\n 'timestep' to enable writing forecasts or forecast ensembles separately\n for each time step. If set to None, incremental writing is disabled and\n the whole forecast is written in a single function call. The 'member'\n option is not currently implemented.\n\n Returns\n -------\n exporter: dict\n The return value is a dictionary containing an exporter object.\n This can be used with\n :py:func:`pysteps.io.exporters.export_forecast_dataset`\n to write the datasets.\n\n \"\"\"\n\n if len(shape) != 2:\n raise ValueError(\"shape has %d elements, 2 expected\" % len(shape))\n\n del kwargs # kwargs not used\n\n if not GDAL_IMPORTED:\n raise MissingOptionalDependency(\n \"gdal package is required for GeoTIFF \" \"exporters but it is not installed\"\n )\n\n if incremental == \"member\":\n raise ValueError(\n \"incremental writing of GeoTIFF files with\"\n + \" the 'member' option is not supported\"\n )\n\n exporter = dict(\n method=\"geotiff\",\n outfnprefix=outfnprefix,\n startdate=startdate,\n timestep=timestep,\n num_timesteps=n_timesteps,\n shape=shape,\n metadata=metadata,\n num_ens_members=n_ens_members,\n incremental=incremental,\n dst=[],\n )\n driver = gdal.GetDriverByName(\"GTiff\")\n exporter[\"driver\"] = driver\n\n if incremental != \"timestep\":\n for i in range(n_timesteps):\n outfn = _get_geotiff_filename(\n outfnprefix, startdate, n_timesteps, timestep, i\n )\n outfn = os.path.join(outpath, outfn)\n dst = _create_geotiff_file(outfn, driver, shape, metadata, n_ens_members)\n exporter[\"dst\"].append(dst)\n else:\n exporter[\"num_files_written\"] = 0\n\n return exporter\n\n\n# TODO(exporters): This is a draft version of the kineros exporter.\n# Revise the variable names and\n# the structure of the file if necessary.\n\n\ndef initialize_forecast_exporter_kineros(\n outpath,\n outfnprefix,\n startdate,\n timestep,\n n_timesteps,\n shape,\n metadata,\n n_ens_members=1,\n incremental=None,\n **kwargs,\n):\n \"\"\"\n Initialize a KINEROS2 format exporter for the rainfall \".pre\" files\n specified in https://www.tucson.ars.ag.gov/kineros/.\n\n Grid points are treated as individual rain gauges and a separate file is\n produced for each ensemble member. The output files are named as\n _N.pre, where is the index of ensemble member starting\n from zero.\n\n Parameters\n ----------\n outpath: str\n Output path.\n\n outfnprefix: str\n Prefix for output file names.\n\n startdate: datetime.datetime\n Start date of the forecast.\n\n timestep: int\n Time step of the forecast (minutes).\n\n n_timesteps: int\n Number of time steps in the forecast this argument is ignored if\n incremental is set to 'timestep'.\n\n shape: tuple of int\n Two-element tuple defining the shape (height,width) of the forecast\n grids.\n\n metadata: dict\n Metadata dictionary containing the projection,x1,x2,y1,y2 and unit\n attributes described in the documentation of\n :py:mod:`pysteps.io.importers`.\n\n n_ens_members: int\n Number of ensemble members in the forecast. This argument is ignored if\n incremental is set to 'member'.\n\n incremental: {None}, optional\n Currently not implemented for this method.\n\n Returns\n -------\n exporter: dict\n The return value is a dictionary containing an exporter object. This c\n an be used with :py:func:`pysteps.io.exporters.export_forecast_dataset`\n to write datasets into the given file format.\n\n \"\"\"\n\n if incremental is not None:\n raise ValueError(\n \"unknown option %s: \" + \"incremental writing is not supported\" % incremental\n )\n\n exporter = {}\n\n # one file for each member\n n_ens_members = np.min((99, n_ens_members))\n fns = []\n for i in range(n_ens_members):\n outfn = \"%s_N%02d%s\" % (outfnprefix, i, \".pre\")\n outfn = os.path.join(outpath, outfn)\n with open(outfn, \"w\") as fd:\n # write header\n fd.writelines(\"! pysteps-generated nowcast.\\n\")\n fd.writelines(\"! created the %s.\\n\" % datetime.now().strftime(\"%c\"))\n # TODO(exporters): Add pySTEPS version here\n fd.writelines(\"! Member = %02d.\\n\" % i)\n fd.writelines(\"! Startdate = %s.\\n\" % startdate.strftime(\"%c\"))\n fns.append(outfn)\n fd.close()\n\n h, w = shape\n\n if metadata[\"unit\"] == \"mm/h\":\n var_name = \"Intensity\"\n var_long_name = \"Intensity in mm/hr\"\n var_unit = \"mm/hr\"\n elif metadata[\"unit\"] == \"mm\":\n var_name = \"Depth\"\n var_long_name = \"Accumulated depth in mm\"\n var_unit = \"mm\"\n else:\n raise ValueError(\"unsupported unit %s\" % metadata[\"unit\"])\n\n xr = np.linspace(metadata[\"x1\"], metadata[\"x2\"], w + 1)[:-1]\n xr += 0.5 * (xr[1] - xr[0])\n yr = np.linspace(metadata[\"y1\"], metadata[\"y2\"], h + 1)[:-1]\n yr += 0.5 * (yr[1] - yr[0])\n\n xy_coords = np.stack(np.meshgrid(xr, yr))\n\n exporter[\"method\"] = \"kineros\"\n exporter[\"ncfile\"] = fns\n exporter[\"XY_coords\"] = xy_coords\n exporter[\"var_name\"] = var_name\n exporter[\"var_long_name\"] = var_long_name\n exporter[\"var_unit\"] = var_unit\n exporter[\"startdate\"] = startdate\n exporter[\"timestep\"] = timestep\n exporter[\"metadata\"] = metadata\n exporter[\"incremental\"] = incremental\n exporter[\"num_timesteps\"] = n_timesteps\n exporter[\"num_ens_members\"] = n_ens_members\n exporter[\"shape\"] = shape\n\n return exporter\n\n\n# TODO(exporters): This is a draft version of the netcdf exporter.\n# Revise the variable names and\n# the structure of the file if necessary.\n\n\ndef initialize_forecast_exporter_netcdf(\n outpath,\n outfnprefix,\n startdate,\n timestep,\n n_timesteps,\n shape,\n metadata,\n n_ens_members=1,\n datatype=np.float32,\n incremental=None,\n fill_value=None,\n scale_factor=None,\n offset=None,\n **kwargs,\n):\n \"\"\"\n Initialize a netCDF forecast exporter. All outputs are written to a\n single file named as '_.nc'.\n\n Parameters\n ----------\n outpath: str\n Output path.\n outfnprefix: str\n Prefix for output file names.\n startdate: datetime.datetime\n Start date of the forecast.\n timestep: int\n Time step of the forecast (minutes).\n n_timesteps: int or list of integers\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed (relative to the input time step). The elements of\n the list are required to be in ascending order.\n shape: tuple of int\n Two-element tuple defining the shape (height,width) of the forecast\n grids.\n metadata: dict\n Metadata dictionary containing the projection, x1, x2, y1, y2,\n unit attributes (projection and variable units) described in the\n documentation of :py:mod:`pysteps.io.importers`.\n n_ens_members: int\n Number of ensemble members in the forecast. This argument is ignored if\n incremental is set to 'member'.\n datatype: np.dtype, optional\n The datatype of the output values. Defaults to np.float32.\n incremental: {None,'timestep','member'}, optional\n Allow incremental writing of datasets into the netCDF files.\\n\n The available options are: 'timestep' = write a forecast or a forecast\n ensemble for a given time step; 'member' = write a forecast sequence\n for a given ensemble member. If set to None, incremental writing is\n disabled.\n fill_value: int, optional\n Fill_value for missing data. Defaults to None, which means that the\n standard netCDF4 fill_value is used.\n scale_factor: float, optional\n The scale factor to scale the data as: store_value = scale_factor *\n precipitation_value + offset. Defaults to None. The scale_factor\n can be used to reduce data storage.\n offset: float, optional\n The offset to offset the data as: store_value = scale_factor *\n precipitation_value + offset. Defaults to None.\n\n Other Parameters\n ----------------\n institution: str\n The instute, company or community that has created the nowcast.\n Default: the pySTEPS community (https://pysteps.github.io)\n references: str\n Any references to be included in the netCDF file. Defaults to \" \".\n comment: str\n Any comments about the data or storage protocol that should be\n included in the netCDF file. Defaults to \" \".\n\n Returns\n -------\n exporter: dict\n The return value is a dictionary containing an exporter object. This c\n an be used with :py:func:`pysteps.io.exporters.export_forecast_dataset`\n to write datasets into the given file format.\n \"\"\"\n\n if not NETCDF4_IMPORTED:\n raise MissingOptionalDependency(\n \"netCDF4 package is required for netcdf \"\n \"exporters but it is not installed\"\n )\n\n if not PYPROJ_IMPORTED:\n raise MissingOptionalDependency(\n \"pyproj package is required for netcdf \" \"exporters but it is not installed\"\n )\n\n if incremental not in [None, \"timestep\", \"member\"]:\n raise ValueError(\n f\"unknown option {incremental}: incremental must be \"\n + \"'timestep' or 'member'\"\n )\n\n n_timesteps_is_list = isinstance(n_timesteps, list)\n if n_timesteps_is_list:\n num_timesteps = len(n_timesteps)\n else:\n num_timesteps = n_timesteps\n\n if incremental == \"timestep\":\n num_timesteps = None\n elif incremental == \"member\":\n n_ens_members = None\n elif incremental is not None:\n raise ValueError(\n f\"unknown argument value incremental='{str(incremental)}': \"\n + \"must be 'timestep' or 'member'\"\n )\n\n n_ens_gt_one = False\n if n_ens_members is not None:\n if n_ens_members > 1:\n n_ens_gt_one = True\n\n # Kwargs to be used as description strings in the netCDF\n institution = kwargs.get(\n \"institution\", \"the pySTEPS community (https://pysteps.github.io)\"\n )\n references = kwargs.get(\"references\", \"\")\n comment = kwargs.get(\"comment\", \"\")\n\n exporter = {}\n\n outfn = os.path.join(outpath, outfnprefix + \".nc\")\n ncf = netCDF4.Dataset(outfn, \"w\", format=\"NETCDF4\")\n\n ncf.Conventions = \"CF-1.7\"\n ncf.title = \"pysteps-generated nowcast\"\n ncf.institution = institution\n ncf.source = \"pysteps\" # TODO(exporters): Add pySTEPS version here\n ncf.history = \"\"\n ncf.references = references\n ncf.comment = comment\n\n h, w = shape\n\n ncf.createDimension(\"ens_number\", size=n_ens_members)\n ncf.createDimension(\"time\", size=num_timesteps)\n ncf.createDimension(\"y\", size=h)\n ncf.createDimension(\"x\", size=w)\n\n if metadata[\"unit\"] == \"mm/h\":\n var_name = \"precip_intensity\"\n var_standard_name = None\n var_long_name = \"instantaneous precipitation rate\"\n var_unit = \"mm h-1\"\n elif metadata[\"unit\"] == \"mm\":\n var_name = \"precip_accum\"\n var_standard_name = None\n var_long_name = \"accumulated precipitation\"\n var_unit = \"mm\"\n elif metadata[\"unit\"] == \"dBZ\":\n var_name = \"reflectivity\"\n var_long_name = \"equivalent reflectivity factor\"\n var_standard_name = \"equivalent_reflectivity_factor\"\n var_unit = \"dBZ\"\n else:\n raise ValueError(\"unknown unit %s\" % metadata[\"unit\"])\n\n xr = np.linspace(metadata[\"x1\"], metadata[\"x2\"], w + 1)[:-1]\n xr += 0.5 * (xr[1] - xr[0])\n yr = np.linspace(metadata[\"y1\"], metadata[\"y2\"], h + 1)[:-1]\n yr += 0.5 * (yr[1] - yr[0])\n\n # flip yr vector if yorigin is upper\n if metadata[\"yorigin\"] == \"upper\":\n yr = np.flip(yr)\n\n var_xc = ncf.createVariable(\"x\", np.float32, dimensions=(\"x\",))\n var_xc[:] = xr\n var_xc.axis = \"X\"\n var_xc.standard_name = \"projection_x_coordinate\"\n var_xc.long_name = \"x-coordinate in Cartesian system\"\n var_xc.units = metadata[\"cartesian_unit\"]\n\n var_yc = ncf.createVariable(\"y\", np.float32, dimensions=(\"y\",))\n var_yc[:] = yr\n var_yc.axis = \"Y\"\n var_yc.standard_name = \"projection_y_coordinate\"\n var_yc.long_name = \"y-coordinate in Cartesian system\"\n var_yc.units = metadata[\"cartesian_unit\"]\n\n x_2d, y_2d = np.meshgrid(xr, yr)\n pr = pyproj.Proj(metadata[\"projection\"])\n lon, lat = pr(x_2d.flatten(), y_2d.flatten(), inverse=True)\n\n var_lon = ncf.createVariable(\"lon\", float, dimensions=(\"y\", \"x\"))\n var_lon[:] = lon.reshape(shape)\n var_lon.standard_name = \"longitude\"\n var_lon.long_name = \"longitude coordinate\"\n # TODO(exporters): Don't hard-code the unit.\n var_lon.units = \"degrees_east\"\n\n var_lat = ncf.createVariable(\"lat\", float, dimensions=(\"y\", \"x\"))\n var_lat[:] = lat.reshape(shape)\n var_lat.standard_name = \"latitude\"\n var_lat.long_name = \"latitude coordinate\"\n # TODO(exporters): Don't hard-code the unit.\n var_lat.units = \"degrees_north\"\n\n ncf.projection = metadata[\"projection\"]\n\n (\n grid_mapping_var_name,\n grid_mapping_name,\n grid_mapping_params,\n ) = _convert_proj4_to_grid_mapping(metadata[\"projection\"])\n # skip writing the grid mapping if a matching name was not found\n if grid_mapping_var_name is not None:\n var_gm = ncf.createVariable(grid_mapping_var_name, int, dimensions=())\n var_gm.grid_mapping_name = grid_mapping_name\n for i in grid_mapping_params.items():\n var_gm.setncattr(i[0], i[1])\n\n if incremental == \"member\" or n_ens_gt_one:\n var_ens_num = ncf.createVariable(\"ens_number\", int, dimensions=(\"ens_number\",))\n if incremental != \"member\":\n var_ens_num[:] = list(range(1, n_ens_members + 1))\n var_ens_num.long_name = \"ensemble member\"\n var_ens_num.standard_name = \"realization\"\n var_ens_num.units = \"\"\n\n var_time = ncf.createVariable(\"time\", int, dimensions=(\"time\",))\n if incremental != \"timestep\":\n if n_timesteps_is_list:\n var_time[:] = np.array(n_timesteps) * timestep * 60\n else:\n var_time[:] = [i * timestep * 60 for i in range(1, n_timesteps + 1)]\n var_time.long_name = \"forecast time\"\n startdate_str = datetime.strftime(startdate, \"%Y-%m-%d %H:%M:%S\")\n var_time.units = \"seconds since %s\" % startdate_str\n\n if incremental == \"member\" or n_ens_gt_one:\n var_f = ncf.createVariable(\n var_name,\n datatype=datatype,\n dimensions=(\"ens_number\", \"time\", \"y\", \"x\"),\n compression=\"zlib\",\n zlib=True,\n complevel=9,\n fill_value=fill_value,\n )\n else:\n var_f = ncf.createVariable(\n var_name,\n datatype=datatype,\n dimensions=(\"time\", \"y\", \"x\"),\n compression=\"zlib\",\n zlib=True,\n complevel=9,\n fill_value=fill_value,\n )\n\n if var_standard_name is not None:\n var_f.standard_name = var_standard_name\n var_f.long_name = var_long_name\n var_f.coordinates = \"y x\"\n var_f.units = var_unit\n if grid_mapping_var_name is not None:\n var_f.grid_mapping = grid_mapping_var_name\n # Add gain and offset\n if scale_factor is not None:\n var_f.scale_factor = scale_factor\n if offset is not None:\n var_f.add_offset = offset\n\n exporter[\"method\"] = \"netcdf\"\n exporter[\"ncfile\"] = ncf\n exporter[\"var_F\"] = var_f\n if incremental == \"member\" or n_ens_gt_one:\n exporter[\"var_ens_num\"] = var_ens_num\n exporter[\"var_time\"] = var_time\n exporter[\"var_name\"] = var_name\n exporter[\"startdate\"] = startdate\n exporter[\"timestep\"] = timestep\n exporter[\"metadata\"] = metadata\n exporter[\"incremental\"] = incremental\n exporter[\"num_timesteps\"] = num_timesteps\n exporter[\"timesteps\"] = n_timesteps\n exporter[\"num_ens_members\"] = n_ens_members\n exporter[\"shape\"] = shape\n\n return exporter\n\n\ndef export_forecast_dataset(field, exporter):\n \"\"\"Write a forecast array into a file.\n\n If the exporter was initialized with n_ens_members>1, the written dataset\n has dimensions (n_ens_members,num_timesteps,shape[0],shape[1]), where shape\n refers to the shape of the two-dimensional forecast grids. Otherwise, the\n dimensions are (num_timesteps,shape[0],shape[1]). If the exporter was\n initialized with incremental!=None, the array is appended to the existing\n dataset either along the ensemble member or time axis.\n\n Parameters\n ----------\n exporter: dict\n An exporter object created with any initialization method implemented\n in :py:mod:`pysteps.io.exporters`.\n field: array_like\n The array to write. The required shape depends on the choice of the\n 'incremental' parameter the exporter was initialized with:\n\n +-----------------+---------------------------------------------------+\n | incremental | required shape |\n +=================+===================================================+\n | None | (num_ens_members,num_timesteps,shape[0],shape[1]) |\n +-----------------+---------------------------------------------------+\n | 'timestep' | (num_ens_members,shape[0],shape[1]) |\n +-----------------+---------------------------------------------------+\n | 'member' | (num_timesteps,shape[0],shape[1]) |\n +-----------------+---------------------------------------------------+\n\n If the exporter was initialized with num_ens_members=1,\n the num_ens_members dimension is dropped.\n \"\"\"\n\n if exporter[\"method\"] == \"netcdf\" and not NETCDF4_IMPORTED:\n raise MissingOptionalDependency(\n \"netCDF4 package is required for netcdf \"\n \"exporters but it is not installed\"\n )\n\n if exporter[\"incremental\"] is None:\n if exporter[\"num_ens_members\"] > 1:\n shp = (\n exporter[\"num_ens_members\"],\n exporter[\"num_timesteps\"],\n exporter[\"shape\"][0],\n exporter[\"shape\"][1],\n )\n else:\n shp = (\n exporter[\"num_timesteps\"],\n exporter[\"shape\"][0],\n exporter[\"shape\"][1],\n )\n if field.shape != shp:\n raise ValueError(\n \"field has invalid shape: %s != %s\" % (str(field.shape), str(shp))\n )\n elif exporter[\"incremental\"] == \"timestep\":\n if exporter[\"num_ens_members\"] > 1:\n shp = (\n exporter[\"num_ens_members\"],\n exporter[\"shape\"][0],\n exporter[\"shape\"][1],\n )\n else:\n shp = exporter[\"shape\"]\n if field.shape != shp:\n raise ValueError(\n \"field has invalid shape: %s != %s\" % (str(field.shape), str(shp))\n )\n elif exporter[\"incremental\"] == \"member\":\n shp = (exporter[\"num_timesteps\"], exporter[\"shape\"][0], exporter[\"shape\"][1])\n if field.shape != shp:\n raise ValueError(\n \"field has invalid shape: %s != %s\" % (str(field.shape), str(shp))\n )\n\n if exporter[\"method\"] == \"geotiff\":\n _export_geotiff(field, exporter)\n elif exporter[\"method\"] == \"netcdf\":\n _export_netcdf(field, exporter)\n elif exporter[\"method\"] == \"kineros\":\n _export_kineros(field, exporter)\n else:\n raise ValueError(\"unknown exporter method %s\" % exporter[\"method\"])\n\n\ndef close_forecast_files(exporter):\n \"\"\"\n Close the files associated with a forecast exporter.\n\n Finish writing forecasts and close the output files opened by a forecast\n exporter.\n\n Parameters\n ----------\n exporter: dict\n An exporter object created with any initialization method implemented\n in :py:mod:`pysteps.io.exporters`.\n \"\"\"\n\n if exporter[\"method\"] == \"geotiff\":\n pass # NOTE: There is no explicit \"close\" method in GDAL.\n # The files are closed when all objects referencing to the GDAL\n # datasets are deleted (i.e. when the exporter object is deleted).\n if exporter[\"method\"] == \"kineros\":\n pass # no need to close the file\n else:\n exporter[\"ncfile\"].close()\n\n\ndef _export_geotiff(F, exporter):\n def init_band(band):\n band.SetScale(1.0)\n band.SetOffset(0.0)\n band.SetUnitType(exporter[\"metadata\"][\"unit\"])\n\n if exporter[\"incremental\"] is None:\n for i in range(exporter[\"num_timesteps\"]):\n if exporter[\"num_ens_members\"] == 1:\n band = exporter[\"dst\"][i].GetRasterBand(1)\n init_band(band)\n band.WriteArray(F[i, :, :])\n else:\n for j in range(exporter[\"num_ens_members\"]):\n band = exporter[\"dst\"][i].GetRasterBand(j + 1)\n init_band(band)\n band.WriteArray(F[j, i, :, :])\n elif exporter[\"incremental\"] == \"timestep\":\n i = exporter[\"num_files_written\"]\n\n outfn = _get_geotiff_filename(\n exporter[\"outfnprefix\"],\n exporter[\"startdate\"],\n exporter[\"num_timesteps\"],\n exporter[\"timestep\"],\n i,\n )\n dst = _create_geotiff_file(\n outfn,\n exporter[\"driver\"],\n exporter[\"shape\"],\n exporter[\"metadata\"],\n exporter[\"num_ens_members\"],\n )\n\n for j in range(exporter[\"num_ens_members\"]):\n band = dst.GetRasterBand(j + 1)\n init_band(band)\n if exporter[\"num_ens_members\"] > 1:\n band.WriteArray(F[j, :, :])\n else:\n band.WriteArray(F)\n\n exporter[\"num_files_written\"] += 1\n elif exporter[\"incremental\"] == \"member\":\n for i in range(exporter[\"num_timesteps\"]):\n # NOTE: This does not work because the GeoTIFF driver does not\n # support adding bands. An alternative solution needs to be\n # implemented.\n exporter[\"dst\"][i].AddBand(gdal.GDT_Float32)\n band = exporter[\"dst\"][i].GetRasterBand(exporter[\"dst\"][i].RasterCount)\n init_band(band)\n band.WriteArray(F[i, :, :])\n\n\ndef _export_kineros(field, exporter):\n num_timesteps = exporter[\"num_timesteps\"]\n num_ens_members = exporter[\"num_ens_members\"]\n\n timestep = exporter[\"timestep\"]\n xgrid = exporter[\"XY_coords\"][0, :, :].flatten()\n ygrid = exporter[\"XY_coords\"][1, :, :].flatten()\n\n timemin = [(t + 1) * timestep for t in range(num_timesteps)]\n\n if field.ndim == 3:\n field = field.reshape((1,) + field.shape)\n\n for n in range(num_ens_members):\n file_name = exporter[\"ncfile\"][n]\n\n field_tmp = field[n, :, :, :].reshape((num_timesteps, -1))\n\n if exporter[\"var_name\"] == \"Depth\":\n field_tmp = np.cumsum(field_tmp, axis=0)\n\n with open(file_name, \"a\") as fd:\n for m in range(field_tmp.shape[1]):\n fd.writelines(\"BEGIN RG%03d\\n\" % (m + 1))\n fd.writelines(\" X = %.2f, Y = %.2f\\n\" % (xgrid[m], ygrid[m]))\n fd.writelines(\" N = %i\\n\" % num_timesteps)\n fd.writelines(\" TIME %s\\n\" % exporter[\"var_name\"].upper())\n fd.writelines(\"! (min) (%s)\\n\" % exporter[\"var_unit\"])\n for t in range(num_timesteps):\n line_new = \"{:6.1f} {:11.2f}\\n\".format(timemin[t], field_tmp[t, m])\n fd.writelines(line_new)\n fd.writelines(\"END\\n\\n\")\n\n\ndef _export_netcdf(field, exporter):\n var_f = exporter[\"var_F\"]\n\n if exporter[\"incremental\"] is None:\n var_f[:] = field\n elif exporter[\"incremental\"] == \"timestep\":\n if exporter[\"num_ens_members\"] > 1:\n var_f[:, var_f.shape[1], :, :] = field\n else:\n var_f[var_f.shape[0], :, :] = field\n var_time = exporter[\"var_time\"]\n if isinstance(exporter[\"timesteps\"], list):\n var_time[len(var_time) - 1] = (\n exporter[\"timesteps\"][len(var_time) - 1] * exporter[\"timestep\"] * 60\n )\n else:\n var_time[len(var_time) - 1] = len(var_time) * exporter[\"timestep\"] * 60\n else:\n var_f[var_f.shape[0], :, :, :] = field\n var_ens_num = exporter[\"var_ens_num\"]\n var_ens_num[len(var_ens_num) - 1] = len(var_ens_num)\n\n\n# TODO(exporters): Write methods for converting Proj.4 projection definitions\n# into CF grid mapping attributes. Currently this has been implemented for\n# the stereographic projection.\n# The conversions implemented here are take from:\n# https://github.com/cf-convention/cf-convention.github.io/blob/master/wkt-proj-4.md\n\n\ndef _convert_proj4_to_grid_mapping(proj4str):\n tokens = proj4str.split(\"+\")\n\n d = {}\n for t in tokens[1:]:\n t = t.split(\"=\")\n if len(t) > 1:\n d[t[0]] = t[1].strip()\n\n params = {}\n # TODO(exporters): implement more projection types here\n if d[\"proj\"] == \"stere\":\n grid_mapping_var_name = \"polar_stereographic\"\n grid_mapping_name = \"polar_stereographic\"\n v = d[\"lon_0\"] if d[\"lon_0\"][-1] not in [\"E\", \"W\"] else d[\"lon_0\"][:-1]\n params[\"straight_vertical_longitude_from_pole\"] = float(v)\n v = d[\"lat_0\"] if d[\"lat_0\"][-1] not in [\"N\", \"S\"] else d[\"lat_0\"][:-1]\n params[\"latitude_of_projection_origin\"] = float(v)\n if \"lat_ts\" in list(d.keys()):\n params[\"standard_parallel\"] = float(d[\"lat_ts\"])\n elif \"k_0\" in list(d.keys()):\n params[\"scale_factor_at_projection_origin\"] = float(d[\"k_0\"])\n params[\"false_easting\"] = float(d[\"x_0\"])\n params[\"false_northing\"] = float(d[\"y_0\"])\n elif d[\"proj\"] == \"aea\": # Albers Conical Equal Area\n grid_mapping_var_name = \"proj\"\n grid_mapping_name = \"albers_conical_equal_area\"\n params[\"false_easting\"] = float(d[\"x_0\"]) if \"x_0\" in d else float(0)\n params[\"false_northing\"] = float(d[\"y_0\"]) if \"y_0\" in d else float(0)\n v = d[\"lon_0\"] if \"lon_0\" in d else float(0)\n params[\"longitude_of_central_meridian\"] = float(v)\n v = d[\"lat_0\"] if \"lat_0\" in d else float(0)\n params[\"latitude_of_projection_origin\"] = float(v)\n v1 = d[\"lat_1\"] if \"lat_1\" in d else float(0)\n v2 = d[\"lat_2\"] if \"lat_2\" in d else float(0)\n params[\"standard_parallel\"] = (float(v1), float(v2))\n elif d[\"proj\"] == \"lcc\":\n grid_mapping_var_name = \"lcc\"\n grid_mapping_name = \"lambert_conformal_conic\"\n params[\"false_easting\"] = float(d[\"x_0\"]) if \"x_0\" in d else float(0)\n params[\"false_northing\"] = float(d[\"y_0\"]) if \"y_0\" in d else float(0)\n v = d[\"lon_0\"] if \"lon_0\" in d else float(0)\n params[\"longitude_of_central_meridian\"] = float(v)\n v = d[\"lat_0\"] if \"lat_0\" in d else float(0)\n params[\"latitude_of_projection_origin\"] = float(v)\n v1 = d[\"lat_1\"] if \"lat_1\" in d else float(0)\n v2 = d[\"lat_2\"] if \"lat_2\" in d else float(0)\n params[\"standard_parallel\"] = (float(v1), float(v2))\n v = d[\"ellps\"] if \"ellps\" in d else \"\"\n if len(v):\n params[\"reference_ellipsoid_name\"] = v\n v = d[\"towgs84\"] if \"towgs84\" in d else \"\"\n if len(v):\n params[\"towgs84\"] = v\n else:\n print(\"unknown projection\", d[\"proj\"])\n return None, None, None\n\n return grid_mapping_var_name, grid_mapping_name, params\n\n\ndef _create_geotiff_file(outfn, driver, shape, metadata, num_bands):\n dst = driver.Create(\n outfn,\n shape[1],\n shape[0],\n num_bands,\n gdal.GDT_Float32,\n [\"COMPRESS=DEFLATE\", \"PREDICTOR=3\"],\n )\n\n sx = (metadata[\"x2\"] - metadata[\"x1\"]) / shape[1]\n sy = (metadata[\"y2\"] - metadata[\"y1\"]) / shape[0]\n dst.SetGeoTransform([metadata[\"x1\"], sx, 0.0, metadata[\"y2\"], 0.0, -sy])\n\n sr = osr.SpatialReference()\n sr.ImportFromProj4(metadata[\"projection\"])\n dst.SetProjection(sr.ExportToWkt())\n\n return dst\n\n\ndef _get_geotiff_filename(prefix, startdate, n_timesteps, timestep, timestep_index):\n if n_timesteps * timestep == 0:\n raise ValueError(\"n_timesteps x timestep can't be 0.\")\n\n timestep_format_str = (\n f\"{{time_str:0{int(np.floor(np.log10(n_timesteps * timestep))) + 1}d}}\"\n )\n\n startdate_str = datetime.strftime(startdate, \"%Y%m%d%H%M\")\n\n timestep_str = timestep_format_str.format(time_str=(timestep_index + 1) * timestep)\n\n return f\"{prefix}_{startdate_str}_{timestep_str}.tif\"\n" }, { "path": "pysteps/io/importers.py", "content": "\"\"\"\npysteps.io.importers\n====================\n\nMethods for importing files containing two-dimensional radar mosaics.\n\nThe methods in this module implement the following interface::\n\n import_xxx(filename, optional arguments)\n\nwhere **xxx** is the name (or abbreviation) of the file format and filename\nis the name of the input file.\n\nThe output of each method is a three-element tuple containing a two-dimensional\nradar mosaic, the corresponding quality field and a metadata dictionary. If the\nfile contains no quality information, the quality field is set to None. Pixels\ncontaining missing data are set to nan.\n\nThe metadata dictionary contains the following recommended key-value pairs:\n\n.. tabularcolumns:: |p{2cm}|L|\n\n+------------------+----------------------------------------------------------+\n| Key | Value |\n+==================+==========================================================+\n| projection | PROJ.4-compatible projection definition |\n+------------------+----------------------------------------------------------+\n| x1 | x-coordinate of the lower-left corner of the data raster |\n+------------------+----------------------------------------------------------+\n| y1 | y-coordinate of the lower-left corner of the data raster |\n+------------------+----------------------------------------------------------+\n| x2 | x-coordinate of the upper-right corner of the data raster|\n+------------------+----------------------------------------------------------+\n| y2 | y-coordinate of the upper-right corner of the data raster|\n+------------------+----------------------------------------------------------+\n| xpixelsize | grid resolution in x-direction |\n+------------------+----------------------------------------------------------+\n| ypixelsize | grid resolution in y-direction |\n+------------------+----------------------------------------------------------+\n| cartesian_unit | the physical unit of the cartesian x- and y-coordinates: |\n| | e.g. 'm' or 'km' |\n+------------------+----------------------------------------------------------+\n| yorigin | a string specifying the location of the first element in |\n| | the data raster w.r.t. y-axis: |\n| | 'upper' = upper border |\n| | 'lower' = lower border |\n+------------------+----------------------------------------------------------+\n| institution | name of the institution who provides the data |\n+------------------+----------------------------------------------------------+\n| unit | the physical unit of the data: 'mm/h', 'mm' or 'dBZ' |\n+------------------+----------------------------------------------------------+\n| transform | the transformation of the data: None, 'dB', 'Box-Cox' or |\n| | others |\n+------------------+----------------------------------------------------------+\n| accutime | the accumulation time in minutes of the data, float |\n+------------------+----------------------------------------------------------+\n| threshold | the rain/no rain threshold with the same unit, |\n| | transformation and accutime of the data. |\n+------------------+----------------------------------------------------------+\n| zerovalue | the value assigned to the no rain pixels with the same |\n| | unit, transformation and accutime of the data. |\n+------------------+----------------------------------------------------------+\n| zr_a | the Z-R constant a in Z = a*R**b |\n+------------------+----------------------------------------------------------+\n| zr_b | the Z-R exponent b in Z = a*R**b |\n+------------------+----------------------------------------------------------+\n\nAvailable Importers\n-------------------\n\n.. autosummary::\n :toctree: ../generated/\n\n import_bom_rf3\n import_fmi_geotiff\n import_fmi_pgm\n import_knmi_hdf5\n import_mch_gif\n import_mch_hdf5\n import_mch_metranet\n import_mrms_grib\n import_odim_hdf5\n import_opera_hdf5\n import_saf_crri\n import_dwd_hdf5\n import_dwd_radolan\n\"\"\"\n\nimport gzip\nimport os\nimport array\nimport datetime\nfrom functools import partial\n\nimport numpy as np\n\nfrom matplotlib.pyplot import imread\n\nfrom pysteps.decorators import postprocess_import\nfrom pysteps.exceptions import DataModelError\nfrom pysteps.exceptions import MissingOptionalDependency\nfrom pysteps.utils import aggregate_fields\n\ntry:\n from osgeo import gdal, gdalconst, osr\n\n # Preserve current behavior explicitly (no GDAL exceptions) and avoid the\n # GDAL 4.0 future-warning emitted when neither mode is selected.\n if hasattr(gdal, \"DontUseExceptions\"):\n gdal.DontUseExceptions()\n GDAL_IMPORTED = True\nexcept ImportError:\n GDAL_IMPORTED = False\n\ntry:\n import h5py\n\n H5PY_IMPORTED = True\nexcept ImportError:\n H5PY_IMPORTED = False\n\ntry:\n import metranet\n\n METRANET_IMPORTED = True\nexcept ImportError:\n METRANET_IMPORTED = False\n\ntry:\n import netCDF4\n\n NETCDF4_IMPORTED = True\nexcept ImportError:\n NETCDF4_IMPORTED = False\n\ntry:\n from PIL import Image\n\n PIL_IMPORTED = True\nexcept ImportError:\n PIL_IMPORTED = False\n\ntry:\n import pyproj\n\n PYPROJ_IMPORTED = True\nexcept ImportError:\n PYPROJ_IMPORTED = False\n\ntry:\n import pygrib\n\n PYGRIB_IMPORTED = True\nexcept ImportError:\n PYGRIB_IMPORTED = False\n\n\ndef _check_coords_range(selected_range, coordinate, full_range):\n \"\"\"\n Check that the coordinates range arguments follow the expected pattern in\n the **import_mrms_grib** function.\"\"\"\n\n if selected_range is None:\n return sorted(full_range)\n\n if not isinstance(selected_range, (list, tuple)):\n if len(selected_range) != 2:\n raise ValueError(\n f\"The {coordinate} range must be None or a two-element tuple or list\"\n )\n\n selected_range = list(selected_range) # Make mutable\n\n for i in range(2):\n if selected_range[i] is None:\n selected_range[i] = full_range\n\n selected_range.sort()\n\n return tuple(selected_range)\n\n\ndef _get_grib_projection(grib_msg):\n \"\"\"Get the projection parameters from the grib file.\"\"\"\n projparams = grib_msg.projparams\n\n # Some versions of pygrib defines the regular lat/lon projections as \"cyl\",\n # which causes errors in pyproj and cartopy. Here we replace it for \"longlat\".\n if projparams[\"proj\"] == \"cyl\":\n projparams[\"proj\"] = \"longlat\"\n\n # Grib C tables (3-2)\n # https://apps.ecmwf.int/codes/grib/format/grib2/ctables/3/2\n # https://en.wikibooks.org/wiki/PROJ.4\n _grib_shapes_of_earth = dict()\n _grib_shapes_of_earth[0] = {\"R\": 6367470}\n _grib_shapes_of_earth[1] = {\"R\": 6367470}\n _grib_shapes_of_earth[2] = {\"ellps\": \"IAU76\"}\n _grib_shapes_of_earth[4] = {\"ellps\": \"GRS80\"}\n _grib_shapes_of_earth[5] = {\"ellps\": \"WGS84\"}\n _grib_shapes_of_earth[6] = {\"R\": 6371229}\n _grib_shapes_of_earth[8] = {\n \"datum\": \"WGS84\",\n \"R\": 6371200,\n }\n _grib_shapes_of_earth[9] = {\"datum\": \"OSGB36\"}\n\n # pygrib defines the ellipsoids using \"a\" and \"b\" only.\n # Here we replace the for the PROJ.4 SpheroidCodes if they are available.\n if grib_msg[\"shapeOfTheEarth\"] in _grib_shapes_of_earth:\n keys_to_remove = [\"a\", \"b\"]\n for key in keys_to_remove:\n if key in projparams:\n del projparams[key]\n\n projparams.update(_grib_shapes_of_earth[grib_msg[\"shapeOfTheEarth\"]])\n\n return projparams\n\n\ndef _get_threshold_value(precip):\n \"\"\"\n Get the the rain/no rain threshold with the same unit, transformation and\n accutime of the data.\n If all the values are NaNs, the returned value is `np.nan`.\n Otherwise, np.min(precip[precip > precip.min()]) is returned.\n\n Returns\n -------\n threshold: float\n \"\"\"\n valid_mask = np.isfinite(precip)\n if valid_mask.any():\n _precip = precip[valid_mask]\n min_precip = _precip.min()\n above_min_mask = _precip > min_precip\n if above_min_mask.any():\n return np.min(_precip[above_min_mask])\n else:\n return min_precip\n else:\n return np.nan\n\n\n@postprocess_import(dtype=\"float32\")\ndef import_mrms_grib(filename, extent=None, window_size=4, **kwargs):\n \"\"\"\n Importer for NSSL's Multi-Radar/Multi-Sensor System\n ([MRMS](https://www.nssl.noaa.gov/projects/mrms/)) rainrate product\n (grib format).\n\n The rainrate values are expressed in mm/h, and the dimensions of the data\n array are [latitude, longitude]. The first grid point (0,0) corresponds to\n the upper left corner of the domain, while (last i, last j) denote the\n lower right corner.\n\n Due to the large size of the dataset (3500 x 7000), a float32 type is used\n by default to reduce the memory footprint. However, be aware that when this\n array is passed to a pystep function, it may be converted to double\n precision, doubling the memory footprint.\n To change the precision of the data, use the ``dtype`` keyword.\n\n Also, by default, the original data is downscaled by 4\n (resulting in a ~4 km grid spacing).\n In case that the original grid spacing is needed, use ``window_size=1``.\n But be aware that a single composite in double precipitation will\n require 186 Mb of memory.\n\n Finally, if desired, the precipitation data can be extracted over a\n sub region of the full domain using the `extent` keyword.\n By default, the entire domain is returned.\n\n Notes\n -----\n In the MRMS grib files, \"-3\" is used to represent \"No Coverage\" or\n \"Missing data\". However, in this reader replace those values by the value\n specified in the `fillna` argument (NaN by default).\n\n Note that \"missing values\" are not the same as \"no precipitation\" values.\n Missing values indicates regions with no valid measures.\n While zero precipitation indicates regions with valid measurements,\n but with no precipitation detected.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n extent: None or array-like\n Longitude and latitude range (in degrees) of the data to be retrieved.\n (min_lon, max_lon, min_lat, max_lat).\n By default (None), the entire domain is retrieved.\n The extent can be in any form that can be converted to a flat array\n of 4 elements array (e.g., lists or tuples).\n window_size: array_like or int\n Array containing down-sampling integer factor along each axis.\n If an integer value is given, the same block shape is used for all the\n image dimensions.\n Default: window_size=4.\n\n {extra_kwargs_doc}\n\n Returns\n -------\n precipitation: 2D array, float32\n Precipitation field in mm/h. The dimensions are [latitude, longitude].\n The first grid point (0,0) corresponds to the upper left corner of the\n domain, while (last i, last j) denote the lower right corner.\n quality: None\n Not implement.\n metadata: dict\n Associated metadata (pixel sizes, map projections, etc.).\n \"\"\"\n\n del kwargs\n\n if not PYGRIB_IMPORTED:\n raise MissingOptionalDependency(\n \"pygrib package is required to import NCEP's MRMS products but it is not installed\"\n )\n\n try:\n grib_file = pygrib.open(filename)\n except OSError:\n raise OSError(f\"Error opening NCEP's MRMS file. \" f\"File Not Found: {filename}\")\n\n if isinstance(window_size, int):\n window_size = (window_size, window_size)\n\n if extent is not None:\n extent = np.asarray(extent)\n if (extent.ndim != 1) or (extent.size != 4):\n raise ValueError(\n \"The extent must be None or a flat array with 4 elements.\\n\"\n f\"Received: extent.shape = {str(extent.shape)}\"\n )\n\n # The MRMS grib file contain one message with the precipitation intensity\n grib_file.rewind()\n grib_msg = grib_file.read(1)[0] # Read the only message\n\n # -------------------------\n # Read the grid information\n\n lr_lon = grib_msg[\"longitudeOfLastGridPointInDegrees\"]\n lr_lat = grib_msg[\"latitudeOfLastGridPointInDegrees\"]\n\n ul_lon = grib_msg[\"longitudeOfFirstGridPointInDegrees\"]\n ul_lat = grib_msg[\"latitudeOfFirstGridPointInDegrees\"]\n\n # Ni - Number of points along a latitude circle (west-east)\n # Nj - Number of points along a longitude meridian (south-north)\n # The lat/lon grid has a 0.01 degrees spacing.\n lats = np.linspace(ul_lat, lr_lat, grib_msg[\"Nj\"])\n lons = np.linspace(ul_lon, lr_lon, grib_msg[\"Ni\"])\n\n precip = grib_msg.values\n no_data_mask = precip == -3 # Missing values\n\n # Create a function with default arguments for aggregate_fields\n block_reduce = partial(aggregate_fields, method=\"mean\", trim=True)\n\n if window_size != (1, 1):\n # Downscale data\n lats = block_reduce(lats, window_size[0])\n lons = block_reduce(lons, window_size[1])\n\n # Update the limits\n ul_lat, lr_lat = (\n lats[0],\n lats[-1],\n ) # Lat from North to south!\n ul_lon, lr_lon = lons[0], lons[-1]\n\n precip[no_data_mask] = 0 # block_reduce does not handle nan values\n precip = block_reduce(precip, window_size, axis=(0, 1))\n\n # Consider that if a single invalid observation is located in the block,\n # then mark that value as invalid.\n no_data_mask = block_reduce(\n no_data_mask.astype(\"int\"),\n window_size,\n axis=(0, 1),\n ).astype(bool)\n\n lons, lats = np.meshgrid(lons, lats)\n precip[no_data_mask] = np.nan\n\n if extent is not None:\n # clip domain\n ul_lon, lr_lon = _check_coords_range(\n (extent[0], extent[1]),\n \"longitude\",\n (ul_lon, lr_lon),\n )\n\n lr_lat, ul_lat = _check_coords_range(\n (extent[2], extent[3]),\n \"latitude\",\n (ul_lat, lr_lat),\n )\n\n mask_lat = (lats >= lr_lat) & (lats <= ul_lat)\n mask_lon = (lons >= ul_lon) & (lons <= lr_lon)\n\n nlats = np.count_nonzero(mask_lat[:, 0])\n nlons = np.count_nonzero(mask_lon[0, :])\n\n precip = precip[mask_lon & mask_lat].reshape(nlats, nlons)\n\n proj_params = _get_grib_projection(grib_msg)\n pr = pyproj.Proj(proj_params)\n proj_def = \" \".join([f\"+{key}={value} \" for key, value in proj_params.items()])\n\n xsize = grib_msg[\"iDirectionIncrementInDegrees\"] * window_size[0]\n ysize = grib_msg[\"jDirectionIncrementInDegrees\"] * window_size[1]\n\n x1, y1 = pr(ul_lon, lr_lat)\n x2, y2 = pr(lr_lon, ul_lat)\n\n metadata = dict(\n institution=\"NOAA National Severe Storms Laboratory\",\n xpixelsize=xsize,\n ypixelsize=ysize,\n unit=\"mm/h\",\n accutime=2.0,\n transform=None,\n zerovalue=0,\n projection=proj_def.strip(),\n yorigin=\"upper\",\n threshold=_get_threshold_value(precip),\n x1=x1 - xsize / 2,\n x2=x2 + xsize / 2,\n y1=y1 - ysize / 2,\n y2=y2 + ysize / 2,\n cartesian_unit=\"degrees\",\n )\n\n return precip, None, metadata\n\n\n@postprocess_import()\ndef import_bom_rf3(filename, **kwargs):\n \"\"\"\n Import a NetCDF radar rainfall product from the BoM Rainfields3.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n\n {extra_kwargs_doc}\n\n Returns\n -------\n out: tuple\n A three-element tuple containing the rainfall field in mm/h imported\n from the Bureau RF3 netcdf, the quality field and the metadata. The\n quality field is currently set to None.\n \"\"\"\n if not NETCDF4_IMPORTED:\n raise MissingOptionalDependency(\n \"netCDF4 package is required to import BoM Rainfields3 products \"\n \"but it is not installed\"\n )\n\n precip, geodata = _import_bom_rf3_data(filename)\n metadata = geodata\n\n metadata[\"transform\"] = None\n metadata[\"zerovalue\"] = np.nanmin(precip)\n metadata[\"threshold\"] = _get_threshold_value(precip)\n\n return precip, None, metadata\n\n\ndef _import_bom_rf3_data(filename):\n ds_rainfall = netCDF4.Dataset(filename)\n geodata = _import_bom_rf3_geodata(ds_rainfall)\n if \"precipitation\" in ds_rainfall.variables.keys():\n precipitation = ds_rainfall.variables[\"precipitation\"][:]\n else:\n precipitation = None\n ds_rainfall.close()\n\n return precipitation, geodata\n\n\ndef _import_bom_rf3_geodata(ds_rainfall):\n geodata = {}\n\n if \"proj\" in ds_rainfall.variables.keys():\n projection = ds_rainfall.variables[\"proj\"]\n if getattr(projection, \"grid_mapping_name\") == \"albers_conical_equal_area\":\n projdef = \"+proj=aea \"\n lon_0 = getattr(projection, \"longitude_of_central_meridian\")\n projdef += \" +lon_0=\" + f\"{lon_0:.3f}\"\n lat_0 = getattr(projection, \"latitude_of_projection_origin\")\n projdef += \" +lat_0=\" + f\"{lat_0:.3f}\"\n standard_parallels = getattr(projection, \"standard_parallel\")\n projdef += \" +lat_1=\" + f\"{standard_parallels[0]:.3f}\"\n projdef += \" +lat_2=\" + f\"{standard_parallels[1]:.3f}\"\n else:\n projdef = None\n geodata[\"projection\"] = projdef\n\n if \"valid_min\" in ds_rainfall.variables[\"x\"].ncattrs():\n xmin = getattr(ds_rainfall.variables[\"x\"], \"valid_min\")\n xmax = getattr(ds_rainfall.variables[\"x\"], \"valid_max\")\n ymin = getattr(ds_rainfall.variables[\"y\"], \"valid_min\")\n ymax = getattr(ds_rainfall.variables[\"y\"], \"valid_max\")\n else:\n xmin = min(ds_rainfall.variables[\"x\"])\n xmax = max(ds_rainfall.variables[\"x\"])\n ymin = min(ds_rainfall.variables[\"y\"])\n ymax = max(ds_rainfall.variables[\"y\"])\n\n xpixelsize = abs(ds_rainfall.variables[\"x\"][1] - ds_rainfall.variables[\"x\"][0])\n ypixelsize = abs(ds_rainfall.variables[\"y\"][1] - ds_rainfall.variables[\"y\"][0])\n factor_scale = 1.0\n if \"units\" in ds_rainfall.variables[\"x\"].ncattrs():\n if getattr(ds_rainfall.variables[\"x\"], \"units\") == \"km\":\n factor_scale = 1000.0\n\n geodata[\"x1\"] = xmin * factor_scale\n geodata[\"y1\"] = ymin * factor_scale\n geodata[\"x2\"] = xmax * factor_scale\n geodata[\"y2\"] = ymax * factor_scale\n geodata[\"xpixelsize\"] = xpixelsize * factor_scale\n geodata[\"ypixelsize\"] = ypixelsize * factor_scale\n geodata[\"cartesian_unit\"] = \"m\"\n geodata[\"yorigin\"] = \"upper\"\n\n # get the accumulation period\n valid_time = None\n\n if \"valid_time\" in ds_rainfall.variables.keys():\n times = ds_rainfall.variables[\"valid_time\"]\n calendar = \"standard\"\n if \"calendar\" in times.ncattrs():\n calendar = times.calendar\n valid_time = netCDF4.num2date(times[:], units=times.units, calendar=calendar)\n\n start_time = None\n if \"start_time\" in ds_rainfall.variables.keys():\n times = ds_rainfall.variables[\"start_time\"]\n calendar = \"standard\"\n if \"calendar\" in times.ncattrs():\n calendar = times.calendar\n start_time = netCDF4.num2date(times[:], units=times.units, calendar=calendar)\n\n time_step = None\n\n if start_time is not None:\n if valid_time is not None:\n time_step = (valid_time - start_time).seconds // 60\n\n geodata[\"accutime\"] = time_step\n\n # get the unit of precipitation\n if \"units\" in ds_rainfall.variables[\"precipitation\"].ncattrs():\n units = getattr(ds_rainfall.variables[\"precipitation\"], \"units\")\n if units in (\"kg m-2\", \"mm\"):\n geodata[\"unit\"] = \"mm\"\n\n geodata[\"institution\"] = \"Commonwealth of Australia, Bureau of Meteorology\"\n\n return geodata\n\n\n@postprocess_import()\ndef import_fmi_geotiff(filename, **kwargs):\n \"\"\"\n Import a reflectivity field (dBZ) from an FMI GeoTIFF file.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n\n {extra_kwargs_doc}\n\n Returns\n -------\n out: tuple\n A three-element tuple containing the precipitation field,\n the associated quality field and metadata.\n The quality field is currently set to None.\n \"\"\"\n if not GDAL_IMPORTED:\n raise MissingOptionalDependency(\n \"gdal package is required to import \"\n \"FMI's radar reflectivity composite in GeoTIFF format \"\n \"but it is not installed\"\n )\n\n f = gdal.Open(filename, gdalconst.GA_ReadOnly)\n\n rb = f.GetRasterBand(1)\n precip = rb.ReadAsArray().astype(float)\n mask = precip == 255\n precip = (precip - 64.0) / 2.0\n precip[mask] = np.nan\n\n sr = osr.SpatialReference()\n pr = f.GetProjection()\n sr.ImportFromWkt(pr)\n\n projdef = sr.ExportToProj4()\n\n gt = f.GetGeoTransform()\n\n metadata = {}\n\n metadata[\"projection\"] = projdef\n metadata[\"x1\"] = gt[0]\n metadata[\"y1\"] = gt[3] + gt[5] * f.RasterYSize\n metadata[\"x2\"] = metadata[\"x1\"] + gt[1] * f.RasterXSize\n metadata[\"y2\"] = gt[3]\n metadata[\"xpixelsize\"] = abs(gt[1])\n metadata[\"ypixelsize\"] = abs(gt[5])\n if gt[5] < 0:\n metadata[\"yorigin\"] = \"upper\"\n else:\n metadata[\"yorigin\"] = \"lower\"\n metadata[\"institution\"] = \"Finnish Meteorological Institute\"\n metadata[\"unit\"] = \"dBZ\"\n metadata[\"transform\"] = \"dB\"\n metadata[\"accutime\"] = 5.0\n metadata[\"threshold\"] = _get_threshold_value(precip)\n metadata[\"zerovalue\"] = np.nanmin(precip)\n metadata[\"cartesian_unit\"] = \"m\"\n metadata[\"zr_a\"] = 223.0\n metadata[\"zr_b\"] = 1.53\n\n return precip, None, metadata\n\n\n@postprocess_import()\ndef import_fmi_pgm(filename, gzipped=False, **kwargs):\n \"\"\"\n Import a 8-bit PGM radar reflectivity composite from the FMI archive.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n gzipped: bool\n If True, the input file is treated as a compressed gzip file.\n\n {extra_kwargs_doc}\n\n Returns\n -------\n out: tuple\n A three-element tuple containing the reflectivity composite in dBZ\n and the associated quality field and metadata. The quality field is\n currently set to None.\n\n Notes\n -----\n Reading georeferencing metadata is supported only for stereographic\n projection. For other projections, the keys related to georeferencing are\n not set.\n \"\"\"\n if not PYPROJ_IMPORTED:\n raise MissingOptionalDependency(\n \"pyproj package is required to import \"\n \"FMI's radar reflectivity composite \"\n \"but it is not installed\"\n )\n\n if gzipped is False:\n precip = imread(filename)\n else:\n precip = imread(gzip.open(filename, \"r\"))\n pgm_metadata = _import_fmi_pgm_metadata(filename, gzipped=gzipped)\n geodata = _import_fmi_pgm_geodata(pgm_metadata)\n\n mask = precip == pgm_metadata[\"missingval\"]\n precip = precip.astype(float)\n precip[mask] = np.nan\n precip = (precip - 64.0) / 2.0\n\n metadata = geodata\n metadata[\"institution\"] = \"Finnish Meteorological Institute\"\n metadata[\"accutime\"] = 5.0\n metadata[\"unit\"] = \"dBZ\"\n metadata[\"transform\"] = \"dB\"\n metadata[\"zerovalue\"] = np.nanmin(precip)\n metadata[\"threshold\"] = _get_threshold_value(precip)\n metadata[\"zr_a\"] = 223.0\n metadata[\"zr_b\"] = 1.53\n\n return precip, None, metadata\n\n\ndef _import_fmi_pgm_geodata(metadata):\n geodata = {}\n\n projdef = \"\"\n\n if \"type\" in metadata.keys() and metadata[\"type\"][0] == \"stereographic\":\n projdef += \"+proj=stere \"\n projdef += \" +lon_0=\" + metadata[\"centrallongitude\"][0] + \"E\"\n projdef += \" +lat_0=\" + metadata[\"centrallatitude\"][0] + \"N\"\n projdef += \" +lat_ts=\" + metadata[\"truelatitude\"][0]\n # These are hard-coded because the projection definition\n # is missing from the PGM files.\n projdef += \" +a=6371288\"\n projdef += \" +x_0=380886.310\"\n projdef += \" +y_0=3395677.920\"\n projdef += \" +no_defs\"\n #\n geodata[\"projection\"] = projdef\n\n ll_lon, ll_lat = [float(v) for v in metadata[\"bottomleft\"]]\n ur_lon, ur_lat = [float(v) for v in metadata[\"topright\"]]\n\n pr = pyproj.Proj(projdef)\n x1, y1 = pr(ll_lon, ll_lat)\n x2, y2 = pr(ur_lon, ur_lat)\n\n geodata[\"x1\"] = x1\n geodata[\"y1\"] = y1\n geodata[\"x2\"] = x2\n geodata[\"y2\"] = y2\n geodata[\"cartesian_unit\"] = \"m\"\n geodata[\"xpixelsize\"] = float(metadata[\"metersperpixel_x\"][0])\n geodata[\"ypixelsize\"] = float(metadata[\"metersperpixel_y\"][0])\n\n geodata[\"yorigin\"] = \"upper\"\n\n return geodata\n\n\ndef _import_fmi_pgm_metadata(filename, gzipped=False):\n metadata = {}\n\n if not gzipped:\n f = open(filename, \"rb\")\n else:\n f = gzip.open(filename, \"rb\")\n\n file_line = f.readline()\n while not file_line.startswith(b\"#\"):\n file_line = f.readline()\n while file_line.startswith(b\"#\"):\n x = file_line.decode()\n x = x[1:].strip().split(\" \")\n if len(x) >= 2:\n k = x[0]\n v = x[1:]\n metadata[k] = v\n else:\n file_line = f.readline()\n continue\n file_line = f.readline()\n file_line = f.readline().decode()\n metadata[\"missingval\"] = int(file_line)\n f.close()\n\n return metadata\n\n\n@postprocess_import()\ndef import_knmi_hdf5(\n filename,\n qty=\"ACRR\",\n accutime=5.0,\n pixelsize=1000.0,\n **kwargs,\n):\n \"\"\"\n Import a precipitation or reflectivity field (and optionally the quality\n field) from a HDF5 file conforming to the KNMI Data Centre specification.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n qty: {'ACRR', 'DBZH'}\n The quantity to read from the file. The currently supported identifiers\n are: 'ACRR'=hourly rainfall accumulation (mm) and 'DBZH'=max-reflectivity\n (dBZ). The default value is 'ACRR'.\n accutime: float\n The accumulation time of the dataset in minutes. A 5 min accumulation\n is used as default, but hourly, daily and monthly accumulations\n are also available.\n pixelsize: float\n The pixel size of a raster cell in meters. The default value for the\n KNMI datasets is a 1000 m grid cell size, but datasets with 2400 m pixel\n size are also available.\n\n {extra_kwargs_doc}\n\n Returns\n -------\n out: tuple\n A three-element tuple containing precipitation accumulation [mm] /\n reflectivity [dBZ] of the KNMI product, the associated quality field\n and metadata. The quality field is currently set to None.\n\n Notes\n -----\n Every KNMI data type has a slightly different naming convention. The\n standard setup is based on the accumulated rainfall product on 1 km2 spatial\n and 5 min temporal resolution.\n See https://data.knmi.nl/datasets?q=radar for a list of all available KNMI\n radar data.\n \"\"\"\n\n # TODO: Add quality field.\n\n if not H5PY_IMPORTED:\n raise MissingOptionalDependency(\n \"h5py package is required to import \"\n \"KNMI's radar datasets \"\n \"but it is not installed\"\n )\n\n if qty not in [\"ACRR\", \"DBZH\"]:\n raise ValueError(\n \"unknown quantity %s: the available options are 'ACRR' and 'DBZH' \"\n )\n\n ####\n # Precipitation fields\n ####\n\n f = h5py.File(filename, \"r\")\n dset = f[\"image1\"][\"image_data\"]\n precip_intermediate = np.copy(dset) # copy the content\n\n # In case precip is a rainfall accumulation (ACRR), precip is divided by 100.0,\n # because the data is saved as hundreds of mm (so, as integers). 65535 is\n # the no data value. The precision of the data is two decimals (0.01 mm).\n if qty == \"ACRR\":\n precip = np.where(\n precip_intermediate == 65535,\n np.nan,\n precip_intermediate / 100.0,\n )\n\n # In case reflectivities are imported, the no data value is 255. Values are\n # saved as integers. The reflectivities are not directly saved in dBZ, but\n # as: dBZ = 0.5 * pixel_value - 32.0 (this used to be 31.5).\n if qty == \"DBZH\":\n precip = np.where(\n precip_intermediate == 255,\n np.nan,\n precip_intermediate * 0.5 - 32.0,\n )\n\n if precip is None:\n raise IOError(\"requested quantity not found\")\n\n ####\n # Meta data\n ####\n\n metadata = {}\n\n if qty == \"ACRR\":\n unit = \"mm\"\n transform = None\n elif qty == \"DBZH\":\n unit = \"dBZ\"\n transform = \"dB\"\n\n # The 'where' group of mch- and Opera-data, is called 'geographic' in the\n # KNMI data.\n geographic = f[\"geographic\"]\n proj4str = \"+proj=stere +lat_0=90 +lon_0=0.0 +lat_ts=60.0 +a=6378137 +b=6356752 +x_0=0 +y_0=0\"\n pr = pyproj.Proj(proj4str)\n metadata[\"projection\"] = proj4str\n\n # Get coordinates\n latlon_corners = geographic.attrs[\"geo_product_corners\"]\n ll_lat = latlon_corners[1]\n ll_lon = latlon_corners[0]\n ur_lat = latlon_corners[5]\n ur_lon = latlon_corners[4]\n lr_lat = latlon_corners[7]\n lr_lon = latlon_corners[6]\n ul_lat = latlon_corners[3]\n ul_lon = latlon_corners[2]\n\n ll_x, ll_y = pr(ll_lon, ll_lat)\n ur_x, ur_y = pr(ur_lon, ur_lat)\n lr_x, lr_y = pr(lr_lon, lr_lat)\n ul_x, ul_y = pr(ul_lon, ul_lat)\n x1 = min(ll_x, ul_x)\n y1 = min(ll_y, lr_y)\n x2 = max(lr_x, ur_x)\n y2 = max(ul_y, ur_y)\n\n # Fill in the metadata\n metadata[\"x1\"] = x1\n metadata[\"y1\"] = y1\n metadata[\"x2\"] = x2\n metadata[\"y2\"] = y2\n metadata[\"xpixelsize\"] = pixelsize\n metadata[\"ypixelsize\"] = pixelsize\n metadata[\"cartesian_unit\"] = \"m\"\n metadata[\"yorigin\"] = \"upper\"\n metadata[\"institution\"] = \"KNMI - Royal Netherlands Meteorological Institute\"\n metadata[\"accutime\"] = accutime\n metadata[\"unit\"] = unit\n metadata[\"transform\"] = transform\n metadata[\"zerovalue\"] = 0.0\n metadata[\"threshold\"] = _get_threshold_value(precip)\n metadata[\"zr_a\"] = 200.0\n metadata[\"zr_b\"] = 1.6\n\n f.close()\n\n return precip, None, metadata\n\n\n@postprocess_import()\ndef import_mch_gif(filename, product, unit, accutime, **kwargs):\n \"\"\"\n Import a 8-bit gif radar reflectivity composite from the MeteoSwiss\n archive.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n product: {\"AQC\", \"CPC\", \"RZC\", \"AZC\"}\n The name of the MeteoSwiss QPE product.\\n\n Currently supported prducts:\n\n +------+----------------------------+\n | Name | Product |\n +======+============================+\n | AQC | Acquire |\n +------+----------------------------+\n | CPC | CombiPrecip |\n +------+----------------------------+\n | RZC | Precip |\n +------+----------------------------+\n | AZC | RZC accumulation |\n +------+----------------------------+\n\n unit: {\"mm/h\", \"mm\", \"dBZ\"}\n the physical unit of the data\n accutime: float\n the accumulation time in minutes of the data\n\n {extra_kwargs_doc}\n\n Returns\n -------\n out: tuple\n A three-element tuple containing the precipitation field in mm/h imported\n from a MeteoSwiss gif file and the associated quality field and metadata.\n The quality field is currently set to None.\n \"\"\"\n if not PIL_IMPORTED:\n raise MissingOptionalDependency(\n \"PIL package is required to import \"\n \"radar reflectivity composite from MeteoSwiss\"\n \"but it is not installed\"\n )\n\n geodata = _import_mch_geodata()\n\n metadata = geodata\n\n # import gif file\n img = Image.open(filename)\n\n if product.lower() in [\"azc\", \"rzc\", \"precip\"]:\n # convert 8-bit GIF colortable to RGB values\n img_rgb = img.convert(\"RGB\")\n\n # load lookup table\n if product.lower() == \"azc\":\n lut_filename = os.path.join(\n os.path.dirname(__file__),\n \"mch_lut_8bit_Metranet_AZC_V104.txt\",\n )\n else:\n lut_filename = os.path.join(\n os.path.dirname(__file__),\n \"mch_lut_8bit_Metranet_v103.txt\",\n )\n lut = np.genfromtxt(lut_filename, skip_header=1)\n lut = dict(\n zip(\n zip(lut[:, 1], lut[:, 2], lut[:, 3]),\n lut[:, -1],\n )\n )\n\n # apply lookup table conversion\n precip = np.zeros(len(img_rgb.getdata()))\n for i, dn in enumerate(img_rgb.getdata()):\n precip[i] = lut.get(dn, np.nan)\n\n # convert to original shape\n width, height = img.size\n precip = precip.reshape(height, width)\n\n # set values outside observational range to NaN,\n # and values in non-precipitating areas to zero.\n precip[precip < 0] = 0\n precip[precip > 9999] = np.nan\n\n elif product.lower() in [\n \"aqc\",\n \"cpc\",\n \"acquire \",\n \"combiprecip\",\n ]:\n # convert digital numbers to physical values\n img = np.array(img).astype(int)\n\n # build lookup table [mm/5min]\n lut = np.zeros(256)\n a = 316.0\n b = 1.5\n for i in range(256):\n if (i < 2) or (i > 250 and i < 255):\n lut[i] = 0.0\n elif i == 255:\n lut[i] = np.nan\n else:\n lut[i] = (10.0 ** ((i - 71.5) / 20.0) / a) ** (1.0 / b)\n\n # apply lookup table\n precip = lut[img]\n\n else:\n raise ValueError(\"unknown product %s\" % product)\n\n metadata[\"accutime\"] = accutime\n metadata[\"unit\"] = unit\n metadata[\"transform\"] = None\n metadata[\"zerovalue\"] = np.nanmin(precip)\n metadata[\"threshold\"] = _get_threshold_value(precip)\n metadata[\"institution\"] = \"MeteoSwiss\"\n metadata[\"product\"] = product\n metadata[\"zr_a\"] = 316.0\n metadata[\"zr_b\"] = 1.5\n\n return precip, None, metadata\n\n\n@postprocess_import()\ndef import_mch_hdf5(filename, qty=\"RATE\", **kwargs):\n \"\"\"\n Import a precipitation field (and optionally the quality field) from a\n MeteoSwiss HDF5 file conforming to the ODIM specification.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n qty: {'RATE', 'ACRR', 'DBZH'}\n The quantity to read from the file. The currently supported identitiers\n are: 'RATE'=instantaneous rain rate (mm/h), 'ACRR'=hourly rainfall\n accumulation (mm) and 'DBZH'=max-reflectivity (dBZ). The default value\n is 'RATE'.\n\n {extra_kwargs_doc}\n\n Returns\n -------\n out: tuple\n A three-element tuple containing the OPERA product for the requested\n quantity and the associated quality field and metadata. The quality\n field is read from the file if it contains a dataset whose quantity\n identifier is 'QIND'.\n \"\"\"\n if not H5PY_IMPORTED:\n raise MissingOptionalDependency(\n \"h5py package is required to import \"\n \"radar reflectivity composites using ODIM HDF5 specification \"\n \"but it is not installed\"\n )\n\n if qty not in [\"ACRR\", \"DBZH\", \"RATE\"]:\n raise ValueError(\n \"unknown quantity %s: the available options are 'ACRR', 'DBZH' and 'RATE'\"\n )\n\n f = h5py.File(filename, \"r\")\n\n precip = None\n quality = None\n\n for dsg in f.items():\n if dsg[0].startswith(\"dataset\"):\n what_grp_found = False\n # check if the \"what\" group is in the \"dataset\" group\n if \"what\" in list(dsg[1].keys()):\n qty_, gain, offset, nodata, undetect = _read_mch_hdf5_what_group(\n dsg[1][\"what\"]\n )\n what_grp_found = True\n\n for dg in dsg[1].items():\n if dg[0][0:4] == \"data\":\n # check if the \"what\" group is in the \"data\" group\n if \"what\" in list(dg[1].keys()):\n (\n qty_,\n gain,\n offset,\n nodata,\n undetect,\n ) = _read_mch_hdf5_what_group(dg[1][\"what\"])\n elif not what_grp_found:\n raise DataModelError(\n \"Non ODIM compliant file: \"\n \"no what group found from {} \"\n \"or its subgroups\".format(dg[0])\n )\n\n if qty_.decode() in [qty, \"QIND\"]:\n arr = dg[1][\"data\"][...]\n mask_n = arr == nodata\n mask_u = arr == undetect\n mask = np.logical_and(~mask_u, ~mask_n)\n\n if qty_.decode() == qty:\n precip = np.empty(arr.shape)\n precip[mask] = arr[mask] * gain + offset\n precip[mask_u] = np.nan\n precip[mask_n] = np.nan\n elif qty_.decode() == \"QIND\":\n quality = np.empty(arr.shape, dtype=float)\n quality[mask] = arr[mask]\n quality[~mask] = np.nan\n\n if precip is None:\n raise IOError(\"requested quantity %s not found\" % qty)\n\n where = f[\"where\"]\n\n geodata = _import_mch_geodata()\n metadata = geodata\n\n # TODO: use those from the hdf5 file instead\n # xpixelsize = where.attrs[\"xscale\"] * 1000.0\n # ypixelsize = where.attrs[\"yscale\"] * 1000.0\n # xsize = where.attrs[\"xsize\"]\n # ysize = where.attrs[\"ysize\"]\n\n if qty == \"ACRR\":\n unit = \"mm\"\n transform = None\n elif qty == \"DBZH\":\n unit = \"dBZ\"\n transform = \"dB\"\n else:\n unit = \"mm/h\"\n transform = None\n\n if np.any(np.isfinite(precip)):\n thr = np.nanmin(precip[precip > np.nanmin(precip)])\n else:\n thr = np.nan\n\n metadata.update(\n {\n \"yorigin\": \"upper\",\n \"institution\": \"MeteoSwiss\",\n \"accutime\": 5.0,\n \"unit\": unit,\n \"transform\": transform,\n \"zerovalue\": np.nanmin(precip),\n \"threshold\": thr,\n \"zr_a\": 316.0,\n \"zr_b\": 1.5,\n }\n )\n\n f.close()\n\n return precip, quality, metadata\n\n\ndef _read_mch_hdf5_what_group(whatgrp):\n qty = whatgrp.attrs[\"quantity\"] if \"quantity\" in whatgrp.attrs.keys() else \"RATE\"\n gain = whatgrp.attrs[\"gain\"] if \"gain\" in whatgrp.attrs.keys() else 1.0\n offset = whatgrp.attrs[\"offset\"] if \"offset\" in whatgrp.attrs.keys() else 0.0\n nodata = whatgrp.attrs[\"nodata\"] if \"nodata\" in whatgrp.attrs.keys() else 0\n undetect = whatgrp.attrs[\"undetect\"] if \"undetect\" in whatgrp.attrs.keys() else -1.0\n\n return qty, gain, offset, nodata, undetect\n\n\n@postprocess_import()\ndef import_mch_metranet(filename, product, unit, accutime):\n \"\"\"\n Import a 8-bit bin radar reflectivity composite from the MeteoSwiss\n archive.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n product: {\"AQC\", \"CPC\", \"RZC\", \"AZC\"}\n The name of the MeteoSwiss QPE product.\\n\n Currently supported prducts:\n\n +------+----------------------------+\n | Name | Product |\n +======+============================+\n | AQC | Acquire |\n +------+----------------------------+\n | CPC | CombiPrecip |\n +------+----------------------------+\n | RZC | Precip |\n +------+----------------------------+\n | AZC | RZC accumulation |\n +------+----------------------------+\n\n unit: {\"mm/h\", \"mm\", \"dBZ\"}\n the physical unit of the data\n accutime: float\n the accumulation time in minutes of the data\n\n {extra_kwargs_doc}\n\n Returns\n -------\n\n out: tuple\n A three-element tuple containing the precipitation field in mm/h imported\n from a MeteoSwiss gif file and the associated quality field and metadata.\n The quality field is currently set to None.\n \"\"\"\n if not METRANET_IMPORTED:\n raise MissingOptionalDependency(\n \"metranet package needed for importing MeteoSwiss \"\n \"radar composites but it is not installed\"\n )\n\n ret = metranet.read_file(filename, physic_value=True, verbose=False)\n precip = ret.data\n\n geodata = _import_mch_geodata()\n\n # read metranet\n metadata = geodata\n metadata[\"institution\"] = \"MeteoSwiss\"\n metadata[\"accutime\"] = accutime\n metadata[\"unit\"] = unit\n metadata[\"transform\"] = None\n metadata[\"zerovalue\"] = np.nanmin(precip)\n metadata[\"threshold\"] = _get_threshold_value(precip)\n metadata[\"zr_a\"] = 316.0\n metadata[\"zr_b\"] = 1.5\n\n return precip, None, metadata\n\n\ndef _import_mch_geodata():\n \"\"\"\n Swiss radar domain CCS4\n These are all hard-coded because the georeferencing is missing from the gif files.\n \"\"\"\n\n geodata = {}\n\n # LV03 Swiss projection definition in Proj4\n projdef = \"\"\n projdef += \"+proj=somerc \"\n projdef += \" +lon_0=7.43958333333333\"\n projdef += \" +lat_0=46.9524055555556\"\n projdef += \" +k_0=1\"\n projdef += \" +x_0=600000\"\n projdef += \" +y_0=200000\"\n projdef += \" +ellps=bessel\"\n projdef += \" +towgs84=674.374,15.056,405.346,0,0,0,0\"\n projdef += \" +units=m\"\n projdef += \" +no_defs\"\n geodata[\"projection\"] = projdef\n\n geodata[\"x1\"] = 255000.0\n geodata[\"y1\"] = -160000.0\n geodata[\"x2\"] = 965000.0\n geodata[\"y2\"] = 480000.0\n\n geodata[\"xpixelsize\"] = 1000.0\n geodata[\"ypixelsize\"] = 1000.0\n geodata[\"cartesian_unit\"] = \"m\"\n geodata[\"yorigin\"] = \"upper\"\n\n return geodata\n\n\n@postprocess_import()\ndef import_odim_hdf5(filename, qty=\"RATE\", **kwargs):\n \"\"\"\n Import a precipitation field (and optionally the quality field) from a\n HDF5 file conforming to the ODIM specification.\n **Important:** Currently, only the Pan-European (OPERA) and the\n Dipartimento della Protezione Civile (DPC) radar composites are correctly supported.\n Other ODIM-compliant files may not be read correctly.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n qty: {'RATE', 'ACRR', 'DBZH'}\n The quantity to read from the file. The currently supported identitiers\n are: 'RATE'=instantaneous rain rate (mm/h), 'ACRR'=hourly rainfall\n accumulation (mm) and 'DBZH'=max-reflectivity (dBZ). The default value\n is 'RATE'.\n\n {extra_kwargs_doc}\n\n Returns\n -------\n out: tuple\n A three-element tuple containing the OPERA product for the requested\n quantity and the associated quality field and metadata. The quality\n field is read from the file if it contains a dataset whose quantity\n identifier is 'QIND'.\n \"\"\"\n if not H5PY_IMPORTED:\n raise MissingOptionalDependency(\n \"h5py package is required to import \"\n \"radar reflectivity composites using ODIM HDF5 specification \"\n \"but it is not installed\"\n )\n\n if qty not in [\"ACRR\", \"DBZH\", \"RATE\"]:\n raise ValueError(\n \"unknown quantity %s: the available options are 'ACRR', 'DBZH' and 'RATE'\"\n )\n\n f = h5py.File(filename, \"r\")\n\n precip = None\n quality = None\n\n for dsg in f.items():\n if dsg[0].startswith(\"dataset\"):\n what_grp_found = False\n # check if the \"what\" group is in the \"dataset\" group\n if \"what\" in list(dsg[1].keys()):\n if \"quantity\" in dsg[1][\"what\"].attrs.keys():\n try:\n (\n qty_,\n gain,\n offset,\n nodata,\n undetect,\n ) = _read_opera_hdf5_what_group(dsg[1][\"what\"])\n what_grp_found = True\n except KeyError:\n pass\n\n for dg in dsg[1].items():\n if dg[0][0:4] == \"data\":\n # check if the \"what\" group is in the \"data\" group\n if \"what\" in list(dg[1].keys()):\n (\n qty_,\n gain,\n offset,\n nodata,\n undetect,\n ) = _read_opera_hdf5_what_group(dg[1][\"what\"])\n elif not what_grp_found:\n raise DataModelError(\n \"Non ODIM compliant file: \"\n \"no what group found from {} \"\n \"or its subgroups\".format(dg[0])\n )\n\n if qty_.decode() in [qty, \"QIND\"]:\n arr = dg[1][\"data\"][...]\n mask_n = arr == nodata\n mask_u = arr == undetect\n mask = np.logical_and(~mask_u, ~mask_n)\n\n if qty_.decode() == qty:\n precip = np.empty(arr.shape)\n precip[mask] = arr[mask] * gain + offset\n if qty != \"DBZH\":\n precip[mask_u] = offset\n else:\n precip[mask_u] = -30.0\n precip[mask_n] = np.nan\n elif qty_.decode() == \"QIND\":\n quality = np.empty(arr.shape, dtype=float)\n quality[mask] = arr[mask]\n quality[~mask] = np.nan\n if quality is None:\n for dgg in dg[\n 1\n ].items(): # da qui ----------------------------\n if dgg[0][0:7] == \"quality\":\n quality_keys = list(dgg[1].keys())\n if \"what\" in quality_keys:\n (\n qty_,\n gain,\n offset,\n nodata,\n undetect,\n ) = _read_opera_hdf5_what_group(dgg[1][\"what\"])\n if qty_.decode() == \"QIND\":\n arr = dgg[1][\"data\"][...]\n mask_n = arr == nodata\n mask_u = arr == undetect\n mask = np.logical_and(~mask_u, ~mask_n)\n quality = np.empty(arr.shape) # , dtype=float)\n quality[mask] = arr[mask] * gain + offset\n quality[~mask] = (\n np.nan\n ) # a qui -----------------------------\n\n if precip is None:\n raise IOError(\"requested quantity %s not found\" % qty)\n\n where = f[\"where\"]\n if isinstance(where.attrs[\"projdef\"], str):\n proj4str = where.attrs[\"projdef\"]\n else:\n proj4str = where.attrs[\"projdef\"].decode()\n pr = pyproj.Proj(proj4str)\n\n ll_lat = where.attrs[\"LL_lat\"]\n ll_lon = where.attrs[\"LL_lon\"]\n ur_lat = where.attrs[\"UR_lat\"]\n ur_lon = where.attrs[\"UR_lon\"]\n if (\n \"LR_lat\" in where.attrs.keys()\n and \"LR_lon\" in where.attrs.keys()\n and \"UL_lat\" in where.attrs.keys()\n and \"UL_lon\" in where.attrs.keys()\n ):\n lr_lat = float(where.attrs[\"LR_lat\"])\n lr_lon = float(where.attrs[\"LR_lon\"])\n ul_lat = float(where.attrs[\"UL_lat\"])\n ul_lon = float(where.attrs[\"UL_lon\"])\n full_cornerpts = True\n else:\n full_cornerpts = False\n\n ll_x, ll_y = pr(ll_lon, ll_lat)\n ur_x, ur_y = pr(ur_lon, ur_lat)\n\n if full_cornerpts:\n lr_x, lr_y = pr(lr_lon, lr_lat)\n ul_x, ul_y = pr(ul_lon, ul_lat)\n x1 = min(ll_x, ul_x)\n y1 = min(ll_y, lr_y)\n x2 = max(lr_x, ur_x)\n y2 = max(ul_y, ur_y)\n else:\n x1 = ll_x\n y1 = ll_y\n x2 = ur_x\n y2 = ur_y\n\n dataset1 = f[\"dataset1\"]\n\n if \"xscale\" in where.attrs.keys() and \"yscale\" in where.attrs.keys():\n xpixelsize = where.attrs[\"xscale\"]\n ypixelsize = where.attrs[\"yscale\"]\n elif (\n \"xscale\" in dataset1[\"where\"].attrs.keys()\n and \"yscale\" in dataset1[\"where\"].attrs.keys()\n ):\n where = dataset1[\"where\"]\n xpixelsize = where.attrs[\"xscale\"]\n ypixelsize = where.attrs[\"yscale\"]\n else:\n xpixelsize = None\n ypixelsize = None\n\n if qty == \"ACRR\":\n unit = \"mm\"\n transform = None\n elif qty == \"DBZH\":\n unit = \"dBZ\"\n transform = \"dB\"\n else:\n unit = \"mm/h\"\n transform = None\n\n metadata = {\n \"projection\": proj4str,\n \"ll_lon\": ll_lon,\n \"ll_lat\": ll_lat,\n \"ur_lon\": ur_lon,\n \"ur_lat\": ur_lat,\n \"x1\": x1,\n \"y1\": y1,\n \"x2\": x2,\n \"y2\": y2,\n \"xpixelsize\": xpixelsize,\n \"ypixelsize\": ypixelsize,\n \"cartesian_unit\": \"m\",\n \"yorigin\": \"upper\",\n \"institution\": \"Odyssey datacentre\",\n \"accutime\": 15.0,\n \"unit\": unit,\n \"transform\": transform,\n \"zerovalue\": np.nanmin(precip),\n \"threshold\": _get_threshold_value(precip),\n }\n\n metadata.update(kwargs)\n\n f.close()\n\n return precip, quality, metadata\n\n\ndef import_opera_hdf5(filename, qty=\"RATE\", **kwargs):\n \"\"\"\n Wrapper to :py:func:`pysteps.io.importers.import_odim_hdf5`\n to maintain backward compatibility with previous pysteps versions.\n\n **Important:** Use :py:func:`~pysteps.io.importers.import_odim_hdf5` instead.\n \"\"\"\n return import_odim_hdf5(filename, qty=qty, **kwargs)\n\n\ndef _read_opera_hdf5_what_group(whatgrp):\n qty = whatgrp.attrs[\"quantity\"] if \"quantity\" in whatgrp.attrs.keys() else b\"QIND\"\n gain = whatgrp.attrs[\"gain\"] if \"gain\" in whatgrp.attrs.keys() else 1.0\n offset = whatgrp.attrs[\"offset\"] if \"offset\" in whatgrp.attrs.keys() else 0.0\n nodata = whatgrp.attrs[\"nodata\"] if \"nodata\" in whatgrp.attrs.keys() else np.nan\n undetect = whatgrp.attrs[\"undetect\"] if \"undetect\" in whatgrp.attrs.keys() else 0.0\n\n return qty, gain, offset, nodata, undetect\n\n\n@postprocess_import()\ndef import_saf_crri(filename, extent=None, **kwargs):\n \"\"\"\n Import a NetCDF radar rainfall product from the Convective Rainfall Rate\n Intensity (CRRI) product from the Satellite Application Facilities (SAF).\n\n Product description available on http://www.nwcsaf.org/crr_description\n (last visited Jan 26, 2020).\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n extent: scalars (left, right, bottom, top), optional\n The spatial extent specified in data coordinates.\n If None, the full extent is imported.\n\n {extra_kwargs_doc}\n\n Returns\n -------\n out: tuple\n A three-element tuple containing the rainfall field in mm/h, the quality\n field and the metadata imported from the CRRI SAF netcdf file.\n The quality field includes values [1, 2, 4, 8, 16, 24, 32] meaning\n \"nodata\", \"internal_consistency\", \"temporal_consistency\", \"good\",\n \"questionable\", \"bad\", and \"interpolated\", respectively.\n \"\"\"\n if not NETCDF4_IMPORTED:\n raise MissingOptionalDependency(\n \"netCDF4 package is required to import CRRI SAF products \"\n \"but it is not installed\"\n )\n\n geodata = _import_saf_crri_geodata(filename)\n metadata = geodata\n\n if extent:\n xcoord = (\n np.arange(\n metadata[\"x1\"],\n metadata[\"x2\"],\n metadata[\"xpixelsize\"],\n )\n + metadata[\"xpixelsize\"] / 2\n )\n ycoord = (\n np.arange(\n metadata[\"y1\"],\n metadata[\"y2\"],\n metadata[\"ypixelsize\"],\n )\n + metadata[\"ypixelsize\"] / 2\n )\n ycoord = ycoord[::-1] # yorigin = \"upper\"\n idx_x = np.logical_and(xcoord < extent[1], xcoord > extent[0])\n idx_y = np.logical_and(ycoord < extent[3], ycoord > extent[2])\n\n # update geodata\n metadata[\"x1\"] = xcoord[idx_x].min() - metadata[\"xpixelsize\"] / 2\n metadata[\"x2\"] = xcoord[idx_x].max() + metadata[\"xpixelsize\"] / 2\n metadata[\"y1\"] = ycoord[idx_y].min() - metadata[\"ypixelsize\"] / 2\n metadata[\"y2\"] = ycoord[idx_y].max() + metadata[\"ypixelsize\"] / 2\n\n else:\n idx_x = None\n idx_y = None\n\n precip, quality = _import_saf_crri_data(filename, idx_x, idx_y)\n\n metadata[\"transform\"] = None\n metadata[\"zerovalue\"] = np.nanmin(precip)\n metadata[\"threshold\"] = _get_threshold_value(precip)\n\n return precip, quality, metadata\n\n\ndef _import_saf_crri_data(filename, idx_x=None, idx_y=None):\n ds_rainfall = netCDF4.Dataset(filename)\n if \"crr_intensity\" in ds_rainfall.variables.keys():\n if idx_x is not None:\n data = np.asarray(ds_rainfall.variables[\"crr_intensity\"][idx_y, idx_x])\n quality = np.asarray(ds_rainfall.variables[\"crr_quality\"][idx_y, idx_x])\n else:\n data = np.asarray(ds_rainfall.variables[\"crr_intensity\"][:])\n quality = np.asarray(ds_rainfall.variables[\"crr_quality\"][:])\n precipitation = np.where(data == 65535, np.nan, data)\n else:\n precipitation = None\n quality = None\n ds_rainfall.close()\n\n return precipitation, quality\n\n\ndef _import_saf_crri_geodata(filename):\n geodata = {}\n\n ds_rainfall = netCDF4.Dataset(filename)\n\n # get projection\n projdef = ds_rainfall.getncattr(\"gdal_projection\")\n geodata[\"projection\"] = projdef\n\n # get x1, y1, x2, y2, xpixelsize, ypixelsize, yorigin\n geotable = ds_rainfall.getncattr(\"gdal_geotransform_table\")\n xmin = ds_rainfall.getncattr(\"gdal_xgeo_up_left\")\n xmax = ds_rainfall.getncattr(\"gdal_xgeo_low_right\")\n ymin = ds_rainfall.getncattr(\"gdal_ygeo_low_right\")\n ymax = ds_rainfall.getncattr(\"gdal_ygeo_up_left\")\n xpixelsize = abs(geotable[1])\n ypixelsize = abs(geotable[5])\n geodata[\"x1\"] = xmin\n geodata[\"y1\"] = ymin\n geodata[\"x2\"] = xmax\n geodata[\"y2\"] = ymax\n geodata[\"xpixelsize\"] = xpixelsize\n geodata[\"ypixelsize\"] = ypixelsize\n geodata[\"cartesian_unit\"] = \"m\"\n geodata[\"yorigin\"] = \"upper\"\n\n # get the accumulation period\n geodata[\"accutime\"] = None\n\n # get the unit of precipitation\n geodata[\"unit\"] = ds_rainfall.variables[\"crr_intensity\"].units\n\n # get institution\n geodata[\"institution\"] = ds_rainfall.getncattr(\"institution\")\n\n ds_rainfall.close()\n\n return geodata\n\n\n@postprocess_import()\ndef import_dwd_hdf5(filename, qty=\"RATE\", **kwargs):\n \"\"\"\n Import a DWD precipitation product field (and optionally the quality\n field) from an HDF5 file conforming to the ODIM specification.\n\n Parameters\n ----------\n filename : str\n Name of the file to import.\n qty : {'RATE', 'ACRR', 'DBZH'}, optional\n Quantity to read from the file. The currently supported identifiers are:\n\n - 'RATE': instantaneous rain rate (mm/h)\n - 'ACRR': hourly rainfall accumulation (mm)\n - 'DBZH': maximum reflectivity (dBZ)\n\n The default is 'RATE'.\n\n {extra_kwargs_doc}\n\n Returns\n -------\n data : np.ndarray\n The requested precipitation product imported from the HDF5 file.\n quality : None\n Placeholder for quality field (not yet implemented).\n metadata : dict\n Dictionary containing geospatial metadata with the following keys:\n\n projection : str\n PROJ.4 string defining the stereographic projection.\n ll_lon, ll_lat : float\n Coordinates of the lower-left corner.\n ur_lon, ur_lat : float\n Coordinates of the upper-right corner.\n x1, y1 : float\n Cartesian coordinates of the lower-left corner.\n x2, y2 : float\n Cartesian coordinates of the upper-right corner.\n xpixelsize, ypixelsize : float\n Pixel size in meters.\n cartesian_unit : str\n Unit of the coordinate system (meters).\n yorigin : {'lower'}\n Origin of the y-axis.\n institution : {'DWD', 'DWD Radolan'}\n Originating institution.\n accutime : int\n Accumulation period of the requested precipitation product.\n unit : str\n Unit of the data.\n transform : str\n Logarithmic transformation applied.\n zerovalue : float\n Value representing no echo.\n threshold : float\n Precipitation threshold.\n \"\"\"\n if not H5PY_IMPORTED:\n raise MissingOptionalDependency(\n \"h5py package is required to import \"\n \"radar reflectivity composites using ODIM HDF5 specification \"\n \"but it is not installed\"\n )\n\n if not PYPROJ_IMPORTED:\n raise MissingOptionalDependency(\n \"pyproj package is required to import \"\n \"DWD's radar reflectivity composite \"\n \"but it is not installed\"\n )\n\n if qty not in [\"ACRR\", \"DBZH\", \"RATE\"]:\n raise ValueError(\n \"unknown quantity %s: the available options are 'ACRR', 'DBZH' and 'RATE'\"\n )\n\n # Open file\n f = h5py.File(filename, \"r\")\n precip = None\n quality = None\n\n # Read data recursively\n file_content = {}\n _read_hdf5_cont(f, file_content)\n f.close()\n\n # Read attributes\n data_prop = {}\n _get_whatgrp(file_content, data_prop)\n\n # Get data as well as no data and no echo masks\n arr = file_content[\"dataset1\"][\"data1\"][\"data\"]\n mask_n = arr == data_prop[\"nodata\"]\n mask_u = arr == data_prop[\"undetect\"]\n mask = np.logical_and(~mask_u, ~mask_n)\n\n # If the requested quantity in the file\n # Transform precipitation data by gain and offset\n if data_prop[\"quantity\"] == qty:\n precip = np.empty(arr.shape)\n precip[mask] = arr[mask] * data_prop[\"gain\"] + data_prop[\"offset\"]\n if qty != \"DBZH\":\n precip[mask_u] = data_prop[\"offset\"]\n else:\n # Set the no echo value manually to -32.5\n # if the file contains horizontal reflectivity\n precip[mask_u] = -32.5\n precip[mask_n] = np.nan\n # Get possible information about data quality\n elif data_prop[\"quantity\"] == \"QIND\":\n quality = np.empty(arr.shape, dtype=float)\n quality[mask] = arr[mask]\n quality[~mask] = np.nan\n\n if precip is None:\n raise IOError(\"requested quantity %s not found\" % qty)\n\n # Get the projection and grid information from the HDF5 file\n pr = pyproj.Proj(file_content[\"where\"][\"projdef\"])\n ll_x, ll_y = pr(\n file_content[\"where\"][\"LL_lon\"],\n file_content[\"where\"][\"LL_lat\"],\n )\n ur_x, ur_y = pr(\n file_content[\"where\"][\"UR_lon\"],\n file_content[\"where\"][\"UR_lat\"],\n )\n\n # Determine domain corners in geographic and carthesian coordinates\n if len([k for k in file_content[\"where\"].keys() if \"_lat\" in k]) == 4:\n lr_x, lr_y = pr(\n file_content[\"where\"][\"LR_lon\"],\n file_content[\"where\"][\"LR_lat\"],\n )\n ul_x, ul_y = pr(\n file_content[\"where\"][\"UL_lon\"],\n file_content[\"where\"][\"UL_lat\"],\n )\n x1 = min(ll_x, ul_x)\n y1 = min(ll_y, lr_y)\n x2 = max(lr_x, ur_x)\n y2 = max(ul_y, ur_y)\n else:\n x1 = ll_x\n y1 = ll_y\n x2 = ur_x\n y2 = ur_y\n\n # Get the grid cell size\n if (\n \"where\" in file_content[\"dataset1\"].keys()\n and \"xscale\" in file_content[\"dataset1\"][\"where\"].keys()\n ):\n xpixelsize = file_content[\"dataset1\"][\"where\"][\"xscale\"]\n ypixelsize = file_content[\"dataset1\"][\"where\"][\"yscale\"]\n elif \"xscale\" in file_content[\"where\"].keys():\n xpixelsize = file_content[\"where\"][\"xscale\"]\n ypixelsize = file_content[\"where\"][\"yscale\"]\n else:\n xpixelsize = None\n ypixelsize = None\n\n # Get the unit and transform\n if qty == \"ACRR\":\n unit = \"mm\"\n transform = None\n elif qty == \"DBZH\":\n unit = \"dBZ\"\n transform = \"dB\"\n else:\n unit = \"mm/h\"\n transform = None\n\n # Extract the time step\n startdate = datetime.datetime.strptime(\n file_content[\"dataset1\"][\"what\"][\"startdate\"]\n + file_content[\"dataset1\"][\"what\"][\"starttime\"],\n \"%Y%m%d%H%M%S\",\n )\n enddate = datetime.datetime.strptime(\n file_content[\"dataset1\"][\"what\"][\"enddate\"]\n + file_content[\"dataset1\"][\"what\"][\"endtime\"],\n \"%Y%m%d%H%M%S\",\n )\n accutime = (enddate - startdate).total_seconds() / 60.0\n\n # Finally, fill out the metadata\n metadata = {\n \"projection\": file_content[\"where\"][\"projdef\"],\n \"ll_lon\": file_content[\"where\"][\"LL_lon\"],\n \"ll_lat\": file_content[\"where\"][\"LL_lat\"],\n \"ur_lon\": file_content[\"where\"][\"UR_lon\"],\n \"ur_lat\": file_content[\"where\"][\"UR_lat\"],\n \"x1\": x1,\n \"y1\": y1,\n \"x2\": x2,\n \"y2\": y2,\n \"xpixelsize\": xpixelsize,\n \"ypixelsize\": ypixelsize,\n \"cartesian_unit\": \"m\",\n \"yorigin\": \"upper\",\n \"institution\": file_content[\"what\"][\"source\"],\n \"accutime\": accutime,\n \"unit\": unit,\n \"transform\": transform,\n \"zerovalue\": np.nanmin(precip),\n \"threshold\": _get_threshold_value(precip),\n }\n\n metadata.update(kwargs)\n\n f.close()\n\n return precip, quality, metadata\n\n\ndef _read_hdf5_cont(f, d):\n \"\"\"\n Recursively read nested dictionaries from a HDF5 file.\n\n\n Parameters:\n -----------\n f : h5py.Group or h5py.File\n The current group or file object from which to read data.\n d : dict\n The dictionary to populate with the contents of the HDF5 group.\n\n Returns:\n --------\n None.\n \"\"\"\n # Set simple types of hdf content\n group_type = h5py._hl.group.Group\n\n for key, value in f.items():\n if isinstance(value, group_type):\n d[key] = {}\n if len(list(value.items())) > 0:\n # Recurse into non-empty group\n _read_hdf5_cont(value, d[key])\n else:\n # Handle empty group with attributes\n d[key] = {attr: value.attrs[attr] for attr in value.attrs}\n d[key] = {\n k: (v.decode() if isinstance(v, np.bytes_) else v)\n for k, v in d[key].items()\n }\n\n else:\n\n # Save h5py.Dataset by group name\n d[key] = np.array(value)\n\n return\n\n\ndef _get_whatgrp(d, g):\n \"\"\"\n Recursively get attributes of the what group containing\n the scaling properties.\n\n Parameters:\n -----------\n d : dict\n Dictionary including content of an ODIM compliant\n HDF5 file.\n g : dict\n Dictionary containing attributes of what group.\n\n\n Returns:\n --------\n None.\n \"\"\"\n if \"what\" in d.keys():\n # Searching for the corresponding what group\n # that contains the scaling properties\n if \"gain\" in d[\"what\"].keys():\n g.update(d[\"what\"])\n else:\n k = [k for k in d.keys() if \"data\" in k][0]\n _get_whatgrp(d[k], g)\n else:\n raise DataModelError(\n \"Non ODIM compliant file: \"\n \"no what group found from {} \"\n \"or its subgroups\".format(d.keys()[0])\n )\n return\n\n\n@postprocess_import()\ndef import_dwd_radolan(filename, product_name):\n \"\"\"\n Import a RADOLAN precipitation product from a binary file.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n product_name: {'WX','RX','EX','RY','RW','AY','RS','YW','WN'}\n The specific product to read from the file. Please see\n https://www.dwd.de/DE/leistungen/radolan/radolan_info/\n radolan_radvor_op_komposit_format_pdf.pdf\n for a detailed description.\n\n {extra_kwargs_doc}\n\n Returns\n -------\n tuple\n A tuple containing:\n - data : np.ndarray\n The desired precipitation product in mm/h imported from a RADOLAN file\n - quality : None\n - metadata : dict\n Dictionary containing geospatial metadata such as:\n - 'projection': PROJ.4 string defining the stereographic projection.\n - 'xpixelsize', 'ypixelsize': Pixel size in meters.\n - 'cartesian_unit': Unit of the coordinate system (meters).\n - 'yorigin': Origin of the y-axis ('upper').\n - 'x1', 'y1': Coordinates of the lower-left corner.\n - 'x2', 'y2': Coordinates of the upper-right corner.\n \"\"\"\n # Determine file size and header size\n size_file = os.path.getsize(filename)\n size_data = np.round(size_file, -3)\n size_header = size_file - size_data\n\n # Open file and read header\n f = open(filename, \"rb\")\n header = f.read(size_header).decode(\"utf-8\")\n\n # Get product name from header\n product = header[:2]\n\n # Check if its the desired product\n assert product == product_name, \"Product not in File!\"\n\n # Distinguish between products saved with 8 or 16bit\n product_cat1 = np.array([\"WX\", \"RX\"])\n product_cat2 = np.array([\"RY\", \"RW\", \"YW\"])\n\n # Determine byte size and data type\n nbyte = 1 if product in product_cat1 else 2\n signed = \"B\" if product in product_cat1 else \"H\"\n\n # Extract the scaling factor and grid dimensions\n fac = int(header.split(\"E-\")[1].split(\"INT\")[0])\n dimsplit = header.split(\"x\")\n dims = np.array((dimsplit[0][-4:], dimsplit[1][:4]), dtype=int)[::-1]\n\n # Read binary data\n data = array.array(signed)\n data.fromfile(f, size_data // nbyte)\n f.close()\n\n # Reshape and transpose data to match grid layout\n data = np.array(np.reshape(data, dims, order=\"F\"), dtype=float).T\n\n # Define no-echo values based on product type\n if product == \"SF\":\n no_echo_value = 0.0\n elif product in product_cat2:\n no_echo_value = -0.01\n else:\n no_echo_value = -32.5\n\n # Apply scaling and handle missing data\n if product in product_cat1:\n data[data >= 249] = np.nan\n data = data / 2.0 + no_echo_value\n elif product in product_cat2:\n data, no_data_mask = _identify_info_bits(data)\n if product == \"AY\":\n data = (10 ** (-fac)) * data / 2.0 + no_echo_value\n else:\n data = (10 ** (-fac)) * data + no_echo_value\n else:\n data, no_data_mask = _identify_info_bits(data)\n data = (10 ** (-fac)) * data / 2.0 + no_echo_value\n\n # Mask out no-data values\n data[no_data_mask] = np.nan\n\n # Load geospatial metadata\n geodata = _import_dwd_geodata(product_name, dims)\n metadata = geodata\n\n return data, None, metadata\n\n\ndef _identify_info_bits(data):\n \"\"\"\n Identifies and processes information bits embedded in RADOLAN data values.\n\n This function decodes metadata flags embedded in the RADOLAN data array, such as:\n - Clutter (bit 16)\n - Negative values (bit 15)\n - No data (bit 14)\n - Secondary data (bit 13)\n\n Parameters\n ----------\n data : np.ndarray\n The raw RADOLAN data array containing encoded information bits.\n\n Returns\n -------\n tuple\n A tuple containing:\n - data : np.ndarray\n The cleaned and decoded data array.\n - no_data_mask : np.ndarray\n A boolean mask indicating positions of no-data values.\n \"\"\"\n # Identify and remove clutter (bit 16)\n clutter_mask = data - 2**15 >= 0.0\n data[clutter_mask] = 0\n\n # Identify and convert negative values (bit 15)\n mask = data - 2**14 >= 0.0\n data[mask] -= 2**14\n\n # Identify no-data values (bit 14)\n no_data_mask = data - 2**13 == 2500.0\n if np.sum(no_data_mask) == 0.0:\n no_data_mask = data - 2**13 == 0.0\n data[no_data_mask] = 0.0\n\n # Identify and remove secondary data flag (bit 13)\n data[data - 2**12 > 0.0] -= 2**12\n\n # Apply negative sign to previously marked negative values\n data[mask] *= -1\n\n return data, no_data_mask\n\n\ndef _import_dwd_geodata(product_name, dims):\n \"\"\"\n Generate geospatial metadata for RADOLAN precipitation products.\n\n Since RADOLAN binary files contain only limited projection metadata,\n this function provides hard-coded geospatial definitions and calculates\n the bounding box of the data grid based on the product type and dimensions.\n\n Parameters\n ----------\n product : str\n The RADOLAN product code (e.g., 'RX', 'WX', 'WN', etc.).\n dims : tuple of int\n The dimensions of the data grid (rows, columns).\n\n Returns\n -------\n geodata : dict\n A dictionary containing:\n - 'projection': PROJ.4 string defining the stereographic projection.\n - 'xpixelsize', 'ypixelsize': Pixel size in meters.\n - 'cartesian_unit': Unit of the coordinate system (meters).\n - 'yorigin': Origin of the y-axis ('upper').\n - 'x1', 'y1': Coordinates of the lower-left corner.\n - 'x2', 'y2': Coordinates of the upper-right corner.\n \"\"\"\n geodata = {}\n\n # Define stereographic projection used by RADOLAN\n projdef = (\n \"+a=6378137.0 +b=6356752.0 +proj=stere +lat_ts=60.0 \"\n \"+lat_0=90.0 +lon_0=10.0 +x_0=0 +y_0=0\"\n )\n geodata[\"projection\"] = projdef\n # Spatial resolution of 1km\n geodata[\"xpixelsize\"] = 1000.0\n geodata[\"ypixelsize\"] = 1000.0\n geodata[\"cartesian_unit\"] = \"m\"\n geodata[\"yorigin\"] = \"upper\"\n\n # Define product categories\n product_cat1 = [\"RX\", \"RY\", \"RW\"]\n product_cat2 = [\"WN\"]\n product_cat3 = [\"WX\", \"YW\"]\n\n # Assign reference coordinates based on product type\n if product_name in product_cat1:\n lon, lat = 3.604382995, 46.95361536\n elif product_name in product_cat2:\n lon, lat = 3.566994635, 45.69642538\n elif product_name in product_cat3:\n lon, lat = 9.0, 51.0\n\n # Project reference coordinates to Cartesian system\n pr = pyproj.Proj(projdef)\n x1, y1 = pr(lon, lat)\n\n # Adjust origin for center-based products\n if product_name in product_cat3:\n x1 -= dims[0] * 1000 // 2\n y1 -= dims[1] * 1000 // 2 - 80000\n\n # Calculate bounding box\n x2 = x1 + dims[0] * 1000\n y2 = y1 + dims[1] * 1000\n\n geodata[\"x1\"] = x1\n geodata[\"y1\"] = y1\n geodata[\"x2\"] = x2\n geodata[\"y2\"] = y2\n\n return geodata\n" }, { "path": "pysteps/io/interface.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.io.interface\n====================\n\nInterface for the io module.\n\n.. currentmodule:: pysteps.io.interface\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nfrom importlib.metadata import entry_points\n\nfrom pysteps.decorators import postprocess_import\nfrom pysteps.io import importers, exporters, interface\nfrom pprint import pprint\n\n_importer_methods = dict(\n bom_rf3=importers.import_bom_rf3,\n dwd_hdf5=importers.import_dwd_hdf5,\n dwd_radolan=importers.import_dwd_radolan,\n fmi_geotiff=importers.import_fmi_geotiff,\n fmi_pgm=importers.import_fmi_pgm,\n knmi_hdf5=importers.import_knmi_hdf5,\n mch_gif=importers.import_mch_gif,\n mch_hdf5=importers.import_mch_hdf5,\n mch_metranet=importers.import_mch_metranet,\n mrms_grib=importers.import_mrms_grib,\n odim_hdf5=importers.import_odim_hdf5,\n opera_hdf5=importers.import_opera_hdf5,\n saf_crri=importers.import_saf_crri,\n)\n\n_exporter_methods = dict(\n geotiff=exporters.initialize_forecast_exporter_geotiff,\n kineros=exporters.initialize_forecast_exporter_kineros,\n netcdf=exporters.initialize_forecast_exporter_netcdf,\n)\n\n\ndef discover_importers():\n \"\"\"\n Search for installed importers plugins in the entrypoint 'pysteps.plugins.importers'\n\n The importers found are added to the `pysteps.io.interface_importer_methods`\n dictionary containing the available importers.\n \"\"\"\n # Backward compatibility with previous entry point 'pysteps.plugins.importers' next to 'pysteps.plugins.importer'\n for entry_point in list(entry_points(group=\"pysteps.plugins.importer\")) + list(\n entry_points(group=\"pysteps.plugins.importers\")\n ):\n _importer = entry_point.load()\n\n importer_function_name = _importer.__name__\n importer_short_name = importer_function_name.replace(\"import_\", \"\")\n\n _postprocess_kws = getattr(_importer, \"postprocess_kws\", dict())\n _importer = postprocess_import(**_postprocess_kws)(_importer)\n if importer_short_name not in _importer_methods:\n _importer_methods[importer_short_name] = _importer\n else:\n RuntimeWarning(\n f\"The importer identifier '{importer_short_name}' is already available in\"\n \"'pysteps.io.interface._importer_methods'.\\n\"\n f\"Skipping {entry_point.module}:{entry_point.attr}\"\n )\n\n if hasattr(importers, importer_function_name):\n RuntimeWarning(\n f\"The importer function '{importer_function_name}' is already an attribute\"\n \"of 'pysteps.io.importers`.\\n\"\n f\"Skipping {entry_point.module}:{entry_point.attr}\"\n )\n else:\n setattr(importers, importer_function_name, _importer)\n\n\ndef importers_info():\n \"\"\"Print all the available importers.\"\"\"\n\n # Importers available in the `io.importers` module\n available_importers = [\n attr for attr in dir(importers) if attr.startswith(\"import_\")\n ]\n\n print(\"\\nImporters available in the pysteps.io.importers module\")\n pprint(available_importers)\n\n # Importers declared in the pysteps.io.get_method interface\n importers_in_the_interface = [\n f.__name__ for f in interface._importer_methods.values()\n ]\n\n print(\"\\nImporters available in the pysteps.io.get_method interface\")\n pprint(\n [\n (short_name, f.__name__)\n for short_name, f in interface._importer_methods.items()\n ]\n )\n\n # Let's use sets to find out if there are importers present in the importer module\n # but not declared in the interface, and viceversa.\n available_importers = set(available_importers)\n importers_in_the_interface = set(importers_in_the_interface)\n\n difference = available_importers ^ importers_in_the_interface\n if len(difference) > 0:\n _diff = available_importers - importers_in_the_interface\n if len(_diff) > 0:\n print(\n \"\\nIMPORTANT:\\nThe following importers are available in pysteps.io.importers module \"\n \"but not in the pysteps.io.get_method interface\"\n )\n pprint(_diff)\n _diff = importers_in_the_interface - available_importers\n if len(_diff) > 0:\n print(\n \"\\nWARNING:\\n\"\n \"The following importers are available in the pysteps.io.get_method \"\n \"interface but not in the pysteps.io.importers module\"\n )\n pprint(_diff)\n\n return available_importers, importers_in_the_interface\n\n\ndef get_method(name, method_type):\n \"\"\"\n Return a callable function for the method corresponding to the given\n name.\n\n Parameters\n ----------\n name: str\n Name of the method. The available options are:\\n\n\n Importers:\n\n .. tabularcolumns:: |p{2cm}|L|\n\n +--------------+------------------------------------------------------+\n | Name | Description |\n +==============+======================================================+\n | bom_rf3 | NefCDF files used in the Boreau of Meterorology |\n | | archive containing precipitation intensity |\n | | composites. |\n +--------------+------------------------------------------------------+\n | dwd_hdf5 | HDF5 file format used by DWD. |\n +--------------+------------------------------------------------------+\n | fmi_geotiff | GeoTIFF files used in the Finnish Meteorological |\n | | Institute (FMI) archive, containing reflectivity |\n | | composites (dBZ). |\n +--------------+------------------------------------------------------+\n | fmi_pgm | PGM files used in the Finnish Meteorological |\n | | Institute (FMI) archive, containing reflectivity |\n | | composites (dBZ). |\n +--------------+------------------------------------------------------+\n | knmi_hdf5 | HDF5 file format used by KNMI. |\n +--------------+------------------------------------------------------+\n | mch_gif | GIF files in the MeteoSwiss (MCH) archive containing |\n | | precipitation composites. |\n +--------------+------------------------------------------------------+\n | mch_hdf5 | HDF5 file format used by MeteoSiss (MCH). |\n +--------------+------------------------------------------------------+\n | mch_metranet | metranet files in the MeteoSwiss (MCH) archive |\n | | containing precipitation composites. |\n +--------------+------------------------------------------------------+\n | mrms_grib | Grib2 files used by the NSSL's MRMS product |\n +--------------+------------------------------------------------------+\n | odim_hdf5 | HDF5 file conforming to the ODIM specification. |\n +--------------+------------------------------------------------------+\n | opera_hdf5 | Wrapper to \"odim_hdf5\" to maintain backward |\n | | compatibility with previous pysteps versions. |\n +--------------+------------------------------------------------------+\n | saf_crri | NetCDF SAF CRRI files containing convective rain |\n | | rate intensity and other |\n +--------------+------------------------------------------------------+\n\n Exporters:\n\n .. tabularcolumns:: |p{2cm}|L|\n\n +-------------+-------------------------------------------------------+\n | Name | Description |\n +=============+=======================================================+\n | geotiff | Export as GeoTIFF files. |\n +-------------+-------------------------------------------------------+\n | kineros | KINEROS2 Rainfall file as specified in |\n | | https://www.tucson.ars.ag.gov/kineros/. |\n | | Grid points are treated as individual rain gauges. |\n | | A separate file is produced for each ensemble member. |\n +-------------+-------------------------------------------------------+\n | netcdf | NetCDF files conforming to the CF 1.7 specification. |\n +-------------+-------------------------------------------------------+\n\n method_type: {'importer', 'exporter'}\n Type of the method (see tables above).\n\n \"\"\"\n\n if isinstance(method_type, str):\n method_type = method_type.lower()\n else:\n raise TypeError(\n \"Only strings supported for for the method_type\"\n + \" argument\\n\"\n + \"The available types are: 'importer' and 'exporter'\"\n ) from None\n\n if isinstance(name, str):\n name = name.lower()\n else:\n raise TypeError(\n \"Only strings supported for the method's names.\\n\"\n + \"Available importers names:\"\n + str(list(_importer_methods.keys()))\n + \"\\nAvailable exporters names:\"\n + str(list(_exporter_methods.keys()))\n ) from None\n\n if method_type == \"importer\":\n methods_dict = _importer_methods\n elif method_type == \"exporter\":\n methods_dict = _exporter_methods\n else:\n raise ValueError(\n \"Unknown method type {}\\n\".format(name)\n + \"The available types are: 'importer' and 'exporter'\"\n ) from None\n\n try:\n return methods_dict[name]\n except KeyError:\n raise ValueError(\n \"Unknown {} method {}\\n\".format(method_type, name)\n + \"The available methods are:\"\n + str(list(methods_dict.keys()))\n ) from None\n" }, { "path": "pysteps/io/mch_lut_8bit_Metranet_AZC_V104.txt", "content": " DN Red Gre Blue mm\n 0 0 0 0 0.00\n 1 147 163 160 0.04\n 2 145 161 161 0.07\n 3 143 159 163 0.11\n 4 141 157 165 0.15\n 5 140 156 167 0.19\n 6 138 154 168 0.23\n 7 136 152 170 0.27\n 8 134 150 172 0.32\n 9 133 149 174 0.37\n 10 131 147 175 0.41\n 11 129 145 177 0.46\n 12 127 143 179 0.52\n 13 126 142 181 0.57\n 14 124 140 182 0.62\n 15 122 138 184 0.68\n 16 120 136 186 0.74\n 17 119 135 188 0.80\n 18 119 135 188 0.87\n 19 117 132 190 0.93\n 20 115 130 192 1.00\n 21 113 127 195 1.07\n 22 111 125 197 1.14\n 23 109 122 199 1.22\n 24 107 120 202 1.30\n 25 105 117 204 1.38\n 26 104 115 207 1.46\n 27 102 113 209 1.55\n 28 100 110 211 1.64\n 29 98 108 214 1.73\n 30 96 105 216 1.83\n 31 94 103 218 1.93\n 32 92 100 221 2.03\n 33 90 98 223 2.14\n 34 89 96 226 2.25\n 35 89 96 226 2.36\n 36 85 95 227 2.48\n 37 82 95 229 2.61\n 38 79 95 231 2.73\n 39 76 94 233 2.86\n 40 73 94 235 3.00\n 41 70 94 236 3.14\n 42 67 93 238 3.29\n 43 64 93 240 3.44\n 44 61 93 242 3.59\n 45 58 92 244 3.76\n 46 55 92 245 3.92\n 47 52 92 247 4.10\n 48 49 91 249 4.28\n 49 46 91 251 4.46\n 50 43 91 253 4.66\n 51 40 91 255 4.86\n 52 40 91 255 5.06\n 53 40 95 253 5.28\n 54 40 100 252 5.50\n 55 40 104 250 5.73\n 56 40 109 249 5.96\n 57 40 114 247 6.21\n 58 40 118 246 6.46\n 59 40 123 244 6.73\n 60 40 128 243 7.00\n 61 40 132 242 7.28\n 62 40 137 240 7.57\n 63 40 141 239 7.88\n 64 40 146 237 8.19\n 65 40 151 236 8.51\n 66 40 155 234 8.85\n 67 40 160 233 9.20\n 68 40 165 232 9.56\n 69 40 165 232 9.93\n 70 40 166 230 10.31\n 71 41 168 228 10.71\n 72 42 169 226 11.13\n 73 43 171 225 11.55\n 74 44 173 223 12.00\n 75 44 174 221 12.45\n 76 45 176 219 12.93\n 77 46 178 218 13.42\n 78 47 179 216 13.93\n 79 48 181 214 14.45\n 80 48 182 212 15.00\n 81 49 184 211 15.56\n 82 50 186 209 16.15\n 83 51 187 207 16.75\n 84 52 189 205 17.38\n 85 53 191 204 18.03\n 86 53 191 204 18.70\n 87 54 192 202 19.39\n 88 55 193 201 20.11\n 89 57 195 200 20.86\n 90 58 196 198 21.63\n 91 60 198 197 22.43\n 92 61 199 196 23.25\n 93 63 201 194 24.11\n 94 64 202 193 24.99\n 95 65 203 192 25.91\n 96 67 205 190 26.86\n 97 68 206 189 27.84\n 98 70 208 188 28.86\n 99 71 209 186 29.91\n 100 73 211 185 31.00\n 101 74 212 184 32.13\n 102 76 214 183 33.30\n 103 76 214 183 34.51\n 104 77 215 181 35.76\n 105 78 216 179 37.05\n 106 80 217 178 38.40\n 107 81 219 176 39.79\n 108 83 220 175 41.22\n 109 84 221 173 42.71\n 110 86 222 172 44.25\n 111 87 224 170 45.85\n 112 88 225 168 47.50\n 113 90 226 167 49.21\n 114 91 227 165 50.98\n 115 93 229 164 52.82\n 116 94 230 162 54.72\n 117 96 231 161 56.68\n 118 97 232 159 58.71\n 119 99 234 158 60.82\n 120 99 234 158 63.00\n 121 102 234 156 65.26\n 122 105 235 154 67.59\n 123 109 235 153 70.01\n 124 112 236 151 72.52\n 125 115 237 149 75.11\n 126 119 237 148 77.79\n 127 122 238 146 80.57\n 128 126 239 145 83.45\n 129 129 239 143 86.43\n 130 132 240 141 89.51\n 131 136 240 140 92.70\n 132 139 241 138 96.01\n 133 142 242 136 99.43\n 134 146 242 135 102.97\n 135 149 243 133 106.63\n 136 153 244 132 110.43\n 137 153 244 132 114.36\n 138 155 244 129 118.43\n 139 157 245 127 122.64\n 140 159 246 125 127.00\n 141 161 246 123 131.51\n 142 163 247 121 136.19\n 143 166 248 119 141.02\n 144 168 248 117 146.03\n 145 170 249 115 151.22\n 146 172 250 113 156.59\n 147 174 250 111 162.14\n 148 177 251 109 167.90\n 149 179 252 107 173.85\n 150 181 252 105 180.02\n 151 183 253 103 186.40\n 152 185 254 101 193.01\n 153 188 255 99 199.85\n 154 188 255 99 206.94\n 155 190 255 97 214.27\n 156 192 255 95 221.86\n 157 195 255 94 229.72\n 158 197 255 92 237.86\n 159 199 255 90 246.28\n 160 202 255 89 255.00\n 161 204 255 87 264.03\n 162 207 255 86 273.37\n 163 209 255 84 283.05\n 164 211 255 82 293.07\n 165 214 255 81 303.44\n 166 216 255 79 314.17\n 167 218 255 77 325.29\n 168 221 255 76 336.79\n 169 223 255 74 348.71\n 170 226 255 73 361.04\n 171 226 255 73 373.81\n 172 226 253 69 387.02\n 173 227 251 65 400.71\n 174 227 249 62 414.87\n 175 228 247 58 429.54\n 176 229 245 55 444.72\n 177 229 243 51 460.44\n 178 230 241 48 476.71\n 179 231 240 44 493.56\n 180 231 238 40 511.00\n 181 232 236 37 529.06\n 182 232 234 33 547.75\n 183 233 232 30 567.10\n 184 234 230 26 587.13\n 185 234 228 23 607.87\n 186 235 226 19 629.35\n 187 236 225 16 651.58\n 188 236 225 16 674.59\n 189 237 223 15 698.41\n 190 238 221 14 723.08\n 191 239 219 13 748.61\n 192 240 217 12 775.05\n 193 241 215 11 802.41\n 194 243 214 10 830.75\n 195 244 212 9 860.08\n 196 245 210 9 890.44\n 197 246 208 8 921.88\n 198 247 206 7 954.43\n 199 249 205 6 988.12\n 200 250 203 5 1023.00\n 201 251 201 4 1059.11\n 202 252 199 3 1096.50\n 203 253 197 2 1135.20\n 204 255 196 2 1175.27\n 205 255 196 2 1216.75\n 206 255 192 5 1259.69\n 207 255 189 8 1304.15\n 208 255 186 11 1350.18\n 209 255 183 14 1397.83\n 210 255 180 17 1447.16\n 211 255 176 20 1498.22\n 212 255 173 23 1551.09\n 213 255 170 26 1605.83\n 214 255 167 29 1662.49\n 215 255 164 32 1721.16\n 216 255 160 35 1781.89\n 217 255 157 38 1844.76\n 218 255 154 41 1909.85\n 219 255 151 44 1977.24\n 220 255 148 47 2047.00\n 221 255 145 50 2119.22\n 222 255 145 50 2193.99\n 223 254 140 49 2271.40\n 224 253 136 48 2351.54\n 225 252 132 47 2434.50\n 226 252 127 47 2520.39\n 227 251 123 46 2609.30\n 228 250 119 45 2701.35\n 229 250 114 44 2796.65\n 230 249 110 44 2895.31\n 231 248 106 43 2997.45\n 232 248 101 42 3103.19\n 233 247 97 41 3212.66\n 234 246 93 41 3325.99\n 235 246 88 40 3443.31\n 236 245 84 39 3564.78\n 237 244 80 38 3690.52\n 238 244 76 38 3820.71\n 239 244 76 38 3955.48\n 240 244 73 39 4095.00\n 241 245 70 40 4239.45\n 242 246 67 41 4388.99\n 243 246 65 42 4543.80\n 244 247 62 43 4704.07\n 245 248 59 44 4870.00\n 246 248 57 45 5041.77\n 247 249 54 47 5219.60\n 248 250 51 48 5403.71\n 249 250 49 49 5594.30\n 250 251 46 50 5791.61\n 251 255 0 0 9999.9\n 252 0 255 0 9999.9\n 253 0 0 255 9999.9\n 254 255 255 255 9999.9\n 255 255 33 56 9999.9\n\n" }, { "path": "pysteps/io/mch_lut_8bit_Metranet_v103.txt", "content": "Index R G B mm/h\n0\t255\t255\t255 -10.0\n1\t235\t235\t235 0.0001 \n2\t145\t161\t161 0.10\n3\t143\t159\t163 0.15\n4\t141\t157\t165 0.20\n5\t140\t156\t167 0.25\n6\t138\t154\t168 0.30\n7\t136\t152\t170 0.35\n8\t134\t150\t172 0.40\n9\t133\t149\t174 0.45\n10\t131\t147\t175 0.50\n11\t129\t145\t177 0.55\n12\t127\t143\t179 0.60\n13\t126\t142\t181 0.65\n14\t124\t140\t182 0.70\n15\t122\t138\t184 0.75\n16\t120\t136\t186 0.80\n17\t119\t135\t187 0.85\n18\t119\t134\t188 0.90\n19\t117\t132\t190 0.95\n20\t115\t130\t192 1.00\n21\t113\t127\t195 1.1\n22\t111\t125\t197 1.25\n23\t109\t122\t199 1.35\n24\t107\t120\t202 1.45\n25\t105\t117\t204 1.55\n26\t104\t115\t207 1.65\n27\t102\t113\t209 1.75\n28\t100\t110\t211 1.85\n29\t98\t108\t214 1.95\n30\t96\t105\t216 2.00\n31\t94\t103\t218 3.05\n32\t92\t100\t221 4.05\n33\t90\t98\t223 5.05\n34\t89\t96\t225 6.05\n35\t89\t96\t226 7.05\n36\t85\t95\t227 8.05\n37\t82\t95\t229 9.05\n38\t79\t95\t231 10.05\n39\t76\t94\t233 11.05\n40\t73\t94\t235 12.05\n41\t70\t94\t236 13.05\n42\t67\t93\t238 14.05\n43\t64\t93\t240 15.05\n44\t61\t93\t242 16.05\n45\t58\t92\t244 17.05\n46\t55\t92\t245 18.05\n47\t52\t92\t247 19.05\n48\t49\t91\t249 20.05\n49\t46\t91\t251 21.05\n50\t43\t91\t253 22.05\n51\t40\t91\t254 23.05\n52\t40\t94\t255 24.05\n53\t40\t95\t253 25.05\n54\t40\t100\t252 26.05\n55\t40\t104\t250 27.05\n56\t40\t109\t249 28.05\n57\t40\t114\t247 29.05\n58\t40\t118\t246 30.05\n59\t40\t123\t244 31.05\n60\t40\t128\t243 32.05\n61\t40\t132\t242 33.05\n62\t40\t137\t240 34.05\n63\t40\t141\t239 35.05\n64\t40\t146\t237 36.05\n65\t40\t151\t236 37.05\n66\t40\t155\t234 38.05\n67\t40\t160\t233 39.05\n68\t40\t163\t232 40.05\n69\t40\t165\t231 41.05\n70\t40\t166\t230 42.05\n71\t41\t168\t228 43.05\n72\t42\t169\t226 44.05\n73\t43\t171\t225 45.05\n74\t44\t173\t223 46.05\n75\t44\t174\t221 47.05\n76\t45\t176\t219 48.05\n77\t46\t178\t218 49.05\n78\t47\t179\t216 50.05\n79\t48\t181\t214 51.05\n80\t48\t182\t212 52.05\n81\t49\t184\t211 53.05\n82\t50\t186\t209 54.05\n83\t51\t187\t207 55.05\n84\t52\t189\t205 56.05\n85\t53\t190\t204 57.05\n86\t53\t191\t203 58.05\n87\t54\t192\t202 59.05\n88\t55\t193\t201 60.05\n89\t57\t195\t200 61.05\n90\t58\t196\t198 62.05\n91\t60\t198\t197 63.05\n92\t61\t199\t196 64.05\n93\t63\t201\t194 65.05\n94\t64\t202\t193 66.05\n95\t65\t203\t192 67.05\n96\t67\t205\t190 68.05\n97\t68\t206\t189 69.05\n98\t70\t208\t188 70.05\n99\t71\t209\t186 71.05\n100\t73\t211\t185 72.05\n101\t74\t212\t184 73.05\n102\t76\t213\t183 74.05\n103\t76\t214\t182 75.05\n104\t77\t215\t181 76.05\n105\t78\t216\t179 77.05\n106\t80\t217\t178 78.05\n107\t81\t219\t176 79.05\n108\t83\t220\t175 80.05\n109\t84\t221\t173 81.05\n110\t86\t222\t172 82.05\n111\t87\t224\t170 83.05\n112\t88\t225\t168 84.05\n113\t90\t226\t167 85.05\n114\t91\t227\t165 86.05\n115\t93\t229\t164 87.05\n116\t94\t230\t162 88.05\n117\t96\t231\t161 89.05\n118\t97\t232\t159 90.05\n119\t99\t233\t158 91.05\n120\t99\t234\t157 92.05\n121\t102\t234\t156 93.05\n122\t105\t235\t154 94.05\n123\t109\t235\t153 95.05\n124\t112\t236\t151 96.05\n125\t115\t237\t149 97.05\n126\t119\t237\t148 98.05\n127\t122\t238\t146 99.05\n128\t126\t239\t145 100.05\n129\t129\t239\t143 101.05\n130\t132\t240\t141 102.05\n131\t136\t240\t140 103.05\n132\t139\t241\t138 104.05\n133\t142\t242\t136 105.05\n134\t146\t242\t135 106.05\n135\t149\t243\t133 107.05\n136\t151\t244\t132 108.05\n137\t153\t244\t131 109.05\n138\t155\t244\t129 110.05\n139\t157\t245\t127 111.05\n140\t159\t246\t125 112.05\n141\t161\t246\t123 113.05\n142\t163\t247\t121 114.05\n143\t166\t248\t119 115.05\n144\t168\t248\t117 116.05\n145\t170\t249\t115 117.05\n146\t172\t250\t113 118.05\n147\t174\t250\t111 119.05\n148\t177\t251\t109 120.05\n149\t179\t252\t107 121.05\n150\t181\t252\t105 122.05\n151\t183\t253\t103 123.05\n152\t185\t254\t101 124.05\n153\t186\t255\t99 125.05\n154\t188\t255\t98 126.05\n155\t190\t255\t97 127.05\n156\t192\t255\t95 128.05\n157\t195\t255\t94 129.05\n158\t197\t255\t92 130.05\n159\t199\t255\t90 131.05\n160\t202\t255\t89 132.05\n161\t204\t255\t87 133.05\n162\t207\t255\t86 134.05\n163\t209\t255\t84 135.05\n164\t211\t255\t82 136.05\n165\t214\t255\t81 137.05\n166\t216\t255\t79 138.05\n167\t218\t255\t77 139.05\n168\t221\t255\t76 141.05\n169\t223\t255\t74 142.05\n170\t224\t255\t73 143.05\n171\t226\t255\t71 144.05\n172\t226\t253\t69 145.05\n173\t227\t251\t65 146.05\n174\t227\t249\t62 147.05\n175\t228\t247\t58 148.05\n176\t229\t245\t55 149.05\n177\t229\t243\t51 150.05\n178\t230\t241\t48 151.05\n179\t231\t240\t44 152.05\n180\t231\t238\t40 153.05\n181\t232\t236\t37 154.05\n182\t232\t234\t33 155.05\n183\t233\t232\t30 156.05\n184\t234\t230\t26 157.05\n185\t234\t228\t23 158.05\n186\t235\t226\t19 159.05\n187\t236\t225\t17 160.05\n188\t236\t224\t16 161.05\n189\t237\t223\t15 162.05\n190\t238\t221\t14 163.05\n191\t239\t219\t13 164.05\n192\t240\t217\t12 165.05\n193\t241\t215\t11 166.05\n194\t243\t214\t10 167.05\n195\t244\t212\t9 168.05\n196\t245\t210\t9 169.05\n197\t246\t208\t8 170.05\n198\t247\t206\t7 171.05\n199\t249\t205\t6 172.05\n200\t250\t203\t5 173.05\n201\t251\t201\t4 174.05\n202\t252\t199\t3 175.05\n203\t253\t197\t2 176.05\n204\t254\t196\t2 177.05\n205\t255\t196\t3 178.05\n206\t255\t192\t5 179.05\n207\t255\t189\t8 180.05\n208\t255\t186\t11 181.05\n209\t255\t183\t14 182.05\n210\t255\t180\t17 183.05\n211\t255\t176\t20 184.05\n212\t255\t173\t23 185.05\n213\t255\t170\t26 186.05\n214\t255\t167\t29 187.05\n215\t255\t164\t32 188.05\n216\t255\t160\t35 189.05\n217\t255\t157\t38 190.05\n218\t255\t154\t41 191.05\n219\t255\t151\t44 192.05\n220\t255\t148\t47 193.05\n221\t255\t147\t48 194.05\n222\t255\t145\t50 195.05\n223\t254\t140\t49 196.05\n224\t253\t136\t48 197.05\n225\t252\t132\t47 198.05\n226\t252\t127\t47 199.05\n227\t251\t123\t46 200.05\n228\t250\t119\t45 201.05\n229\t250\t114\t44 202.05\n230\t249\t110\t44 203.05\n231\t248\t106\t43 204.05\n232\t248\t101\t42 205.05\n233\t247\t97\t41 206.05\n234\t246\t93\t41 207.05\n235\t246\t88\t40 208.05\n236\t245\t84\t39 209.05\n237\t244\t80\t38 210.05\n238\t244\t78\t38 220.05\n239\t244\t76\t38 230.05\n240\t244\t73\t39 240.05\n241\t245\t70\t40 250.05\n242\t246\t67\t41 260.05\n243\t246\t65\t42 270.05\n244\t247\t62\t43 280.05\n245\t248\t59\t44 290.05\n246\t248\t57\t45 300.05\n247\t249\t54\t47 310.05\n248\t250\t51\t48 320.05\n249\t250\t49\t49 330.05\n250\t251\t46\t50 340.05\n251\t255\t0\t0 9999.9 \n252\t0\t255\t0 9999.9 \n253\t0 0\t255 9999.9 \n254\t255 255 255 9999.9 \n255\t0\t0\t55 9999.9 \n" }, { "path": "pysteps/io/nowcast_importers.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.io.nowcast_importers\n============================\n\nMethods for importing nowcast files.\n\nThe methods in this module implement the following interface::\n\n import_xxx(filename, optional arguments)\n\nwhere xxx is the name (or abbreviation) of the file format and filename is the\nname of the input file.\n\nThe output of each method is a two-element tuple containing the nowcast array\nand a metadata dictionary.\n\nThe metadata dictionary contains the following mandatory key-value pairs:\n\n.. tabularcolumns:: |p{2cm}|L|\n\n+------------------+----------------------------------------------------------+\n| Key | Value |\n+==================+==========================================================+\n| projection | PROJ.4-compatible projection definition |\n+------------------+----------------------------------------------------------+\n| x1 | x-coordinate of the lower-left corner of the data raster |\n+------------------+----------------------------------------------------------+\n| y1 | y-coordinate of the lower-left corner of the data raster |\n+------------------+----------------------------------------------------------+\n| x2 | x-coordinate of the upper-right corner of the data raster|\n+------------------+----------------------------------------------------------+\n| y2 | y-coordinate of the upper-right corner of the data raster|\n+------------------+----------------------------------------------------------+\n| xpixelsize | grid resolution in x-direction |\n+------------------+----------------------------------------------------------+\n| ypixelsize | grid resolution in y-direction |\n+------------------+----------------------------------------------------------+\n| yorigin | a string specifying the location of the first element in |\n| | the data raster w.r.t. y-axis: |\n| | 'upper' = upper border |\n| | 'lower' = lower border |\n+------------------+----------------------------------------------------------+\n| institution | name of the institution who provides the data |\n+------------------+----------------------------------------------------------+\n| timestep | time step of the input data (minutes) |\n+------------------+----------------------------------------------------------+\n| unit | the physical unit of the data: 'mm/h', 'mm' or 'dBZ' |\n+------------------+----------------------------------------------------------+\n| transform | the transformation of the data: None, 'dB', 'Box-Cox' or |\n| | others |\n+------------------+----------------------------------------------------------+\n| accutime | the accumulation time in minutes of the data, float |\n+------------------+----------------------------------------------------------+\n| threshold | the rain/no rain threshold with the same unit, |\n| | transformation and accutime of the data. |\n+------------------+----------------------------------------------------------+\n| zerovalue | it is the value assigned to the no rain pixels with the |\n| | same unit, transformation and accutime of the data. |\n+------------------+----------------------------------------------------------+\n\nAvailable Nowcast Importers\n---------------------------\n\n.. autosummary::\n :toctree: ../generated/\n\n import_netcdf_pysteps\n\"\"\"\n\nimport numpy as np\n\nfrom pysteps.decorators import postprocess_import\nfrom pysteps.exceptions import MissingOptionalDependency, DataModelError\n\ntry:\n import netCDF4\n\n NETCDF4_IMPORTED = True\nexcept ImportError:\n NETCDF4_IMPORTED = False\n\n\n@postprocess_import(dtype=\"single\")\ndef import_netcdf_pysteps(filename, onerror=\"warn\", **kwargs):\n \"\"\"\n Read a nowcast or an ensemble of nowcasts from a NetCDF file conforming\n to the CF 1.7 specification.\n\n If an error occurs during the import, the corresponding error message\n is shown, and ( None, None ) is returned.\n\n Parameters\n ----------\n filename: str\n Name of the file to import.\n onerror: str\n Define the behavior if an exception is raised during the import.\n - \"warn\": Print an error message and return (None, None)\n - \"raise\": Raise an exception\n\n {extra_kwargs_doc}\n\n Returns\n -------\n precipitation: 2D array, float32\n Precipitation field in mm/h. The dimensions are [latitude, longitude].\n The first grid point (0,0) corresponds to the upper left corner of the\n domain, while (last i, last j) denote the lower right corner.\n metadata: dict\n Associated metadata (pixel sizes, map projections, etc.).\n \"\"\"\n if not NETCDF4_IMPORTED:\n raise MissingOptionalDependency(\n \"netCDF4 package is required to import pysteps netcdf \"\n \"nowcasts but it is not installed\"\n )\n\n onerror = onerror.lower()\n if onerror not in [\"warn\", \"raise\"]:\n raise ValueError(\"'onerror' keyword must be 'warn' or 'raise'.\")\n\n try:\n ds = netCDF4.Dataset(filename, \"r\")\n\n var_names = list(ds.variables.keys())\n\n if \"precip_intensity\" in var_names:\n precip = ds.variables[\"precip_intensity\"]\n unit = \"mm/h\"\n accutime = None\n transform = None\n elif \"precip_accum\" in var_names:\n precip = ds.variables[\"precip_accum\"]\n unit = \"mm\"\n accutime = None\n transform = None\n elif \"hourly_precip_accum\" in var_names:\n precip = ds.variables[\"hourly_precip_accum\"]\n unit = \"mm\"\n accutime = 60.0\n transform = None\n elif \"reflectivity\" in var_names:\n precip = ds.variables[\"reflectivity\"]\n unit = \"dBZ\"\n accutime = None\n transform = \"dB\"\n else:\n raise DataModelError(\n \"Non CF compilant file: \"\n \"the netCDF file does not contain any \"\n \"supported variable name.\\n\"\n \"Supported names: 'precip_intensity', 'hourly_precip_accum', \"\n \"or 'reflectivity'\\n\"\n \"file: \" + filename\n )\n\n precip = precip[...].squeeze().astype(float)\n\n if isinstance(precip, np.ma.MaskedArray):\n invalid_mask = np.ma.getmaskarray(precip)\n precip = precip.data\n precip[invalid_mask] = np.nan\n\n metadata = {}\n\n time_var = ds.variables[\"time\"]\n leadtimes = time_var[:] / 60.0 # minutes leadtime\n metadata[\"leadtimes\"] = leadtimes\n timestamps = netCDF4.num2date(time_var[:], time_var.units)\n metadata[\"timestamps\"] = timestamps\n\n if \"polar_stereographic\" in var_names:\n vn = \"polar_stereographic\"\n\n attr_dict = {}\n for attr_name in ds.variables[vn].ncattrs():\n attr_dict[attr_name] = ds[vn].getncattr(attr_name)\n\n proj_str = _convert_grid_mapping_to_proj4(attr_dict)\n metadata[\"projection\"] = proj_str\n\n # geodata\n metadata[\"xpixelsize\"] = abs(ds.variables[\"x\"][1] - ds.variables[\"x\"][0])\n metadata[\"ypixelsize\"] = abs(ds.variables[\"y\"][1] - ds.variables[\"y\"][0])\n\n xmin = np.min(ds.variables[\"x\"]) - 0.5 * metadata[\"xpixelsize\"]\n xmax = np.max(ds.variables[\"x\"]) + 0.5 * metadata[\"xpixelsize\"]\n ymin = np.min(ds.variables[\"y\"]) - 0.5 * metadata[\"ypixelsize\"]\n ymax = np.max(ds.variables[\"y\"]) + 0.5 * metadata[\"ypixelsize\"]\n\n # TODO: this is only a quick solution\n metadata[\"x1\"] = xmin\n metadata[\"y1\"] = ymin\n metadata[\"x2\"] = xmax\n metadata[\"y2\"] = ymax\n\n metadata[\"yorigin\"] = \"upper\" # TODO: check this\n\n # TODO: Read the metadata to the dictionary.\n if (accutime is None) and (leadtimes.size > 1):\n accutime = leadtimes[1] - leadtimes[0]\n metadata[\"accutime\"] = accutime\n metadata[\"unit\"] = unit\n metadata[\"transform\"] = transform\n metadata[\"zerovalue\"] = np.nanmin(precip)\n if metadata[\"zerovalue\"] == np.nanmax(precip):\n metadata[\"threshold\"] = metadata[\"zerovalue\"]\n else:\n metadata[\"threshold\"] = np.nanmin(precip[precip > metadata[\"zerovalue\"]])\n\n ds.close()\n\n return precip, metadata\n except Exception as er:\n if onerror == \"warn\":\n print(\"There was an error processing the file\", er)\n return None, None\n else:\n raise er\n\n\ndef _convert_grid_mapping_to_proj4(grid_mapping):\n gm_keys = list(grid_mapping.keys())\n\n # TODO: implement more projection types here\n if grid_mapping[\"grid_mapping_name\"] == \"polar_stereographic\":\n proj_str = \"+proj=stere\"\n proj_str += \" +lon_0=%s\" % grid_mapping[\"straight_vertical_longitude_from_pole\"]\n proj_str += \" +lat_0=%s\" % grid_mapping[\"latitude_of_projection_origin\"]\n if \"standard_parallel\" in gm_keys:\n proj_str += \" +lat_ts=%s\" % grid_mapping[\"standard_parallel\"]\n if \"scale_factor_at_projection_origin\" in gm_keys:\n proj_str += \" +k_0=%s\" % grid_mapping[\"scale_factor_at_projection_origin\"]\n proj_str += \" +x_0=%s\" % grid_mapping[\"false_easting\"]\n proj_str += \" +y_0=%s\" % grid_mapping[\"false_northing\"]\n\n return proj_str\n else:\n return None\n" }, { "path": "pysteps/io/readers.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.io.readers\n==================\n\nModule with the reader functions.\n\n.. autosummary::\n :toctree: ../generated/\n\n read_timeseries\n\"\"\"\n\nimport numpy as np\n\n\ndef read_timeseries(inputfns, importer, **kwargs):\n \"\"\"\n Read a time series of input files using the methods implemented in the\n :py:mod:`pysteps.io.importers` module and stack them into a 3d array of\n shape (num_timesteps, height, width).\n\n Parameters\n ----------\n inputfns: tuple\n Input files returned by a function implemented in the\n :py:mod:`pysteps.io.archive` module.\n importer: function\n A function implemented in the :py:mod:`pysteps.io.importers` module.\n kwargs: dict\n Optional keyword arguments for the importer.\n\n Returns\n -------\n out: tuple\n A three-element tuple containing the read data and quality rasters and\n associated metadata. If an input file name is None, the corresponding\n precipitation and quality fields are filled with nan values. If all\n input file names are None or if the length of the file name list is\n zero, a three-element tuple containing None values is returned.\n\n \"\"\"\n\n # check for missing data\n precip_ref = None\n if all(ifn is None for ifn in inputfns):\n return None, None, None\n else:\n if len(inputfns[0]) == 0:\n return None, None, None\n for ifn in inputfns[0]:\n if ifn is not None:\n precip_ref, quality_ref, metadata = importer(ifn, **kwargs)\n break\n\n if precip_ref is None:\n return None, None, None\n\n precip = []\n quality = []\n timestamps = []\n for i, ifn in enumerate(inputfns[0]):\n if ifn is not None:\n precip_, quality_, _ = importer(ifn, **kwargs)\n precip.append(precip_)\n quality.append(quality_)\n timestamps.append(inputfns[1][i])\n else:\n precip.append(precip_ref * np.nan)\n if quality_ref is not None:\n quality.append(quality_ref * np.nan)\n else:\n quality.append(None)\n timestamps.append(inputfns[1][i])\n\n # Replace this with stack?\n precip = np.concatenate([precip_[None, :, :] for precip_ in precip])\n # TODO: Q should be organized as R, but this is not trivial as Q_ can be also None or a scalar\n metadata[\"timestamps\"] = np.array(timestamps)\n\n return precip, quality, metadata\n" }, { "path": "pysteps/motion/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nImplementations of optical flow methods.\"\"\"\n\nfrom .interface import get_method\n" }, { "path": "pysteps/motion/_proesmans.pyx", "content": "# -*- coding: utf-8 -*-\n\n\"\"\"\nCython module for the Proesmans optical flow algorithm\n\"\"\"\n\n#from cython.parallel import parallel, prange\nimport numpy as np\nfrom scipy.ndimage import convolve\n\ncimport cython\ncimport numpy as np\n\nctypedef np.float64_t float64\nctypedef np.intp_t intp\n\nfrom libc.math cimport floor, sqrt\n\ncdef float64 _INTENSITY_SCALE = 1.0 / 255.0\n\ndef _compute_advection_field(float64 [:, :, :] R, lam, intp num_iter,\n intp n_levels):\n R_p = [_construct_image_pyramid(R[0, :, :], n_levels),\n _construct_image_pyramid(R[1, :, :], n_levels)]\n\n cdef intp m = R_p[0][-1].shape[0]\n cdef intp n = R_p[0][-1].shape[1]\n\n cdef np.ndarray[float64, ndim=4] V_cur = np.zeros((2, 2, m, n))\n cdef np.ndarray[float64, ndim=4] V_next\n\n cdef np.ndarray[float64, ndim=3] GAMMA = np.empty((2, R.shape[1], R.shape[2]))\n\n for i in range(n_levels-1, -1, -1):\n _proesmans(np.stack([R_p[0][i], R_p[1][i]]), V_cur, num_iter, lam)\n\n m = R_p[0][i-1].shape[0]\n n = R_p[0][i-1].shape[1]\n\n V_next = np.zeros((2, 2, m, n))\n\n if i > 0:\n _initialize_next_level(V_cur, V_next)\n V_cur = V_next\n\n _compute_consistency_maps(V_cur, GAMMA)\n\n return V_cur, GAMMA\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\n@cython.nonecheck(False)\n@cython.cdivision(True)\ncdef _compute_next_pyramid_level(float64 [:, :] I_src,\n float64 [:, :] I_dest):\n cdef intp dh = I_dest.shape[0]\n cdef intp dw = I_dest.shape[1]\n cdef intp x, y\n\n for y in range(dh):\n for x in range(dw):\n I_dest[y, x] = (I_src[2*y, 2*x] + I_src[2*y, 2*x+1] + \\\n I_src[2*y+1, 2*x] + I_src[2*y+1, 2*x+1]) / 4.0\n\ncdef _construct_image_pyramid(float64 [:, :] R, intp n_levels):\n cdef intp m = R.shape[0]\n cdef intp n = R.shape[1]\n cdef np.ndarray[float64, ndim=2] R_next\n\n R_out = [R]\n cdef float64 [:, :] R_cur = R\n for i in range(1, n_levels):\n R_next = np.zeros((int(m/2), int(n/2)))\n _compute_next_pyramid_level(R_cur, R_next)\n R_cur = R_next\n\n R_out.append(R_cur)\n m = int(m / 2)\n n = int(n / 2)\n\n return R_out\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\n@cython.nonecheck(False)\n@cython.cdivision(True)\ncdef _proesmans(float64 [:, :, :] R, float64 [:, :, :, :] V, intp num_iter,\n float64 lam):\n cdef intp x, y\n cdef intp i, j\n cdef float64 xd, yd\n cdef float64 It\n cdef float64 ic\n cdef float64 gx, gy\n\n cdef intp m = R.shape[1]\n cdef intp n = R.shape[2]\n\n cdef np.ndarray[float64, ndim=4] G = np.zeros((2, 2, R.shape[1], R.shape[2]))\n\n G[0, :, :, :] = _compute_gradients(R[0, :, :])\n G[1, :, :, :] = _compute_gradients(R[1, :, :])\n\n cdef np.ndarray[float64, ndim=3] GAMMA = np.zeros((2, R.shape[1], R.shape[2]))\n cdef float64 v_avg_1, v_avg_2\n cdef float64 v_next_1, v_next_2\n\n cdef float64 [:, :] R_j_1\n cdef float64 [:, :] R_j_2\n cdef float64 [:, :] G_j_1\n cdef float64 [:, :] G_j_2\n cdef float64 [:, :, :] V_j\n cdef float64 [:, :] GAMMA_j\n\n for i in range(num_iter):\n _compute_consistency_maps(V, GAMMA)\n\n for j in range(2):\n R_j_1 = R[j, :, :]\n R_j_2 = R[1-j, :, :]\n G_j_1 = G[j, 0, :, :]\n G_j_2 = G[j, 1, :, :]\n V_j = V[j, :, :, :]\n GAMMA_j = GAMMA[j, :, :]\n\n for y in range(1, m-1):\n #for y in prange(1, m - 1, schedule='static', nogil=True):\n for x in range(1, n-1):\n v_avg_1 = _compute_laplacian(GAMMA_j, V_j, x, y, 0)\n v_avg_2 = _compute_laplacian(GAMMA_j, V_j, x, y, 1)\n\n xd = x + v_avg_1\n yd = y + v_avg_2\n if xd >= 0 and xd < n - 1 and yd >= 0 and yd < m - 1:\n It = (_linear_interpolate(R_j_2, xd, yd) - \\\n R_j_1[y, x]) * _INTENSITY_SCALE\n gx = G_j_1[y, x]\n gy = G_j_2[y, x]\n ic = lam * It / (1.0 + lam * (gx * gx + gy * gy))\n v_next_1 = v_avg_1 - gx * ic\n v_next_2 = v_avg_2 - gy * ic\n else:\n # use consistency-weighted average as the next value\n # if (xd,yd) is outside the image\n v_next_1 = v_avg_1\n v_next_2 = v_avg_2\n\n V_j[0, y, x] = v_next_1\n V_j[1, y, x] = v_next_2\n\n _fill_edges(V[j, :, :, :])\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\n@cython.nonecheck(False)\n@cython.cdivision(True)\ncdef float64 _compute_laplacian(float64 [:, :] gi, float64 [:, :, :] Vi, intp x,\n intp y, intp j): #nogil:\n cdef float64 v\n cdef float64 sumWeights = (gi[y-1, x] + gi[y, x-1] + \\\n gi[y, x+1] + gi[y+1, x]) / 6.0 + \\\n (gi[y-1, x-1] + gi[y-1, x+1] + \\\n gi[y+1, x-1] + gi[y+1, x+1]) / 12.0\n\n if sumWeights > 1e-8:\n v = (gi[y-1, x] * Vi[j, y-1, x] + gi[y, x-1] * Vi[j, y, x-1] + \\\n gi[y, x+1] * Vi[j, y, x+1] + gi[y+1, x] * Vi[j, y+1, x]) / 6.0 + \\\n (gi[y-1, x-1] * Vi[j, y-1, x-1] + gi[y-1, x+1] * Vi[j, y-1, x+1] + \\\n gi[y+1, x-1] * Vi[j, y+1, x-1] + gi[y+1, x+1] * Vi[j, y+1, x+1]) / 12.0\n\n return v / sumWeights\n else:\n return 0.0\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\n@cython.nonecheck(False)\n@cython.cdivision(True)\ncdef void _compute_consistency_maps(float64 [:, :, :, :] V,\n float64 [:, :, :] GAMMA):\n cdef intp x, y\n cdef intp i\n cdef intp m, n\n cdef float64 xd, yd\n cdef float64 ub, vb\n cdef float64 uDiff, vDiff\n cdef float64 c\n cdef float64 c_sum\n cdef intp c_count\n cdef float64 K\n cdef float64 g\n\n cdef float64 [:, :] V11, V12, V21, V22\n\n m = V.shape[2]\n n = V.shape[3]\n\n for i in range(2):\n c_sum = 0.0\n c_count = 0\n\n V11 = V[i, 0, :, :]\n V12 = V[i, 1, :, :]\n V21 = V[1-i, 0, :, :]\n V22 = V[1-i, 1, :, :]\n\n #for y in prange(m, schedule='guided', nogil=True):\n for y in range(m):\n for x in range(n):\n xd = x + V[i, 0, y, x]\n yd = y + V[i, 1, y, x]\n\n if xd >= 0 and yd >= 0 and xd < n and yd < m:\n ub = _linear_interpolate(V21, xd, yd)\n vb = _linear_interpolate(V22, xd, yd)\n\n uDiff = V11[y, x] + ub\n vDiff = V12[y, x] + vb\n\n c = sqrt(uDiff * uDiff + vDiff * vDiff)\n\n GAMMA[i, y, x] = c\n c_sum += c\n c_count += 1\n else:\n GAMMA[i, y, x] = -1.0\n\n if c_count > 0:\n K = 0.9 * c_sum / c_count\n else:\n K = 0.0\n\n #for y in prange(m, schedule='guided', nogil=True):\n for y in range(m):\n for x in range(n):\n if K > 1e-8:\n if GAMMA[i, y, x] >= 0.0:\n g = GAMMA[i, y, x]\n GAMMA[i, y, x] = 1.0 / (1.0 + (g / K) * (g / K))\n else:\n GAMMA[i, y, x] = 1.0\n else:\n GAMMA[i, y, x] = 1.0\n\ncdef np.ndarray[float64, ndim=3] _compute_gradients(float64 [:, :] I):\n # use 3x3 Sobel kernels for computing partial derivatives\n cdef np.ndarray[float64, ndim=2] Kx = np.zeros((3, 3))\n cdef np.ndarray[float64, ndim=2] Ky = np.zeros((3, 3))\n\n Kx[0, 0] = 1.0 / 8.0 * _INTENSITY_SCALE\n Kx[0, 1] = 0.0\n Kx[0, 2] = -1.0 / 8.0 * _INTENSITY_SCALE\n Kx[1, 0] = 2.0 / 8.0 * _INTENSITY_SCALE\n Kx[1, 1] = 0.0\n Kx[1, 2] = -2.0 / 8.0 * _INTENSITY_SCALE\n Kx[2, 0] = 1.0 / 8.0 * _INTENSITY_SCALE\n Kx[2, 1] = 0.0\n Kx[2, 2] = -1.0 / 8.0 * _INTENSITY_SCALE\n\n Ky[0, 0] = 1.0 / 8.0 * _INTENSITY_SCALE\n Ky[0, 1] = 2.0 / 8.0 * _INTENSITY_SCALE\n Ky[0, 2] = 1.0 / 8.0 * _INTENSITY_SCALE\n Ky[1, 0] = 0.0\n Ky[1, 1] = 0.0\n Ky[1, 2] = 0.0\n Ky[2, 0] = -1.0 / 8.0 * _INTENSITY_SCALE\n Ky[2, 1] = -2.0 / 8.0 * _INTENSITY_SCALE\n Ky[2, 2] = -1.0 / 8.0 * _INTENSITY_SCALE\n\n cdef np.ndarray[float64, ndim=3] G = np.zeros((2, I.shape[0], I.shape[1]))\n\n G[0, :, :] = convolve(I, Kx, mode=\"constant\", cval=0.0)\n G[1, :, :] = convolve(I, Ky, mode=\"constant\", cval=0.0)\n\n return G\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\n@cython.nonecheck(False)\n@cython.cdivision(True)\ncdef void _fill_edges(float64 [:, :, :] V): #nogil:\n cdef intp x, y\n cdef intp i\n\n cdef intp m = V.shape[1]\n cdef intp n = V.shape[2]\n\n for i in range(2):\n # top and bottom edges\n for x in range(1, n-1):\n V[i, 0, x] = V[i, 1, x]\n V[i, m - 1, x] = V[i, m - 2, x]\n\n # left and right edges\n for y in range(1, m-1):\n V[i, y, 0] = V[i, y, 1]\n V[i, y, n-1] = V[i, y, n-2]\n\n # corners\n V[i, 0, 0] = V[i, 1, 1]\n V[i, 0, n - 1] = V[i, 1, n - 2]\n V[i, m - 1, 0] = V[i, m - 2, 1]\n V[i, m - 1, n - 1] = V[i, m - 2, n - 2]\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\n@cython.nonecheck(False)\n@cython.cdivision(True)\ncdef _initialize_next_level(float64 [:, :, :, :] V_prev,\n float64 [:, :, :, :] V_next):\n cdef intp m_prev = V_prev.shape[2]\n cdef intp n_prev = V_prev.shape[3]\n\n cdef intp m_next = V_next.shape[2]\n cdef intp n_next = V_next.shape[3]\n\n cdef float64 vxf, vyf\n cdef float64 vxb, vyb\n cdef float64 xc, yc\n cdef intp xci, yci\n cdef intp xn, yn\n\n cdef float64 [:, :] V_prev_1 = V_prev[0, 0, :, :]\n cdef float64 [:, :] V_prev_2 = V_prev[0, 1, :, :]\n cdef float64 [:, :] V_prev_3 = V_prev[1, 0, :, :]\n cdef float64 [:, :] V_prev_4 = V_prev[1, 1, :, :]\n\n for yn in range(m_next):\n yc = yn / 2.0\n yci = yn / 2\n for xn in range(n_next):\n xc = xn / 2.0\n xci = xn / 2\n\n if xn % 2 != 0 or yn % 2 != 0:\n vxf = _linear_interpolate(V_prev_1, xc, yc)\n vyf = _linear_interpolate(V_prev_2, xc, yc)\n vxb = _linear_interpolate(V_prev_3, xc, yc)\n vyb = _linear_interpolate(V_prev_4, xc, yc)\n else:\n if xci > n_prev - 1:\n xci = n_prev - 1\n if yci > m_prev - 1:\n yci = m_prev - 1\n vxf = V_prev[0, 0, yci, xci]\n vyf = V_prev[0, 1, yci, xci]\n vxb = V_prev[1, 0, yci, xci]\n vyb = V_prev[1, 1, yci, xci]\n\n V_next[0, 0, yn, xn] = 2.0 * vxf\n V_next[0, 1, yn, xn] = 2.0 * vyf\n V_next[1, 0, yn, xn] = 2.0 * vxb\n V_next[1, 1, yn, xn] = 2.0 * vyb\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\n@cython.nonecheck(False)\n@cython.cdivision(True)\ncdef float64 _linear_interpolate(float64 [:, :] I, float64 x, float64 y): #nogil:\n cdef intp x0 = int(x)\n cdef intp x1 = x0 + 1\n cdef intp y0 = int(y)\n cdef intp y1 = y0 + 1\n\n if x0 < 0:\n x0 = 0\n if x0 > I.shape[1] - 1:\n x0 = I.shape[1]-1\n if x1 < 0:\n x1 = 0\n if x1 > I.shape[1] - 1:\n x1 = I.shape[1]-1\n if y0 < 0:\n y0 = 0\n if y0 > I.shape[0] - 1:\n y0 = I.shape[0]-1\n if y1 < 0:\n y1 = 0\n if y1 > I.shape[0] - 1:\n y1 = I.shape[0]-1\n\n cdef float64 I_a = I[y0, x0]\n cdef float64 I_b = I[y1, x0]\n cdef float64 I_c = I[y0, x1]\n cdef float64 I_d = I[y1, x1]\n\n cdef float64 w_a = (x1-x) * (y1-y)\n cdef float64 w_b = (x1-x) * (y-y0)\n cdef float64 w_c = (x-x0) * (y1-y)\n cdef float64 w_d = (x-x0) * (y-y0)\n\n return w_a*I_a + w_b*I_b + w_c*I_c + w_d*I_d\n" }, { "path": "pysteps/motion/_vet.pyx", "content": "# -*- coding: utf-8 -*-\n\n\"\"\"\nCython module for morphing and cost functions implementations used in\nin the Variation Echo Tracking Algorithm\n\"\"\"\nfrom cython.parallel import prange, parallel\n\nimport numpy as np\ncimport numpy as np\n\ncimport cython\n\nctypedef np.float64_t float64\nctypedef np.int8_t int8\nctypedef np.intp_t intp\n\nfrom libc.math cimport floor, round\n\ncdef inline float64 float_abs(float64 a) nogil: return a if a > 0. else -a\n\"\"\" Return the absolute value of a float \"\"\"\n\ncdef inline intp int_min(intp a, intp b) nogil: return a if a < b else b\n\ncdef inline intp int_max(intp a, intp b) nogil: return a if a > b else b\n\n@cython.cdivision(True)\ncdef inline float64 _linear_interpolation(float64 x,\n float64 x1,\n float64 x2,\n float64 y1,\n float64 y2) nogil:\n \"\"\"\n Linear interpolation at x.\n y(x) = y1 + (x-x1) * (y2-y1) / (x2-x1)\n \"\"\"\n\n if float_abs(x1 - x2) < 1e-6:\n return y1\n\n return y1 + (x - x1) * (y2 - y1) / (x2 - x1)\n\n@cython.cdivision(True)\ncdef inline float64 _bilinear_interpolation(float64 x,\n float64 y,\n float64 x1,\n float64 x2,\n float64 y1,\n float64 y2,\n float64 q11,\n float64 q12,\n float64 q21,\n float64 q22) nogil:\n \"\"\"https://en.wikipedia.org/wiki/Bilinear_interpolation\"\"\"\n\n cdef float64 f_x_y1, f_x_y2\n\n f_x_y1 = _linear_interpolation(x, x1, x2, q11, q21)\n f_x_y2 = _linear_interpolation(x, x1, x2, q12, q22)\n return _linear_interpolation(y, y1, y2, f_x_y1, f_x_y2)\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\n@cython.nonecheck(False)\n@cython.cdivision(True)\ndef _warp(np.ndarray[float64, ndim=2] image,\n np.ndarray[int8, ndim=2] mask,\n np.ndarray[float64, ndim=3] displacement,\n bint gradient=False):\n \"\"\"\n Morph image by applying a displacement field (Warping).\n \n The new image is created by selecting for each position the values of the\n input image at the positions given by the x and y displacements. \n The routine works in a backward sense. \n The displacement vectors have to refer to their destination.\n \n For more information in Morphing functions see Section 3 in \n `Beezley and Mandel (2008)`_.\n \n Beezley, J. D., & Mandel, J. (2008). \n Morphing ensemble Kalman filters. Tellus A, 60(1), 131-140.\n \n .. _`Beezley and Mandel (2008)`: http://dx.doi.org/10.1111/\\\n j.1600-0870.2007.00275.x\n\n \n The displacement field in x and y directions and the image must have the\n same dimensions.\n \n The morphing is executed in parallel over x axis.\n \n The value of displaced pixels that fall outside the limits takes the \n value of the nearest edge. Those pixels are indicated by values greater\n than 1 in the output mask.\n \n Parameters\n ----------\n \n image : ndarray (ndim = 2)\n Image to morph\n \n displacement : ndarray (ndim = 3)\n Displacement field to be applied (Warping). \n \n The dimensions are:\n displacement [ x (0) or y (1) , \n i index of pixel, j index of pixel ]\n\n gradient : bool, optional\n If True, the gradient of the morphing function is returned.\n\n\n Returns\n -------\n \n image : ndarray (float64 ,ndim = 2)\n Morphed image.\n \n mask : ndarray (int8 ,ndim = 2)\n Invalid values mask. Points outside the boundaries are masked.\n Values greater than 1, indicate masked values.\n\n gradient_values : ndarray (float64 ,ndim = 3), optional\n If gradient keyword is True, the gradient of the function is also\n returned.\n \"\"\"\n\n cdef intp nx = image.shape[0]\n cdef intp ny = image.shape[1]\n\n cdef np.ndarray[float64, ndim = 2] new_image = (\n np.zeros([nx, ny], dtype=np.float64))\n\n cdef np.ndarray[int8, ndim = 2] morphed_mask = (\n np.zeros([nx, ny], dtype=np.int8))\n\n morphed_mask[mask > 0] = 1.0\n\n cdef np.ndarray[float64, ndim = 3] gradient_values = (\n np.zeros([2, nx, ny], dtype=np.float64))\n\n cdef intp x, y\n\n cdef intp x_max_int = nx - 1\n cdef intp y_max_int = ny - 1\n\n cdef float64 x_max_float = x_max_int\n cdef float64 y_max_float = y_max_int\n\n cdef float64 x_float, y_float, dx, dy\n\n cdef intp x_floor\n cdef intp x_ceil\n cdef intp y_floor\n cdef intp y_ceil\n\n cdef float64 f00, f10, f01, f11\n\n for x in prange(nx, schedule='static', nogil=True):\n for y in range(ny):\n\n x_float = ( x) - displacement[0, x, y]\n y_float = ( y) - displacement[1, x, y]\n\n if x_float < 0:\n morphed_mask[x, y] = 1\n x_float = 0\n x_floor = 0\n x_ceil = 0\n\n elif x_float > x_max_float:\n morphed_mask[x, y] = 1\n x_float = x_max_float\n x_floor = x_max_int\n x_ceil = x_max_int\n\n else:\n x_floor = floor(x_float)\n x_ceil = x_floor + 1\n if x_ceil > x_max_int:\n x_ceil = x_max_int\n\n if y_float < 0:\n morphed_mask[x, y] = 1\n y_float = 0\n y_floor = 0\n y_ceil = 0\n elif y_float > y_max_float:\n morphed_mask[x, y] = 1\n y_float = y_max_float\n y_floor = y_max_int\n y_ceil = y_max_int\n else:\n y_floor = floor(y_float)\n y_ceil = y_floor + 1\n if y_ceil > y_max_int:\n y_ceil = y_max_int\n\n dx = x_float - x_floor\n dy = y_float - y_floor\n\n # This assumes that the spacing between grid points=1.\n\n # Bilinear interpolation coeficients\n f00 = image[x_floor, y_floor]\n f10 = image[x_ceil, y_floor] - image[x_floor, y_floor]\n f01 = image[x_floor, y_ceil] - image[x_floor, y_floor]\n f11 = (image[x_floor, y_floor] - image[x_ceil, y_floor]\n - image[x_floor, y_ceil] + image[x_ceil, y_ceil])\n\n # Bilinear interpolation\n new_image[x, y] = f00 + dx * f10 + dy * f01 + dx * dy * f11\n\n if gradient:\n gradient_values[0, x, y] = f10 + dy * f11\n gradient_values[1, x, y] = f01 + dx * f11\n\n f00 = mask[x_floor, y_floor]\n f10 = mask[x_ceil, y_floor] - mask[x_floor, y_floor]\n f01 = mask[x_floor, y_ceil] - mask[x_floor, y_floor]\n f11 = (mask[x_floor, y_floor] - mask[x_ceil, y_floor]\n - mask[x_floor, y_ceil] + mask[x_ceil, y_ceil])\n\n morphed_mask[x, y] = (f00 + dx * f10 + dy * f01\n + dx * dy * f11)\n\n morphed_mask[morphed_mask != 0] = 1\n if gradient:\n return new_image, morphed_mask, gradient_values\n else:\n return new_image, morphed_mask\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\n@cython.nonecheck(False)\n@cython.cdivision(True)\ndef _cost_function(np.ndarray[float64, ndim=3] sector_displacement,\n np.ndarray[float64, ndim=2] template_image,\n np.ndarray[float64, ndim=2] input_image,\n np.ndarray[int8, ndim=2] mask,\n float smooth_gain,\n bint gradient = False):\n \"\"\"\n Variational Echo Tracking Cost function.\n \n This function computes the Variational Echo Tracking (VET) \n Cost function presented by `Laroche and Zawazdki (1995)`_ and used in the \n McGill Algorithm for Prediction by Lagrangian Extrapolation (MAPLE) \n described in\n `Germann and Zawadzki (2002)`_.\n \n \n .. _`Laroche and Zawazdki (1995)`: \\\n http://dx.doi.org/10.1175/1520-0426(1995)012<0721:ROHWFS>2.0.CO;2\n \n .. _`Germann and Zawadzki (2002)`: \\\n http://dx.doi.org/10.1175/1520-0493(2002)130<2859:SDOTPO>2.0.CO;2\n \n \n The cost function is a the sum of the residuals of the squared image \n differences along with a smoothness constrain. \n \n This cost function implementation, supports displacement vector \n sectorization.\n The displacement vector represent the displacement applied to the pixels in\n each individual sector.\n \n This help to reduce the number of degrees of freedom of the cost function \n when hierarchical approaches are used to obtain the minima of \n the cost function (from low resolution to full image resolution).\n For example, in the MAPLE algorithm an Scaling Guess procedure is used to \n find the displacement vectors.\n The echo motion field is retrieved in three runs with increasing resolution.\n The retrieval starts with (left) a uniform field, which is used as a first \n guess to retrieve (middle) the field on a 5 × 5 grid, which in turn is the \n first guess of (right) the final minimization with a 25 × 25 grid\n \n The shape of the sector is deduced from the image shape and the displacement\n vector shape. \n \n IMPORTANT: The number of sectors in each dimension (x and y) must be a \n factor full image size.\n \n The value of displaced pixels that fall outside the limits takes the \n value of the nearest edge.\n \n The cost function is computed in parallel over the x axis.\n \n .. _ndarray: \\\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\n \n Parameters\n ----------\n \n sector_displacement : ndarray_ (ndim=3) \n Array of displacements to apply to each sector. The dimensions are:\n sector_displacement [ x (0) or y (1) displacement, \n i index of sector, j index of sector ] \n \n \n template_image : ndarray_ (ndim=2)\n Input image array where the sector displacement is applied.\n \n input_image : ndarray_\n Image array to be used as reference \n \n smooth_gain : float\n Smoothness constrain gain\n\n mask : ndarray_ (ndim=2)\n Data mask. If is True, the data is marked as not valid and is not\n used in the computations.\n\n gradient : bool, optional\n If True, the gradient of the morphing function is returned.\n\n Returns\n -------\n \n penalty or gradient values.\n\n penalty : float\n Value of the cost function\n\n gradient_values : ndarray (float64 ,ndim = 3), optional\n If gradient keyword is True, the gradient of the function is also\n returned.\n \n \n References\n ----------\n \n Laroche, S., and I. Zawadzki, 1995: \n Retrievals of horizontal winds from single-Doppler clear-air data by methods\n of cross-correlation and variational analysis. \n J. Atmos. Oceanic Technol., 12, 721–738.\n doi: http://dx.doi.org/10.1175/1520-0426(1995)012<0721:ROHWFS>2.0.CO;2\n \n Germann, U. and I. Zawadzki, 2002: \n Scale-Dependence of the Predictability of Precipitation from Continental \n Radar Images.\n Part I: Description of the Methodology. Mon. Wea. Rev., 130, 2859–2873,\n doi: 10.1175/1520-0493(2002)130<2859:SDOTPO>2.0.CO;2. \n \n \"\"\"\n\n cdef intp x_sectors = sector_displacement.shape[1]\n cdef intp y_sectors = sector_displacement.shape[2]\n\n cdef intp x_image_size = template_image.shape[0]\n cdef intp y_image_size = template_image.shape[1]\n\n if x_image_size % x_sectors != 0:\n raise ValueError(\"Error computing cost function.\\n\",\n \"The number of sectors in x axis (axis=0)\"\n + \" don't divide the image size\")\n\n if y_image_size % y_sectors != 0:\n raise ValueError(\"Error computing cost function.\\n\",\n \"The number of sectors in y axis (axis=1) don't\"\n + \" divide the image size\")\n\n cdef intp x_sector_size = (\n (round(x_image_size / x_sectors)))\n\n cdef intp y_sector_size = (\n (round(y_image_size / y_sectors)))\n\n cdef np.ndarray[float64, ndim = 3] displacement = (\n np.zeros([2, x_image_size, y_image_size], dtype=np.float64))\n\n cdef intp i, j, xy, l, m, ll, mm, i_sec, j_sec\n cdef intp l0, m0, l1, m1, i_shift, j_shift, axis\n\n i_shift = (x_sector_size // 2)\n j_shift = (y_sector_size // 2)\n\n #Assume regular grid with constant grid spacing.\n\n cdef np.ndarray[float64, ndim = 1] x\n cdef np.ndarray[float64, ndim = 1] y\n x = np.arange(x_image_size, dtype='float64')\n y = np.arange(y_image_size, dtype='float64')\n\n cdef np.ndarray[float64, ndim = 1] x_guess\n cdef np.ndarray[float64, ndim = 1] y_guess\n\n x_guess = x.reshape((x_sectors, x_sector_size)).mean(axis=1)\n y_guess = y.reshape((y_sectors, y_sector_size)).mean(axis=1)\n\n cdef float64 sector_area\n\n cdef np.ndarray[float64, ndim = 3] interp_coef\n\n interp_coef = np.zeros([4, x_image_size, y_image_size], dtype=np.float64)\n\n cdef np.ndarray[intp, ndim = 1] l_i = np.zeros(x_image_size, dtype=np.intp)\n cdef np.ndarray[intp, ndim = 1] m_j = np.zeros(y_image_size, dtype=np.intp)\n\n cdef np.ndarray[intp, ndim = 1] i_min = np.full(x_sectors,\n x_image_size,\n dtype=np.intp)\n\n cdef np.ndarray[intp, ndim = 1] i_max = np.full(x_sectors,\n x_image_size,\n dtype=np.intp)\n\n cdef np.ndarray[intp, ndim = 1] j_min = np.full(y_sectors,\n y_image_size,\n dtype=np.intp)\n\n cdef np.ndarray[intp, ndim = 1] j_max = np.full(y_sectors,\n y_image_size,\n dtype=np.intp)\n\n ####################################\n # Compute interpolation coefficients\n for i in prange(x_image_size, schedule='static', nogil=True):\n\n l0 = int_min((i - i_shift) // x_sector_size, x_sectors - 2)\n l0 = int_max(l0, 0)\n l1 = l0 + 1\n\n l_i[i] = l0\n\n for j in range(y_image_size):\n m0 = int_min((j - j_shift) // y_sector_size, y_sectors - 2)\n m0 = int_max(m0, 0)\n m1 = m0 + 1\n\n m_j[j] = m0\n\n sector_area = (x_guess[l1] - x_guess[l0]) * (y_guess[m1] - y_guess[m0])\n\n interp_coef[0, i, j] = (x_guess[l1] * y_guess[m1]\n - x[i] * y_guess[m1]\n - x_guess[l1] * y[j]\n + x[i] * y[j]) / sector_area\n\n interp_coef[1, i, j] = (-x_guess[l1] * y_guess[m0]\n + x[i] * y_guess[m0]\n + x_guess[l1] * y[j]\n - x[i] * y[j]) / sector_area\n\n interp_coef[2, i, j] = (-x_guess[l0] * y_guess[m1]\n + x[i] * y_guess[m1]\n + x_guess[l0] * y[j]\n - x[i] * y[j]) / sector_area\n\n interp_coef[3, i, j] = (x_guess[l0] * y_guess[m0]\n - x[i] * y_guess[m0]\n - x_guess[l0] * y[j]\n + x[i] * y[j]) / sector_area\n\n for xy in range(2):\n displacement[xy, i, j] = (\n sector_displacement[xy, l0, m0] * interp_coef[0, i, j]\n + sector_displacement[xy, l0, m1] * interp_coef[1, i, j]\n + sector_displacement[xy, l1, m0] * interp_coef[2, i, j]\n + sector_displacement[xy, l1, m1] * interp_coef[3, i, j]\n )\n\n ##############################################\n # Compute limits used in gradient computations\n for l, i, counts in zip(*np.unique(l_i,\n return_index=True,\n return_counts=True)):\n i_min[l] = i\n i_max[l] = i + counts - 1\n\n for m, j, counts in zip(*np.unique(m_j,\n return_index=True,\n return_counts=True)):\n j_min[m] = j\n j_max[m] = j + counts\n\n cdef np.ndarray[float64, ndim = 2] morphed_image\n cdef np.ndarray[int8, ndim = 2] morph_mask\n cdef np.ndarray[float64, ndim = 3] _gradient_data\n cdef np.ndarray[float64, ndim = 3] grad_residuals\n cdef np.ndarray[float64, ndim = 3] grad_smooth\n\n cdef np.ndarray[float64, ndim = 2] buffer = \\\n np.zeros([x_image_size, y_image_size], dtype=np.float64)\n\n grad_smooth = np.zeros([2, x_sectors, y_sectors], dtype=np.float64)\n\n grad_residuals = np.zeros([2, x_sectors, y_sectors], dtype=np.float64)\n\n cdef float64 residuals = 0\n\n # Compute residual part of the cost function\n if gradient:\n morphed_image, morph_mask, _gradient_data = _warp(template_image,\n mask,\n displacement,\n gradient=True)\n\n morph_mask[mask > 0] = 1\n\n buffer = (2 * (input_image - morphed_image))\n buffer[morph_mask == 1] = 0\n\n _gradient_data[0, :] *= buffer\n _gradient_data[1, :] *= buffer\n\n for l in range(x_sectors): # schedule='dynamic', nogil=True):\n for m in range(y_sectors):\n for i in range(i_min[l], i_max[l]):\n for j in range(j_min[m], j_max[m]):\n grad_residuals[0, l, m] = grad_residuals[0, l, m] + \\\n (_gradient_data[0, i, j]\n * interp_coef[0, i, j])\n\n grad_residuals[1, l, m] = grad_residuals[1, l, m] + \\\n (_gradient_data[1, i, j]\n * interp_coef[0, i, j])\n\n for m in range(1, y_sectors):\n for i in range(i_min[l], i_max[l]):\n for j in range(j_min[m - 1], j_max[m - 1]):\n grad_residuals[0, l, m] = grad_residuals[0, l, m] + \\\n (_gradient_data[0, i, j]\n * interp_coef[1, i, j])\n\n grad_residuals[1, l, m] = grad_residuals[1, l, m] + \\\n (_gradient_data[1, i, j] # TODO: Check this line!\n * interp_coef[1, i, j])\n\n for l in range(1, x_sectors): #, schedule='dynamic', nogil=True):\n for m in range(y_sectors):\n for i in range(i_min[l - 1], i_max[l - 1]):\n for j in range(j_min[m], j_max[m]):\n grad_residuals[0, l, m] += (_gradient_data[0, i, j]\n * interp_coef[2, i, j])\n grad_residuals[1, l, m] += (_gradient_data[1, i, j]\n * interp_coef[2, i, j])\n\n for m in range(1, y_sectors):\n for i in range(i_min[l - 1], i_max[l - 1]):\n for j in range(j_min[m - 1], j_max[m - 1]):\n grad_residuals[0, l, m] += (_gradient_data[0, i, j]\n * interp_coef[3, i, j])\n grad_residuals[1, l, m] += (_gradient_data[1, i, j]\n * interp_coef[3, i, j])\n\n\n else:\n\n morphed_image, morph_mask = _warp(template_image,\n mask,\n displacement,\n gradient=False)\n morph_mask[mask > 0] = 1\n residuals = np.sum((morphed_image - input_image)[morph_mask == 0] ** 2)\n\n # Compute smoothness constraint part of the cost function\n cdef float64 smoothness_penalty = 0\n\n cdef float64 df_dx2 = 0\n cdef float64 df_dxdy = 0\n cdef float64 df_dy2 = 0\n\n cdef float64 inloop_smoothness_penalty\n\n if smooth_gain > 0.:\n\n for axis in range(2): #, schedule='dynamic', nogil=True):\n\n inloop_smoothness_penalty = 0\n\n for l in range(1, x_sectors - 1):\n\n for m in range(1, y_sectors - 1):\n df_dx2 = (sector_displacement[axis, l + 1, m]\n - 2 * sector_displacement[axis, l, m]\n + sector_displacement[axis, l - 1, m])\n\n df_dx2 = df_dx2 / (x_sector_size * x_sector_size)\n\n df_dy2 = (sector_displacement[axis, l, m + 1]\n - 2 * sector_displacement[axis, l, m]\n + sector_displacement[axis, l, m - 1])\n\n df_dy2 = df_dy2 / (y_sector_size * y_sector_size)\n\n df_dxdy = (sector_displacement[axis, l + 1, m + 1]\n - sector_displacement[axis, l + 1, m - 1]\n - sector_displacement[axis, l - 1, m + 1]\n + sector_displacement[axis, l - 1, m - 1])\n df_dxdy = df_dxdy / (4 * x_sector_size * y_sector_size)\n\n if gradient:\n grad_smooth[axis, l, m] -= 2 * df_dx2\n grad_smooth[axis, l + 1, m] += df_dx2\n grad_smooth[axis, l - 1, m] += df_dx2\n\n grad_smooth[axis, l, m] -= 2 * df_dy2\n grad_smooth[axis, l, m - 1] += df_dy2\n grad_smooth[axis, l, m + 1] += df_dy2\n\n grad_smooth[axis, l - 1, m - 1] += df_dxdy\n grad_smooth[axis, l - 1, m + 1] -= df_dxdy\n grad_smooth[axis, l + 1, m - 1] -= df_dxdy\n grad_smooth[axis, l + 1, m + 1] += df_dxdy\n\n inloop_smoothness_penalty = (df_dx2 * df_dx2\n + 2 * df_dxdy * df_dxdy\n + df_dy2 * df_dy2)\n\n smoothness_penalty += inloop_smoothness_penalty\n\n smoothness_penalty *= smooth_gain #* x_sector_size * y_sector_size\n\n if gradient:\n grad_smooth *= 2 * smooth_gain #* x_sector_size * y_sector_size\n\n return grad_residuals + grad_smooth\n else:\n return residuals, smoothness_penalty\n" }, { "path": "pysteps/motion/constant.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.motion.constant\n=======================\n\nImplementation of a constant advection field estimation by maximizing the\ncorrelation between two images.\n\n.. autosummary::\n :toctree: ../generated/\n\n constant\n\"\"\"\n\nimport numpy as np\nimport scipy.optimize as op\nfrom scipy.ndimage import map_coordinates\n\n\ndef constant(R, **kwargs):\n \"\"\"\n Compute a constant advection field by finding a translation vector that\n maximizes the correlation between two successive images.\n\n Parameters\n ----------\n R: array_like\n Array of shape (T,m,n) containing a sequence of T two-dimensional input\n images of shape (m,n). If T > 2, two last elements along axis 0 are used.\n\n Returns\n -------\n out: array_like\n The constant advection field having shape (2, m, n), where out[0, :, :]\n contains the x-components of the motion vectors and out[1, :, :]\n contains the y-components.\n \"\"\"\n m, n = R.shape[1:]\n X, Y = np.meshgrid(np.arange(n), np.arange(m))\n\n def f(v):\n XYW = [Y + v[1], X + v[0]]\n R_w = map_coordinates(\n R[-2, :, :], XYW, mode=\"constant\", cval=np.nan, order=0, prefilter=False\n )\n\n mask = np.logical_and(np.isfinite(R[-1, :, :]), np.isfinite(R_w))\n\n return -np.corrcoef(R[-1, :, :][mask], R_w[mask])[0, 1]\n\n options = {\"initial_simplex\": (np.array([(0, 1), (1, 0), (1, 1)]))}\n result = op.minimize(f, (1, 1), method=\"Nelder-Mead\", options=options)\n\n return np.stack([-result.x[0] * np.ones((m, n)), -result.x[1] * np.ones((m, n))])\n" }, { "path": "pysteps/motion/darts.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.motion.darts\n====================\n\nImplementation of the DARTS algorithm.\n\n.. autosummary::\n :toctree: ../generated/\n\n DARTS\n\"\"\"\n\nimport numpy as np\nimport time\nfrom numpy.linalg import lstsq, svd\n\nfrom pysteps import utils\nfrom pysteps.decorators import check_input_frames\n\n\n@check_input_frames(just_ndim=True)\ndef DARTS(input_images, **kwargs):\n \"\"\"\n Compute the advection field from a sequence of input images by using the\n DARTS method. :cite:`RCW2011`\n\n Parameters\n ----------\n input_images: array-like\n Array of shape (T,m,n) containing a sequence of T two-dimensional input\n images of shape (m,n).\n\n Other Parameters\n ----------------\n N_x: int\n Number of DFT coefficients to use for the input images, x-axis (default=50).\n N_y: int\n Number of DFT coefficients to use for the input images, y-axis (default=50).\n N_t: int\n Number of DFT coefficients to use for the input images, time axis (default=4).\n N_t must be strictly smaller than T.\n M_x: int\n Number of DFT coefficients to compute for the output advection field,\n x-axis (default=2).\n M_y: int\n Number of DFT coefficients to compute for the output advection field,\n y-axis (default=2).\n fft_method: str\n A string defining the FFT method to use, see utils.fft.get_method.\n Defaults to 'numpy'.\n output_type: {\"spatial\", \"spectral\"}\n The type of the output: \"spatial\"=apply the inverse FFT to obtain the\n spatial representation of the advection field, \"spectral\"=return the\n (truncated) DFT representation.\n n_threads: int\n Number of threads to use for the FFT computation. Applicable if\n fft_method is 'pyfftw'.\n verbose: bool\n If True, print information messages.\n lsq_method: {1, 2}\n The method to use for solving the linear equations in the least squares\n sense: 1=numpy.linalg.lstsq, 2=explicit computation of the Moore-Penrose\n pseudoinverse and SVD.\n verbose: bool\n if set to True, it prints information about the program\n\n Returns\n -------\n out: ndarray\n Three-dimensional array (2,m,n) containing the dense x- and y-components\n of the motion field in units of pixels / timestep as given by the input\n array R.\n\n \"\"\"\n\n N_x = kwargs.get(\"N_x\", 50)\n N_y = kwargs.get(\"N_y\", 50)\n N_t = kwargs.get(\"N_t\", 4)\n M_x = kwargs.get(\"M_x\", 2)\n M_y = kwargs.get(\"M_y\", 2)\n fft_method = kwargs.get(\"fft_method\", \"numpy\")\n output_type = kwargs.get(\"output_type\", \"spatial\")\n lsq_method = kwargs.get(\"lsq_method\", 2)\n verbose = kwargs.get(\"verbose\", True)\n\n if N_t >= input_images.shape[0] - 1:\n raise ValueError(\n \"N_t = %d >= %d = T-1, but N_t < T-1 required\"\n % (N_t, input_images.shape[0] - 1)\n )\n\n if output_type not in [\"spatial\", \"spectral\"]:\n raise ValueError(\n \"invalid output_type=%s, must be 'spatial' or 'spectral'\" % output_type\n )\n\n if np.any(~np.isfinite(input_images)):\n raise ValueError(\"the input images contain non-finite values\")\n\n if verbose:\n print(\"Computing the motion field with the DARTS method.\")\n t0 = time.time()\n\n input_images = np.moveaxis(input_images, (0, 1, 2), (2, 0, 1))\n\n fft = utils.get_method(\n fft_method,\n shape=input_images.shape[:2],\n fftn_shape=input_images.shape,\n **kwargs,\n )\n\n T_x = input_images.shape[1]\n T_y = input_images.shape[0]\n T_t = input_images.shape[2]\n\n if verbose:\n print(\"-----\")\n print(\"DARTS\")\n print(\"-----\")\n\n print(\" Computing the FFT of the reflectivity fields...\", end=\"\", flush=True)\n starttime = time.time()\n\n input_images = fft.fftn(input_images)\n\n if verbose:\n print(\"Done in %.2f seconds.\" % (time.time() - starttime))\n\n print(\" Constructing the y-vector...\", end=\"\", flush=True)\n starttime = time.time()\n\n m = (2 * N_x + 1) * (2 * N_y + 1) * (2 * N_t + 1)\n n = (2 * M_x + 1) * (2 * M_y + 1)\n\n y = np.zeros(m, dtype=complex)\n\n k_t, k_y, k_x = np.unravel_index(\n np.arange(m), (2 * N_t + 1, 2 * N_y + 1, 2 * N_x + 1)\n )\n\n for i in range(m):\n k_x_ = k_x[i] - N_x\n k_y_ = k_y[i] - N_y\n k_t_ = k_t[i] - N_t\n\n y[i] = k_t_ * input_images[k_y_, k_x_, k_t_]\n\n if verbose:\n print(\"Done in %.2f seconds.\" % (time.time() - starttime))\n\n A = np.zeros((m, n), dtype=complex)\n B = np.zeros((m, n), dtype=complex)\n\n if verbose:\n print(\" Constructing the H-matrix...\", end=\"\", flush=True)\n starttime = time.time()\n\n c1 = -1.0 * T_t / (T_x * T_y)\n\n kp_y, kp_x = np.unravel_index(np.arange(n), (2 * M_y + 1, 2 * M_x + 1))\n\n for i in range(m):\n k_x_ = k_x[i] - N_x\n k_y_ = k_y[i] - N_y\n k_t_ = k_t[i] - N_t\n\n kp_x_ = kp_x[:] - M_x\n kp_y_ = kp_y[:] - M_y\n\n i_ = k_y_ - kp_y_\n j_ = k_x_ - kp_x_\n\n R_ = input_images[i_, j_, k_t_]\n\n c2 = c1 / T_y * i_\n A[i, :] = c2 * R_\n\n c2 = c1 / T_x * j_\n B[i, :] = c2 * R_\n\n if verbose:\n print(\"Done in %.2f seconds.\" % (time.time() - starttime))\n\n print(\" Solving the linear systems...\", end=\"\", flush=True)\n starttime = time.time()\n\n if lsq_method == 1:\n x = lstsq(np.hstack([A, B]), y, rcond=0.01)[0]\n else:\n x = _leastsq(A, B, y)\n\n if verbose:\n print(\"Done in %.2f seconds.\" % (time.time() - starttime))\n\n h, w = 2 * M_y + 1, 2 * M_x + 1\n\n U = np.zeros((h, w), dtype=complex)\n V = np.zeros((h, w), dtype=complex)\n\n i, j = np.unravel_index(np.arange(h * w), (h, w))\n\n V[i, j] = x[0 : h * w]\n U[i, j] = x[h * w : 2 * h * w]\n\n k_x, k_y = np.meshgrid(np.arange(-M_x, M_x + 1), np.arange(-M_y, M_y + 1))\n\n if output_type == \"spatial\":\n U = np.real(\n fft.ifft2(_fill(U, input_images.shape[0], input_images.shape[1], k_x, k_y))\n )\n V = np.real(\n fft.ifft2(_fill(V, input_images.shape[0], input_images.shape[1], k_x, k_y))\n )\n\n if verbose:\n print(\"--- %s seconds ---\" % (time.time() - t0))\n\n return np.stack([U, V])\n\n\ndef _leastsq(A, B, y):\n M = np.hstack([A, B])\n M_ct = M.conjugate().T\n MM = np.dot(M_ct, M)\n\n U, s, V = svd(MM, full_matrices=False)\n\n mask = s > 0.01 * s[0]\n s = 1.0 / s[mask]\n\n MM_inv = np.dot(\n np.dot(V[: len(s), :].conjugate().T, np.diag(s)), U[:, : len(s)].conjugate().T\n )\n\n return np.dot(MM_inv, np.dot(M_ct, y))\n\n\ndef _fill(X, h, w, k_x, k_y):\n X_f = np.zeros((h, w), dtype=complex)\n X_f[k_y, k_x] = X\n\n return X_f\n" }, { "path": "pysteps/motion/farneback.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.motion.farneback\n========================\n\nThe Farneback dense optical flow module.\n\nThis module implements the interface to the local `Farneback`_ routine\navailable in OpenCV_.\n\n.. _OpenCV: https://opencv.org/\n\n.. _`Farneback`:\\\n https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af\n\n.. autosummary::\n :toctree: ../generated/\n\n farneback\n\"\"\"\n\nimport numpy as np\nfrom numpy.ma.core import MaskedArray\nimport scipy.ndimage as sndi\nimport time\n\nfrom pysteps.decorators import check_input_frames\nfrom pysteps.exceptions import MissingOptionalDependency\nfrom pysteps.utils.images import morph_opening\n\ntry:\n import cv2\n\n CV2_IMPORTED = True\nexcept ImportError:\n CV2_IMPORTED = False\n\n\n@check_input_frames(2)\ndef farneback(\n input_images,\n pyr_scale=0.5,\n levels=3,\n winsize=15,\n iterations=3,\n poly_n=5,\n poly_sigma=1.1,\n flags=0,\n size_opening=3,\n sigma=60.0,\n verbose=False,\n):\n \"\"\"Estimate a dense motion field from a sequence of 2D images using\n the `Farneback`_ optical flow algorithm.\n\n This function computes dense optical flow between each pair of consecutive\n input frames using OpenCV's Farneback method. If more than two frames are\n provided, the motion fields estimated from all consecutive pairs are\n averaged to obtain a single representative advection field.\n\n After the pairwise motion fields are averaged, the resulting motion field\n can optionally be smoothed with a Gaussian filter. In that case, its\n amplitude is rescaled so that the mean motion magnitude is preserved.\n\n .. _OpenCV: https://opencv.org/\n\n .. _`Farneback`:\\\n https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga5d10ebbd59fe09c5f650289ec0ece5af\n\n .. _MaskedArray:\\\n https://docs.scipy.org/doc/numpy/reference/maskedarray.baseclass.html#numpy.ma.MaskedArray\n\n .. _ndarray:\\\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\n\n Parameters\n ----------\n input_images: ndarray_ or MaskedArray_\n Array of shape (T, m, n) containing a sequence of *T* two-dimensional\n input images of shape (m, n). The indexing order in **input_images** is\n assumed to be (time, latitude, longitude).\n\n *T* = 2 is the minimum required number of images.\n With *T* > 2, all the resulting motion vectors are averaged together.\n\n In case of ndarray_, invalid values (Nans or infs) are masked,\n otherwise the mask of the MaskedArray_ is used. Such mask defines a\n region where features are not detected for the tracking algorithm.\n\n pyr_scale : float, optional\n Parameter specifying the image scale (<1) used to build pyramids for\n each image; pyr_scale=0.5 means a classical pyramid, where each next\n layer is twice smaller than the previous one. This and the following\n parameter descriptions are adapted from the original OpenCV\n documentation (see https://docs.opencv.org).\n\n levels : int, optional\n Number of pyramid layers including the initial image; levels=1 means\n that no extra layers are created and only the original images are used.\n\n winsize : int, optional\n Averaging window size; larger values increase the algorithm robustness\n to image noise and give more stable motion estimates. Small windows\n (e.g. 10) lead to unrealistic motion.\n iterations : int, optional\n Number of iterations the algorithm does at each pyramid level.\n \n poly_n : int\n Size of the pixel neighborhood used to find polynomial expansion in\n each pixel; larger values mean that the image will be approximated with\n smoother surfaces, yielding more robust algorithm and more blurred\n motion field, typically poly_n = 5 or 7.\n \n poly_sigma : float\n Standard deviation of the Gaussian that is used to smooth derivatives\n used as a basis for the polynomial expansion; for poly_n=5, you can set\n poly_sigma=1.1, for poly_n=7, a good value would be poly_sigma=1.5.\n \n flags : int, optional\n Operation flags that can be a combination of the following:\n\n OPTFLOW_USE_INITIAL_FLOW uses the input 'flow' as an initial flow\n approximation.\n\n OPTFLOW_FARNEBACK_GAUSSIAN uses the Gaussian winsize x winsize filter\n instead of a box filter of the same size for optical flow estimation;\n usually, this option gives a more accurate flow than with a box filter,\n at the cost of lower speed; normally, winsize for a Gaussian window\n should be set to a larger value to achieve the same level of robustness.\n \n size_opening : int, optional\n Non-OpenCV parameter:\n The structuring element size for the filtering of isolated pixels [px].\n\n sigma : float, optional\n Non-OpenCV parameter:\n The smoothing bandwidth of the motion field. The motion field amplitude\n is adjusted by multiplying by the ratio of average magnitude before and\n after smoothing to avoid damping of the motion field.\n\n verbose: bool, optional\n If set to True, print some information about the program.\n\n Returns\n -------\n out : ndarray_, shape (2,m,n)\n Return the advection field having shape\n (2, m, n), where out[0, :, :] contains the x-components of the motion\n vectors and out[1, :, :] contains the y-components.\n The velocities are in units of pixels / timestep, where timestep is the\n time difference between the two input images.\n Return a zero motion field of shape (2, m, n) when no motion is\n detected.\n \n References\n ----------\n Farnebäck, G.: Two-frame motion estimation based on polynomial expansion, \n In Image Analysis, pages 363–370. Springer, 2003.\n Driedger, N., Mahidjiba, A. and Hortal, A.P. (2022, June 1-8): Evaluation of optical flow\n methods for radar precipitation extrapolation.\n Canadian Meteorological and Oceanographic Society Congress, contributed abstract 11801.\n \"\"\"\n\n if len(input_images.shape) != 3:\n raise ValueError(\n \"input_images has %i dimensions, but a \"\n \"three-dimensional array is expected\" % len(input_images.shape)\n )\n\n input_images = input_images.copy()\n\n if verbose:\n print(\"Computing the motion field with the Farneback method.\")\n t0 = time.time()\n\n if not CV2_IMPORTED:\n raise MissingOptionalDependency(\n \"OpenCV (cv2) is required for the Farneback optical flow method, but it is not installed\"\n )\n\n nr_pairs = input_images.shape[0] - 1\n domain_size = (input_images.shape[1], input_images.shape[2])\n u_sum = np.zeros(domain_size)\n v_sum = np.zeros(domain_size)\n for n in range(nr_pairs):\n # extract consecutive images\n prvs_img = input_images[n, :, :].copy()\n next_img = input_images[n + 1, :, :].copy()\n\n # Check if a MaskedArray is used. If not, mask the ndarray\n if not isinstance(prvs_img, MaskedArray):\n prvs_img = np.ma.masked_invalid(prvs_img)\n np.ma.set_fill_value(prvs_img, prvs_img.min())\n\n if not isinstance(next_img, MaskedArray):\n next_img = np.ma.masked_invalid(next_img)\n np.ma.set_fill_value(next_img, next_img.min())\n\n # scale between 0 and 255\n im_min = prvs_img.min()\n im_max = prvs_img.max()\n if (im_max - im_min) > 1e-8:\n prvs_img = (prvs_img.filled() - im_min) / (im_max - im_min) * 255\n else:\n prvs_img = prvs_img.filled() - im_min\n\n im_min = next_img.min()\n im_max = next_img.max()\n if (im_max - im_min) > 1e-8:\n next_img = (next_img.filled() - im_min) / (im_max - im_min) * 255\n else:\n next_img = next_img.filled() - im_min\n\n # convert to 8-bit\n prvs_img = np.ndarray.astype(prvs_img, \"uint8\")\n next_img = np.ndarray.astype(next_img, \"uint8\")\n\n # remove small noise with a morphological operator (opening)\n if size_opening > 0:\n prvs_img = morph_opening(prvs_img, prvs_img.min(), size_opening)\n next_img = morph_opening(next_img, next_img.min(), size_opening)\n\n flow = cv2.calcOpticalFlowFarneback(\n prvs_img,\n next_img,\n None,\n pyr_scale,\n levels,\n winsize,\n iterations,\n poly_n,\n poly_sigma,\n flags,\n )\n\n fa, fb = np.dsplit(flow, 2)\n u_sum += fa.reshape(domain_size)\n v_sum += fb.reshape(domain_size)\n\n # Compute the average motion field\n u = u_sum / nr_pairs\n v = v_sum / nr_pairs\n\n # Smoothing\n if sigma > 0:\n uv2 = u * u + v * v # squared magnitude of motion field\n us = sndi.gaussian_filter(u, sigma, mode=\"nearest\")\n vs = sndi.gaussian_filter(v, sigma, mode=\"nearest\")\n uvs2 = us * us + vs * vs # squared magnitude of smoothed motion field\n\n mean_uv2 = np.nanmean(uv2)\n mean_uvs2 = np.nanmean(uvs2)\n if mean_uvs2 > 0:\n mult = np.sqrt(mean_uv2 / mean_uvs2)\n else:\n mult = 1.0\n else:\n mult = 1.0\n us = u\n vs = v\n if verbose:\n print(\"mult factor of smoothed motion field=\", mult)\n\n UV = np.stack([us * mult, vs * mult])\n\n if verbose:\n print(\"--- %s seconds ---\" % (time.time() - t0))\n\n return UV\n" }, { "path": "pysteps/motion/interface.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.motion.interface\n========================\n\nInterface for the motion module. It returns a callable optical flow routine for\ncomputing the motion field.\n\nThe methods in the motion module implement the following interface:\n\n ``motion_method(precip, **keywords)``\n\nwhere precip is a (T,m,n) array containing a sequence of T two-dimensional input\nimages of shape (m,n). The first dimension represents the images time dimension\nand the value of T depends on the type of the method.\n\nThe output is a three-dimensional array (2,m,n) containing the dense x- and\ny-components of the motion field in units of pixels / timestep as given by the\ninput array R.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nimport numpy as np\n\nfrom pysteps.motion.constant import constant\nfrom pysteps.motion.darts import DARTS\nfrom pysteps.motion.lucaskanade import dense_lucaskanade\nfrom pysteps.motion.proesmans import proesmans\nfrom pysteps.motion.vet import vet\nfrom pysteps.motion.farneback import farneback\n\n_methods = dict()\n_methods[\"constant\"] = constant\n_methods[\"lk\"] = dense_lucaskanade\n_methods[\"lucaskanade\"] = dense_lucaskanade\n_methods[\"darts\"] = DARTS\n_methods[\"proesmans\"] = proesmans\n_methods[\"vet\"] = vet\n_methods[\"farneback\"] = farneback\n_methods[None] = lambda precip, *args, **kw: np.zeros(\n (2, precip.shape[1], precip.shape[2])\n)\n\n\ndef get_method(name):\n \"\"\"\n Return a callable function for the optical flow method corresponding to\n the given name. The available options are:\\n\n\n +--------------------------------------------------------------------------+\n | Python-based implementations |\n +-------------------+------------------------------------------------------+\n | Name | Description |\n +===================+======================================================+\n | None | returns a zero motion field |\n +-------------------+------------------------------------------------------+\n | constant | constant advection field estimated by maximizing the |\n | | correlation between two images |\n +-------------------+------------------------------------------------------+\n | darts | implementation of the DARTS method of Ruzanski et |\n | | al. (2011) |\n +-------------------+------------------------------------------------------+\n | lucaskanade | OpenCV implementation of the Lucas-Kanade method |\n | | with interpolated motion vectors for areas with no |\n | | precipitation |\n +-------------------+------------------------------------------------------+\n | proesmans | the anisotropic diffusion method of Proesmans et |\n | | al. (1994) |\n +-------------------+------------------------------------------------------+\n | vet | implementation of the VET method of |\n | | Laroche and Zawadzki (1995) and |\n | | Germann and Zawadzki (2002) |\n +-------------------+------------------------------------------------------+\n | farneback | OpenCV implementation of the Farneback (2003) method.|\n +-------------------+------------------------------------------------------+\n\n +--------------------------------------------------------------------------+\n | Methods implemented in C (these require separate compilation and linkage)|\n +-------------------+------------------------------------------------------+\n | Name | Description |\n +===================+======================================================+\n | brox | implementation of the variational method of |\n | | Brox et al. (2004) from IPOL |\n | | (http://www.ipol.im/pub/art/2013/21) |\n +-------------------+------------------------------------------------------+\n | clg | implementation of the Combined Local-Global (CLG) |\n | | method of Bruhn et al., 2005 from IPOL |\n | | (http://www.ipol.im/pub/art/2015/44) |\n +-------------------+------------------------------------------------------+\n\n \"\"\"\n\n if isinstance(name, str):\n name = name.lower()\n\n if name in [\"brox\", \"clg\"]:\n raise NotImplementedError(\"Method %s not implemented\" % name)\n else:\n try:\n motion_method = _methods[name]\n return motion_method\n except KeyError:\n raise ValueError(\n \"Unknown method {}\\n\".format(name)\n + \"The available methods are:\"\n + str(list(_methods.keys()))\n ) from None\n" }, { "path": "pysteps/motion/lucaskanade.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.motion.lucaskanade\n==========================\n\nThe Lucas-Kanade (LK) local feature tracking module.\n\nThis module implements the interface to the local `Lucas-Kanade`_ routine\navailable in OpenCV_.\n\nFor its dense method, it additionally interpolates the sparse vectors over a\nregular grid to return a motion field.\n\n.. _OpenCV: https://opencv.org/\n\n.. _`Lucas-Kanade`:\\\n https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323\n\n.. autosummary::\n :toctree: ../generated/\n\n dense_lucaskanade\n\"\"\"\n\nimport numpy as np\nfrom numpy.ma.core import MaskedArray\n\nfrom pysteps.decorators import check_input_frames\n\nfrom pysteps import utils, feature\nfrom pysteps.tracking.lucaskanade import track_features\nfrom pysteps.utils.cleansing import decluster, detect_outliers\nfrom pysteps.utils.images import morph_opening\n\nimport time\n\n\n@check_input_frames(2)\ndef dense_lucaskanade(\n input_images,\n lk_kwargs=None,\n fd_method=\"shitomasi\",\n fd_kwargs=None,\n interp_method=\"idwinterp2d\",\n interp_kwargs=None,\n dense=True,\n nr_std_outlier=3,\n k_outlier=30,\n size_opening=3,\n decl_scale=20,\n verbose=False,\n):\n \"\"\"\n Run the Lucas-Kanade optical flow routine and interpolate the motion\n vectors.\n\n .. _OpenCV: https://opencv.org/\n\n .. _`Lucas-Kanade`:\\\n https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323\n\n .. _MaskedArray:\\\n https://docs.scipy.org/doc/numpy/reference/maskedarray.baseclass.html#numpy.ma.MaskedArray\n\n .. _ndarray:\\\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\n\n Interface to the OpenCV_ implementation of the local `Lucas-Kanade`_ optical\n flow method applied in combination to a feature detection routine.\n\n The sparse motion vectors are finally interpolated to return the whole\n motion field.\n\n Parameters\n ----------\n input_images: ndarray_ or MaskedArray_\n Array of shape (T, m, n) containing a sequence of *T* two-dimensional\n input images of shape (m, n). The indexing order in **input_images** is\n assumed to be (time, latitude, longitude).\n\n *T* = 2 is the minimum required number of images.\n With *T* > 2, all the resulting sparse vectors are pooled together for\n the final interpolation on a regular grid.\n\n In case of ndarray_, invalid values (Nans or infs) are masked,\n otherwise the mask of the MaskedArray_ is used. Such mask defines a\n region where features are not detected for the tracking algorithm.\n\n lk_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the `Lucas-Kanade`_\n features tracking algorithm. See the documentation of\n :py:func:`pysteps.tracking.lucaskanade.track_features`.\n\n fd_method: {\"shitomasi\", \"blob\", \"tstorm\"}, optional\n Name of the feature detection routine. See feature detection methods in\n :py:mod:`pysteps.feature`.\n\n fd_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the features\n detection algorithm.\n See the documentation of :py:mod:`pysteps.feature`.\n\n interp_method: {\"idwinterp2d\", \"rbfinterp2d\"}, optional\n Name of the interpolation method to use. See interpolation methods in\n :py:mod:`pysteps.utils.interpolate`.\n\n interp_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the interpolation\n algorithm. See the documentation of :py:mod:`pysteps.utils.interpolate`.\n\n dense: bool, optional\n If True, return the three-dimensional array (2, m, n) containing\n the dense x- and y-components of the motion field.\n\n If False, return the sparse motion vectors as 2-D **xy** and **uv**\n arrays, where **xy** defines the vector positions, **uv** defines the\n x and y direction components of the vectors.\n\n nr_std_outlier: int, optional\n Maximum acceptable deviation from the mean in terms of number of\n standard deviations. Any sparse vector with a deviation larger than\n this threshold is flagged as outlier and excluded from the\n interpolation.\n See the documentation of\n :py:func:`pysteps.utils.cleansing.detect_outliers`.\n\n k_outlier: int or None, optional\n The number of nearest neighbors used to localize the outlier detection.\n If set to None, it employs all the data points (global detection).\n See the documentation of\n :py:func:`pysteps.utils.cleansing.detect_outliers`.\n\n size_opening: int, optional\n The size of the structuring element kernel in pixels. This is used to\n perform a binary morphological opening on the input fields in order to\n filter isolated echoes due to clutter. If set to zero, the filtering\n is not performed.\n See the documentation of\n :py:func:`pysteps.utils.images.morph_opening`.\n\n decl_scale: int, optional\n The scale declustering parameter in pixels used to reduce the number of\n redundant sparse vectors before the interpolation.\n Sparse vectors within this declustering scale are averaged together.\n If set to less than 2 pixels, the declustering is not performed.\n See the documentation of\n :py:func:`pysteps.utils.cleansing.decluster`.\n\n verbose: bool, optional\n If set to True, print some information about the program.\n\n Returns\n -------\n out: ndarray_ or tuple\n If **dense=True** (the default), return the advection field having shape\n (2, m, n), where out[0, :, :] contains the x-components of the motion\n vectors and out[1, :, :] contains the y-components.\n The velocities are in units of pixels / timestep, where timestep is the\n time difference between the two input images.\n Return a zero motion field of shape (2, m, n) when no motion is\n detected.\n\n If **dense=False**, it returns a tuple containing the 2-dimensional\n arrays **xy** and **uv**, where x, y define the vector locations,\n u, v define the x and y direction components of the vectors.\n Return two empty arrays when no motion is detected.\n\n See also\n --------\n pysteps.motion.lucaskanade.track_features\n\n References\n ----------\n Bouguet, J.-Y.: Pyramidal implementation of the affine Lucas Kanade\n feature tracker description of the algorithm, Intel Corp., 5, 4, 2001\n\n Lucas, B. D. and Kanade, T.: An iterative image registration technique with\n an application to stereo vision, in: Proceedings of the 1981 DARPA Imaging\n Understanding Workshop, pp. 121–130, 1981.\n \"\"\"\n\n input_images = input_images.copy()\n\n if verbose:\n print(\"Computing the motion field with the Lucas-Kanade method.\")\n t0 = time.time()\n\n nr_fields = input_images.shape[0]\n domain_size = (input_images.shape[1], input_images.shape[2])\n\n feature_detection_method = feature.get_method(fd_method)\n interpolation_method = utils.get_method(interp_method)\n\n if fd_kwargs is None:\n fd_kwargs = dict()\n if fd_method == \"tstorm\":\n fd_kwargs.update({\"output_feat\": True})\n\n if lk_kwargs is None:\n lk_kwargs = dict()\n\n if interp_kwargs is None:\n interp_kwargs = dict()\n\n xy = np.empty(shape=(0, 2))\n uv = np.empty(shape=(0, 2))\n for n in range(nr_fields - 1):\n # extract consecutive images\n prvs_img = input_images[n, :, :].copy()\n next_img = input_images[n + 1, :, :].copy()\n\n # Check if a MaskedArray is used. If not, mask the ndarray\n if not isinstance(prvs_img, MaskedArray):\n prvs_img = np.ma.masked_invalid(prvs_img)\n np.ma.set_fill_value(prvs_img, prvs_img.min())\n\n if not isinstance(next_img, MaskedArray):\n next_img = np.ma.masked_invalid(next_img)\n np.ma.set_fill_value(next_img, next_img.min())\n\n # remove small noise with a morphological operator (opening)\n if size_opening > 0:\n prvs_img = morph_opening(prvs_img, prvs_img.min(), size_opening)\n next_img = morph_opening(next_img, next_img.min(), size_opening)\n\n # features detection\n points = feature_detection_method(prvs_img, **fd_kwargs).astype(np.float32)\n\n # skip loop if no features to track\n if points.shape[0] == 0:\n continue\n\n # get sparse u, v vectors with Lucas-Kanade tracking\n xy_, uv_ = track_features(prvs_img, next_img, points, **lk_kwargs)\n\n # skip loop if no vectors\n if xy_.shape[0] == 0:\n continue\n\n # stack vectors\n xy = np.append(xy, xy_, axis=0)\n uv = np.append(uv, uv_, axis=0)\n\n # return zero motion field is no sparse vectors are found\n if xy.shape[0] == 0:\n if dense:\n return np.zeros((2, domain_size[0], domain_size[1]))\n else:\n return xy, uv\n\n # detect and remove outliers\n outliers = detect_outliers(uv, nr_std_outlier, xy, k_outlier, verbose)\n xy = xy[~outliers, :]\n uv = uv[~outliers, :]\n\n if verbose:\n print(\"--- LK found %i sparse vectors ---\" % xy.shape[0])\n\n # return sparse vectors if required\n if not dense:\n return xy, uv\n\n # decluster sparse motion vectors\n if decl_scale > 1:\n xy, uv = decluster(xy, uv, decl_scale, 1, verbose)\n\n # return zero motion field if no sparse vectors are left for interpolation\n if xy.shape[0] == 0:\n return np.zeros((2, domain_size[0], domain_size[1]))\n\n # interpolation\n xgrid = np.arange(domain_size[1])\n ygrid = np.arange(domain_size[0])\n uvgrid = interpolation_method(xy, uv, xgrid, ygrid, **interp_kwargs)\n\n if verbose:\n print(\"--- total time: %.2f seconds ---\" % (time.time() - t0))\n\n return uvgrid\n" }, { "path": "pysteps/motion/proesmans.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.motion.proesmans\n========================\n\nImplementation of the anisotropic diffusion method of Proesmans et al. (1994).\n\n.. autosummary::\n :toctree: ../generated/\n\n proesmans\n\"\"\"\n\nimport numpy as np\nfrom scipy.ndimage import gaussian_filter\n\nfrom pysteps.decorators import check_input_frames\nfrom pysteps.motion._proesmans import _compute_advection_field\n\n\n@check_input_frames(2, 2)\ndef proesmans(\n input_images,\n lam=50.0,\n num_iter=100,\n num_levels=6,\n filter_std=0.0,\n verbose=True,\n full_output=False,\n):\n \"\"\"\n Implementation of the anisotropic diffusion method of Proesmans et al.\n (1994).\n\n Parameters\n ----------\n input_images: array_like\n Array of shape (2, m, n) containing the first and second input image.\n lam: float\n Multiplier of the smoothness term. Smaller values give a smoother motion\n field.\n num_iter: float\n The number of iterations to use.\n num_levels: int\n The number of image pyramid levels to use.\n filter_std: float\n Standard deviation of an optional Gaussian filter that is applied before\n computing the optical flow.\n verbose: bool, optional\n Verbosity enabled if True (default).\n full_output: bool, optional\n If True, the output is a two-element tuple containing the\n forward-backward advection and consistency fields. The first element\n is shape (2, 2, m, n), where the index along the first dimension refers\n to the forward and backward advection fields. The second element is an\n array of shape (2, m, n), where the index along the first dimension\n refers to the forward and backward consistency fields.\n Default: False.\n\n Returns\n -------\n out: ndarray\n If full_output=False, the advection field having shape (2, m, n), where\n out[0, :, :] contains the x-components of the motion vectors and\n out[1, :, :] contains the y-components. The velocities are in units of\n pixels / timestep, where timestep is the time difference between the\n two input images.\n\n References\n ----------\n :cite:`PGPO1994`\n\n \"\"\"\n del verbose # Not used\n\n im1 = input_images[-2, :, :].copy()\n im2 = input_images[-1, :, :].copy()\n\n im = np.stack([im1, im2])\n im_min = np.min(im)\n im_max = np.max(im)\n if im_max - im_min > 1e-8:\n im = (im - im_min) / (im_max - im_min) * 255.0\n\n if filter_std > 0.0:\n im[0, :, :] = gaussian_filter(im[0, :, :], filter_std)\n im[1, :, :] = gaussian_filter(im[1, :, :], filter_std)\n\n advfield, quality = _compute_advection_field(im, lam, num_iter, num_levels)\n\n if not full_output:\n return advfield[0]\n else:\n return advfield, quality\n" }, { "path": "pysteps/motion/vet.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.motion.vet\n==================\n\nVariational Echo Tracking (VET) Module\n\nThis module implements the VET algorithm presented\nby `Laroche and Zawadzki (1995)`_ and used in the\nMcGill Algorithm for Prediction by Lagrangian Extrapolation (MAPLE) described\nin `Germann and Zawadzki (2002)`_.\n\n\n.. _`Laroche and Zawadzki (1995)`:\\\n http://dx.doi.org/10.1175/1520-0426(1995)012<0721:ROHWFS>2.0.CO;2\n\n.. _`Germann and Zawadzki (2002)`:\\\n http://dx.doi.org/10.1175/1520-0493(2002)130<2859:SDOTPO>2.0.CO;2\n\nThe morphing and the cost functions are implemented in Cython and parallelized\nfor performance.\n\n.. currentmodule:: pysteps.motion.vet\n\n.. autosummary::\n :toctree: ../generated/\n\n vet\n vet_cost_function\n vet_cost_function_gradient\n morph\n round_int\n ceil_int\n get_padding\n\"\"\"\n\nimport numpy\nfrom numpy.ma.core import MaskedArray\nfrom scipy.ndimage import zoom\nfrom scipy.optimize import minimize\n\nfrom pysteps.decorators import check_input_frames\nfrom pysteps.motion._vet import _warp, _cost_function\n\n\ndef round_int(scalar):\n \"\"\"\n Round number to nearest integer. Returns and integer value.\n \"\"\"\n return int(numpy.round(scalar))\n\n\ndef ceil_int(scalar):\n \"\"\"\n Round number to nearest integer. Returns and integer value.\n \"\"\"\n return int(numpy.ceil(scalar))\n\n\ndef get_padding(dimension_size, sectors):\n \"\"\"\n Get the padding at each side of the one dimensions of the image\n so the new image dimensions are divided evenly in the\n number of *sectors* specified.\n\n Parameters\n ----------\n dimension_size: int\n Actual dimension size.\n sectors: int\n number of sectors over which the the image will be divided.\n\n Returns\n -------\n pad_before , pad_after: int, int\n Padding at each side of the image for the corresponding dimension.\n \"\"\"\n reminder = dimension_size % sectors\n\n if reminder != 0:\n pad = sectors - reminder\n pad_before = pad // 2\n if pad % 2 == 0:\n pad_after = pad_before\n else:\n pad_after = pad_before + 1\n\n return pad_before, pad_after\n\n return 0, 0\n\n\ndef morph(image, displacement, gradient=False):\n \"\"\"\n Morph image by applying a displacement field (Warping).\n\n The new image is created by selecting for each position the values of the\n input image at the positions given by the x and y displacements.\n The routine works in a backward sense.\n The displacement vectors have to refer to their destination.\n\n For more information in Morphing functions see Section 3 in\n `Beezley and Mandel (2008)`_.\n\n Beezley, J. D., & Mandel, J. (2008).\n Morphing ensemble Kalman filters. Tellus A, 60(1), 131-140.\n\n .. _`Beezley and Mandel (2008)`: http://dx.doi.org/10.1111/\\\n j.1600-0870.2007.00275.x\n\n The displacement field in x and y directions and the image must have the\n same dimensions.\n\n The morphing is executed in parallel over x axis.\n\n The value of displaced pixels that fall outside the limits takes the\n value of the nearest edge. Those pixels are indicated by values greater\n than 1 in the output mask.\n\n Parameters\n ----------\n image: ndarray (ndim = 2)\n Image to morph\n displacement: ndarray (ndim = 3)\n Displacement field to be applied (Warping). The first dimension\n corresponds to the coordinate to displace.\n\n The dimensions are: displacement [ i/x (0) or j/y (1) ,\n i index of pixel, j index of pixel ]\n gradient: bool, optional\n If True, the gradient of the morphing function is returned.\n\n Returns\n -------\n image: ndarray (float64 ,ndim = 2)\n Morphed image.\n mask: ndarray (int8 ,ndim = 2)\n Invalid values mask. Points outside the boundaries are masked.\n Values greater than 1, indicate masked values.\n gradient_values: ndarray (float64 ,ndim = 3), optional\n If gradient keyword is True, the gradient of the function is also\n returned.\n \"\"\"\n\n if not isinstance(image, MaskedArray):\n _mask = numpy.zeros_like(image, dtype=\"int8\")\n else:\n _mask = numpy.asarray(numpy.ma.getmaskarray(image), dtype=\"int8\", order=\"C\")\n\n _image = numpy.asarray(image, dtype=\"float64\", order=\"C\")\n _displacement = numpy.asarray(displacement, dtype=\"float64\", order=\"C\")\n\n return _warp(_image, _mask, _displacement, gradient=gradient)\n\n\ndef vet_cost_function_gradient(*args, **kwargs):\n \"\"\"\n Compute the vet cost function gradient.\n See :py:func:`vet_cost_function` for more information.\n \"\"\"\n kwargs[\"gradient\"] = True\n return vet_cost_function(*args, **kwargs)\n\n\ndef vet_cost_function(\n sector_displacement_1d,\n input_images,\n blocks_shape,\n mask,\n smooth_gain,\n debug=False,\n gradient=False,\n):\n \"\"\"\n .. _`scipy minimization`: \\\n https://docs.scipy.org/doc/scipy-0.18.1/reference/generated/scipy.optimize.minimize.html\n \n Variational Echo Tracking Cost Function.\n\n This function is designed to be used with the `scipy minimization`_.\n The function first argument is the variable to be used in the\n minimization procedure.\n\n The sector displacement must be a flat array compatible with the\n dimensions of the input image and sectors shape (see parameters section\n below for more details).\n\n .. _ndarray:\\\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\n\n Parameters\n ----------\n sector_displacement_1d: ndarray_\n Array of displacements to apply to each sector. The dimensions are:\n sector_displacement_2d\n [ x (0) or y (1) displacement, i index of sector, j index of sector ].\n The shape of the sector displacements must be compatible with the\n input image and the block shape.\n The shape should be (2, mx, my) where mx and my are the numbers of\n sectors in the x and the y dimension.\n input_images: ndarray_\n Input images, sequence of 2D arrays, or 3D arrays.\n The first dimension represents the images time dimension.\n\n The template_image (first element in first dimensions) denotes the\n reference image used to obtain the displacement (2D array).\n The second is the target image.\n\n The expected dimensions are (2,nx,ny).\n Be aware the the 2D images dimensions correspond to (lon,lat) or (x,y).\n blocks_shape: ndarray_ (ndim=2)\n Number of sectors in each dimension (x and y).\n blocks_shape.shape = (mx,my)\n mask: ndarray_ (ndim=2)\n Data mask. If is True, the data is marked as not valid and is not\n used in the computations.\n smooth_gain: float\n Smoothness constrain gain\n debug: bool, optional\n If True, print debugging information.\n gradient: bool, optional\n If True, the gradient of the morphing function is returned.\n\n Returns\n -------\n penalty or gradient values.\n\n penalty: float\n Value of the cost function\n gradient_values: ndarray (float64 ,ndim = 3), optional\n If gradient keyword is True, the gradient of the function is also\n returned.\n \"\"\"\n\n sector_displacement_2d = sector_displacement_1d.reshape(\n *((2,) + tuple(blocks_shape))\n )\n\n if input_images.shape[0] == 3:\n three_times = True\n previous_image = input_images[0]\n center_image = input_images[1]\n next_image = input_images[2]\n\n else:\n previous_image = None\n center_image = input_images[0]\n next_image = input_images[1]\n three_times = False\n\n if gradient:\n gradient_values = _cost_function(\n sector_displacement_2d,\n center_image,\n next_image,\n mask,\n smooth_gain,\n gradient=True,\n )\n if three_times:\n gradient_values += _cost_function(\n sector_displacement_2d,\n previous_image,\n center_image,\n mask,\n smooth_gain,\n gradient=True,\n )\n\n return gradient_values.ravel()\n\n else:\n residuals, smoothness_penalty = _cost_function(\n sector_displacement_2d,\n center_image,\n next_image,\n mask,\n smooth_gain,\n gradient=False,\n )\n\n if three_times:\n _residuals, _smoothness = _cost_function(\n sector_displacement_2d,\n previous_image,\n center_image,\n mask,\n smooth_gain,\n gradient=False,\n )\n\n residuals += _residuals\n smoothness_penalty += _smoothness\n\n if debug:\n print(\"\\nresiduals\", residuals)\n print(\"smoothness_penalty\", smoothness_penalty)\n\n return residuals + smoothness_penalty\n\n\n@check_input_frames(2, 3)\ndef vet(\n input_images,\n sectors=((32, 16, 4, 2), (32, 16, 4, 2)),\n smooth_gain=1e6,\n first_guess=None,\n intermediate_steps=False,\n verbose=True,\n indexing=\"yx\",\n padding=0,\n options=None,\n):\n \"\"\"\n Variational Echo Tracking Algorithm presented in\n `Laroche and Zawadzki (1995)`_ and used in the McGill Algorithm for\n Prediction by Lagrangian Extrapolation (MAPLE) described in\n `Germann and Zawadzki (2002)`_.\n\n .. _`Laroche and Zawadzki (1995)`:\\\n http://dx.doi.org/10.1175/1520-0426(1995)012<0721:ROHWFS>2.0.CO;2\n\n .. _`Germann and Zawadzki (2002)`:\\\n http://dx.doi.org/10.1175/1520-0493(2002)130<2859:SDOTPO>2.0.CO;2\n\n This algorithm computes the displacement field between two images\n ( the input_image with respect to the template image).\n The displacement is sought by minimizing the sum of the residuals of the\n squared differences of the images pixels and the contribution of a\n smoothness constraint.\n In the case that a MaskedArray is used as input, the residuals term in\n the cost function is only computed over areas with non-masked values.\n Otherwise, it is computed over the entire domain.\n\n To find the minimum, a scaling guess procedure is applied,\n from larger to smaller scales.\n This reduces the chances that the minimization procedure\n converges to a local minimum.\n The first scaling guess is defined by the scaling sectors keyword.\n\n The smoothness of the returned displacement field is controlled by the\n smoothness constraint gain (**smooth_gain** keyword).\n\n If a first guess is not given, zero displacements are used as the first\n guess.\n\n The cost function is minimized using the `scipy minimization`_ function,\n with the 'CG' method by default.\n This method proved to give the best results under many different conditions\n and is the most similar one to the original VET implementation in\n `Laroche and Zawadzki (1995)`_.\n\n\n The method CG uses a nonlinear conjugate gradient algorithm by Polak and\n Ribiere, a variant of the Fletcher-Reeves method described in\n Nocedal and Wright (2006), pp. 120-122.\n\n .. _`scipy minimization`: \\\n https://docs.scipy.org/doc/scipy-0.18.1/reference/generated/scipy.optimize.minimize.html\n\n .. _MaskedArray: https://docs.scipy.org/doc/numpy/reference/\\\n maskedarray.baseclass.html#numpy.ma.MaskedArray\n\n .. _ndarray:\\\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\n\n Parameters\n ----------\n input_images: ndarray_ or MaskedArray\n Input images, sequence of 2D arrays, or 3D arrays.\n The first dimension represents the images time dimension.\n\n The template_image (first element in first dimensions) denotes the\n reference image used to obtain the displacement (2D array).\n The second is the target image.\n\n The expected dimensions are (2,ni,nj).\n sectors: list or array, optional\n Number of sectors on each dimension used in the scaling procedure.\n If dimension is 1, the same sectors will be used both image dimensions\n (x and y). If **sectors** is a 1D array, the same number of sectors\n is used in both dimensions.\n smooth_gain: float, optional\n Smooth gain factor\n first_guess: ndarray_, optional\n The shape of the first guess should have the same shape as the initial\n sectors shapes used in the scaling procedure.\n If first_guess is not present zeros are used as first guess.\n\n E.g.:\n If the first sector shape in the scaling procedure is (ni,nj), then\n the first_guess should have (2, ni, nj ) shape.\n intermediate_steps: bool, optional\n If True, also return a list with the first guesses obtained during the\n scaling procedure. False, by default.\n verbose: bool, optional\n Verbosity enabled if True (default).\n indexing: str, optional\n Input indexing order.'ij' and 'xy' indicates that the\n dimensions of the input are (time, longitude, latitude), while\n 'yx' indicates (time, latitude, longitude).\n The displacement field dimensions are ordered accordingly in a way that\n the first dimension indicates the displacement along x (0) or y (1).\n That is, UV displacements are always returned.\n padding: int\n Padding width in grid points. A border is added to the input array\n to reduce the effects of the minimization at the border.\n options: dict, optional\n A dictionary of solver options.\n See `scipy minimization`_ function for more details.\n\n Returns\n -------\n displacement_field: ndarray_\n Displacement Field (2D array representing the transformation) that\n warps the template image into the input image.\n The dimensions are (2,ni,nj), where the first\n dimension indicates the displacement along x (0) or y (1) in units of\n pixels / timestep as given by the input_images array.\n intermediate_steps: list of ndarray_\n List with the first guesses obtained during the scaling procedure.\n\n References\n ----------\n Laroche, S., and I. Zawadzki, 1995:\n Retrievals of horizontal winds from single-Doppler clear-air data by\n methods of cross-correlation and variational analysis.\n J. Atmos. Oceanic Technol., 12, 721–738.\n doi: http://dx.doi.org/10.1175/1520-0426(1995)012<0721:ROHWFS>2.0.CO;2\n\n Germann, U. and I. Zawadzki, 2002:\n Scale-Dependence of the Predictability of Precipitation from Continental\n Radar Images. Part I: Description of the Methodology.\n Mon. Wea. Rev., 130, 2859–2873,\n doi: 10.1175/1520-0493(2002)130<2859:SDOTPO>2.0.CO;2.\n\n Nocedal, J, and S J Wright. 2006. Numerical Optimization. Springer New York.\n \"\"\"\n\n if verbose:\n\n def debug_print(*args, **kwargs):\n print(*args, **kwargs)\n\n else:\n\n def debug_print(*args, **kwargs):\n del args\n del kwargs\n\n if options is None:\n options = dict()\n else:\n options = dict(options)\n\n options.setdefault(\"eps\", 0.1)\n options.setdefault(\"gtol\", 0.1)\n options.setdefault(\"maxiter\", 100)\n options.setdefault(\"disp\", False)\n optimization_method = options.pop(\"method\", \"CG\")\n\n # Set to None to suppress pylint warning.\n pad_i = None\n pad_j = None\n sectors_in_i = None\n sectors_in_j = None\n\n debug_print(\"Running VET algorithm\")\n\n valid_indexing = [\"yx\", \"xy\", \"ij\"]\n\n if indexing not in valid_indexing:\n raise ValueError(\n \"Invalid indexing values: {0}\\n\".format(indexing)\n + \"Supported values: {0}\".format(str(valid_indexing))\n )\n\n # Convert input_images to a MaskedArray if it is a regular ndarray\n if not isinstance(input_images, MaskedArray):\n input_images = numpy.ma.masked_invalid(input_images)\n\n mask = numpy.ma.getmaskarray(input_images)\n\n if padding > 0:\n padding_tuple = ((0, 0), (padding, padding), (padding, padding))\n\n input_images_data = numpy.pad(\n numpy.ma.getdata(input_images),\n padding_tuple,\n \"constant\",\n constant_values=numpy.nan,\n )\n\n mask = numpy.pad(mask, padding_tuple, \"constant\", constant_values=True)\n\n input_images = numpy.ma.MaskedArray(data=input_images_data, mask=mask)\n\n input_images.data[mask] = 0 # Remove any Nan from the raw data\n\n # Create a 2D mask with the right data type for _vet\n mask = numpy.asarray(numpy.any(mask, axis=0), dtype=\"int8\", order=\"C\")\n\n input_images = numpy.asarray(input_images.data, dtype=\"float64\", order=\"C\")\n\n # Check that the sectors divide the domain\n sectors = numpy.asarray(sectors, dtype=\"int\", order=\"C\")\n\n if sectors.ndim == 1:\n new_sectors = numpy.zeros(\n (2,) + sectors.shape, dtype=\"int\", order=\"C\"\n ) + sectors.reshape((1, sectors.shape[0]))\n sectors = new_sectors\n elif sectors.ndim > 2 or sectors.ndim < 1:\n raise ValueError(\n \"Incorrect sectors dimensions.\\n\"\n + \"Only 1D or 2D arrays are supported to define\"\n + \"the number of sectors used in\"\n + \"the scaling procedure\"\n )\n\n # Sort sectors in descending order\n sectors[0, :].sort()\n sectors[1, :].sort()\n\n # Prepare first guest\n first_guess_shape = (2, int(sectors[0, 0]), int(sectors[1, 0]))\n\n if first_guess is None:\n first_guess = numpy.zeros(first_guess_shape, order=\"C\")\n else:\n if first_guess.shape != first_guess_shape:\n raise ValueError(\n \"The shape of the initial guess do not match the number of \"\n + \"sectors of the first scaling guess\\n\"\n + \"first_guess.shape={}\\n\".format(str(first_guess.shape))\n + \"Expected shape={}\".format(str(first_guess_shape))\n )\n else:\n first_guess = numpy.asarray(first_guess, order=\"C\", dtype=\"float64\")\n\n scaling_guesses = list()\n\n previous_sectors_in_i = sectors[0, 0]\n previous_sectors_in_j = sectors[1, 0]\n\n for n, (sectors_in_i, sectors_in_j) in enumerate(zip(sectors[0, :], sectors[1, :])):\n # Minimize for each sector size\n pad_i = get_padding(input_images.shape[1], sectors_in_i)\n pad_j = get_padding(input_images.shape[2], sectors_in_j)\n\n if (pad_i != (0, 0)) or (pad_j != (0, 0)):\n _input_images = numpy.pad(input_images, ((0, 0), pad_i, pad_j), \"edge\")\n\n _mask = numpy.pad(mask, (pad_i, pad_j), \"constant\", constant_values=1)\n _mask = numpy.ascontiguousarray(_mask)\n if first_guess is None:\n first_guess = numpy.pad(first_guess, ((0, 0), pad_i, pad_j), \"edge\")\n first_guess = numpy.ascontiguousarray(first_guess)\n\n else:\n _input_images = input_images\n _mask = mask\n\n sector_shape = (\n _input_images.shape[1] // sectors_in_i,\n _input_images.shape[2] // sectors_in_j,\n )\n\n debug_print(\"original image shape: \" + str(input_images.shape))\n debug_print(\"padded image shape: \" + str(_input_images.shape))\n debug_print(\"padded template_image image shape: \" + str(_input_images.shape))\n\n debug_print(\n \"\\nNumber of sectors: {0:d},{1:d}\".format(sectors_in_i, sectors_in_j)\n )\n\n debug_print(\"Sector Shape:\", sector_shape)\n\n if n > 0:\n first_guess = zoom(\n first_guess,\n (\n 1,\n sectors_in_i / previous_sectors_in_i,\n sectors_in_j / previous_sectors_in_j,\n ),\n order=1,\n mode=\"nearest\",\n )\n\n debug_print(\"Minimizing\")\n\n result = minimize(\n vet_cost_function,\n first_guess.flatten(),\n jac=vet_cost_function_gradient,\n args=(_input_images, (sectors_in_i, sectors_in_j), _mask, smooth_gain),\n method=optimization_method,\n options=options,\n )\n\n first_guess = result.x.reshape(*first_guess.shape)\n\n if verbose:\n vet_cost_function(\n result.x,\n _input_images,\n (sectors_in_i, sectors_in_j),\n _mask,\n smooth_gain,\n debug=True,\n )\n if indexing == \"yx\":\n scaling_guesses.append(first_guess[::-1, ...])\n else:\n scaling_guesses.append(first_guess)\n\n previous_sectors_in_i = sectors_in_i\n previous_sectors_in_j = sectors_in_j\n\n first_guess = zoom(\n first_guess,\n (\n 1,\n _input_images.shape[1] / sectors_in_i,\n _input_images.shape[2] / sectors_in_j,\n ),\n order=1,\n mode=\"nearest\",\n )\n\n first_guess = numpy.ascontiguousarray(first_guess)\n # Remove the extra padding if any\n ni = _input_images.shape[1]\n nj = _input_images.shape[2]\n\n first_guess = first_guess[:, pad_i[0] : ni - pad_i[1], pad_j[0] : nj - pad_j[1]]\n\n if indexing == \"yx\":\n first_guess = first_guess[::-1, ...]\n\n if padding > 0:\n first_guess = first_guess[:, padding:-padding, padding:-padding]\n\n if intermediate_steps:\n return first_guess, scaling_guesses\n\n return first_guess\n" }, { "path": "pysteps/noise/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nMethods for generating stochastic perturbations of 2d precipitation and\nvelocity fields.\n\"\"\"\n\nfrom .interface import get_method\nfrom . import utils, motion, fftgenerators\n" }, { "path": "pysteps/noise/fftgenerators.py", "content": "\"\"\"\npysteps.noise.fftgenerators\n===========================\n\nMethods for noise generators based on FFT filtering of white noise.\n\nThe methods in this module implement the following interface for filter\ninitialization depending on their parametric or nonparametric nature::\n\n initialize_param_2d_xxx_filter(field, **kwargs)\n\nor::\n\n initialize_nonparam_2d_xxx_filter(field, **kwargs)\n\nwhere field is an array of shape (m, n) or (t, m, n) that defines the target field\nand optional parameters are supplied as keyword arguments.\n\nThe output of each initialization method is a dictionary containing the keys field\nand input_shape. The first is a two-dimensional array of shape (m, int(n/2)+1)\nthat defines the filter. The second one is the shape of the input field for the\nfilter.\n\nThe methods in this module implement the following interface for the generation\nof correlated noise::\n\n generate_noise_2d_xxx_filter(field, randstate=np.random, seed=None, **kwargs)\n\nwhere field (m, n) is a filter returned from the corresponding initialization\nmethod, and randstate and seed can be used to set the random generator and\nits seed. Additional keyword arguments can be included as a dictionary.\n\nThe output of each generator method is a two-dimensional array containing the\nfield of correlated noise cN of shape (m, n).\n\n.. autosummary::\n :toctree: ../generated/\n\n initialize_param_2d_fft_filter\n initialize_nonparam_2d_fft_filter\n initialize_nonparam_2d_nested_filter\n initialize_nonparam_2d_ssft_filter\n generate_noise_2d_fft_filter\n generate_noise_2d_ssft_filter\n\"\"\"\n\nimport numpy as np\nfrom scipy import optimize\n\nfrom .. import utils\n\n\ndef initialize_param_2d_fft_filter(field, **kwargs):\n \"\"\"\n Takes one ore more 2d input fields, fits two spectral slopes, beta1 and beta2,\n to produce one parametric, global and isotropic fourier filter.\n\n Parameters\n ----------\n field: array-like\n Two- or three-dimensional array containing one or more input fields.\n All values are required to be finite. If more than one field are passed,\n the average fourier filter is returned. It assumes that fields are stacked\n by the first axis: [nr_fields, y, x].\n\n Other Parameters\n ----------------\n win_fun: {'hann', 'tukey' or None}\n Optional tapering function to be applied to the input field, generated with\n :py:func:`pysteps.utils.tapering.compute_window_function`.\n (default None).\n model: {'power-law'}\n The name of the parametric model to be used to fit the power spectrum of\n the input field (default 'power-law').\n weighted: bool\n Whether or not to apply 1/sqrt(power) as weight in the numpy.polyfit()\n function (default False).\n rm_rdisc: bool\n Whether or not to remove the rain/no-rain disconituity (default False).\n It assumes no-rain pixels are assigned with lowest value.\n fft_method: str or tuple\n A string or a (function,kwargs) tuple defining the FFT method to use\n (see \"FFT methods\" in :py:func:`pysteps.utils.interface.get_method`).\n Defaults to \"numpy\".\n\n Returns\n -------\n out: dict\n A dictionary containing the keys field, input_shape, model and pars.\n The first is a two-dimensional array of shape (m, int(n/2)+1) that\n defines the filter. The second one is the shape of the input field for\n the filter. The last two are the model and fitted parameters,\n respectively.\n\n This dictionary can be passed to\n :py:func:`pysteps.noise.fftgenerators.generate_noise_2d_fft_filter` to\n generate noise fields.\n \"\"\"\n\n if len(field.shape) < 2 or len(field.shape) > 3:\n raise ValueError(\"the input is not two- or three-dimensional array\")\n if np.any(~np.isfinite(field)):\n raise ValueError(\n \"field contains non-finite values, this typically happens when the input\\n\"\n + \"precipitation field provided to pysteps contains (mostly)zero values.\\n\"\n + \"To prevent this error please call pysteps.utils.check_norain first,\\n\"\n + \"using the same win_fun as used in this method (tukey by default)\\n\"\n + \"and then only call this method if that check fails.\"\n )\n\n # defaults\n win_fun = kwargs.get(\"win_fun\", None)\n model = kwargs.get(\"model\", \"power-law\")\n weighted = kwargs.get(\"weighted\", False)\n rm_rdisc = kwargs.get(\"rm_rdisc\", False)\n fft = kwargs.get(\"fft_method\", \"numpy\")\n if type(fft) == str:\n fft_shape = field.shape if len(field.shape) == 2 else field.shape[1:]\n fft = utils.get_method(fft, shape=fft_shape)\n\n field = field.copy()\n\n # remove rain/no-rain discontinuity\n if rm_rdisc:\n field[field > field.min()] -= field[field > field.min()].min() - field.min()\n\n # dims\n if len(field.shape) == 2:\n field = field[None, :, :]\n nr_fields = field.shape[0]\n M, N = field.shape[1:]\n\n if win_fun is not None:\n tapering = utils.tapering.compute_window_function(M, N, win_fun)\n\n # make sure non-rainy pixels are set to zero\n field -= field.min(axis=(1, 2))[:, None, None]\n else:\n tapering = np.ones((M, N))\n\n if model.lower() == \"power-law\":\n # compute average 2D PSD\n F = np.zeros((M, N), dtype=complex)\n for i in range(nr_fields):\n F += fft.fftshift(fft.fft2(field[i, :, :] * tapering))\n F /= nr_fields\n F = abs(F) ** 2 / F.size\n\n # compute radially averaged 1D PSD\n psd = utils.spectral.rapsd(F)\n L = max(M, N)\n\n # wavenumbers\n if L % 2 == 1:\n wn = np.arange(0, int(L / 2) + 1)\n else:\n wn = np.arange(0, int(L / 2))\n\n # compute single spectral slope beta as first guess\n if weighted:\n p0 = np.polyfit(np.log(wn[1:]), np.log(psd[1:]), 1, w=np.sqrt(psd[1:]))\n else:\n p0 = np.polyfit(np.log(wn[1:]), np.log(psd[1:]), 1)\n beta = p0[0]\n\n # create the piecewise function with two spectral slopes beta1 and beta2\n # and scaling break x0\n def piecewise_linear(x, x0, y0, beta1, beta2):\n return np.piecewise(\n x,\n [x < x0, x >= x0],\n [\n lambda x: beta1 * x + y0 - beta1 * x0,\n lambda x: beta2 * x + y0 - beta2 * x0,\n ],\n )\n\n # fit the two betas and the scaling break\n p0 = [2.0, 0, beta, beta] # first guess\n bounds = (\n [2.0, 0, -4, -4],\n [5.0, 20, -1.0, -1.0],\n ) # TODO: provide better bounds\n if weighted:\n p, e = optimize.curve_fit(\n piecewise_linear,\n np.log(wn[1:]),\n np.log(psd[1:]),\n p0=p0,\n bounds=bounds,\n sigma=1 / np.sqrt(psd[1:]),\n )\n else:\n p, e = optimize.curve_fit(\n piecewise_linear, np.log(wn[1:]), np.log(psd[1:]), p0=p0, bounds=bounds\n )\n\n # compute 2d filter\n YC, XC = utils.arrays.compute_centred_coord_array(M, N)\n R = np.sqrt(XC * XC + YC * YC)\n R = fft.fftshift(R)\n pf = p.copy()\n pf[2:] = pf[2:] / 2\n F = np.exp(piecewise_linear(np.log(R), *pf))\n F[~np.isfinite(F)] = 1\n\n f = piecewise_linear\n\n else:\n raise ValueError(\"unknown parametric model %s\" % model)\n\n return {\n \"field\": F,\n \"input_shape\": field.shape[1:],\n \"use_full_fft\": True,\n \"model\": f,\n \"pars\": p,\n }\n\n\ndef initialize_nonparam_2d_fft_filter(field, **kwargs):\n \"\"\"\n Takes one ore more 2d input fields and produces one non-parametric, global\n and anisotropic fourier filter.\n\n Parameters\n ----------\n field: array-like\n Two- or three-dimensional array containing one or more input fields.\n All values are required to be finite. If more than one field are passed,\n the average fourier filter is returned. It assumes that fields are stacked\n by the first axis: [nr_fields, y, x].\n\n Other Parameters\n ----------------\n win_fun: {'hann', 'tukey', None}\n Optional tapering function to be applied to the input field, generated with\n :py:func:`pysteps.utils.tapering.compute_window_function`\n (default 'tukey').\n donorm: bool\n Option to normalize the real and imaginary parts.\n Default: False\n rm_rdisc: bool\n Whether or not to remove the rain/no-rain disconituity (default True).\n It assumes no-rain pixels are assigned with lowest value.\n fft_method: str or tuple\n A string or a (function,kwargs) tuple defining the FFT method to use\n (see \"FFT methods\" in :py:func:`pysteps.utils.interface.get_method`).\n Defaults to \"numpy\".\n\n Returns\n -------\n out: dict\n A dictionary containing the keys field and input_shape. The first is a\n two-dimensional array of shape (m, int(n/2)+1) that defines the filter.\n The second one is the shape of the input field for the filter.\n\n It can be passed to\n :py:func:`pysteps.noise.fftgenerators.generate_noise_2d_fft_filter`.\n \"\"\"\n if len(field.shape) < 2 or len(field.shape) > 3:\n raise ValueError(\"the input is not two- or three-dimensional array\")\n if np.any(~np.isfinite(field)):\n raise ValueError(\n \"field contains non-finite values, this typically happens when the input\\n\"\n + \"precipitation field provided to pysteps contains (mostly)zero values.\\n\"\n + \"To prevent this error please call pysteps.utils.check_norain first,\\n\"\n + \"using the same win_fun as used in this method (tukey by default)\\n\"\n + \"and then only call this method if that check fails.\"\n )\n\n # defaults\n win_fun = kwargs.get(\"win_fun\", \"tukey\")\n donorm = kwargs.get(\"donorm\", False)\n rm_rdisc = kwargs.get(\"rm_rdisc\", True)\n use_full_fft = kwargs.get(\"use_full_fft\", False)\n fft = kwargs.get(\"fft_method\", \"numpy\")\n if type(fft) == str:\n fft_shape = field.shape if len(field.shape) == 2 else field.shape[1:]\n fft = utils.get_method(fft, shape=fft_shape)\n\n field = field.copy()\n\n # remove rain/no-rain discontinuity\n if rm_rdisc:\n field[field > field.min()] -= field[field > field.min()].min() - field.min()\n\n # dims\n if len(field.shape) == 2:\n field = field[None, :, :]\n nr_fields = field.shape[0]\n field_shape = field.shape[1:]\n if use_full_fft:\n fft_shape = (field.shape[1], field.shape[2])\n else:\n fft_shape = (field.shape[1], int(field.shape[2] / 2) + 1)\n\n # make sure non-rainy pixels are set to zero\n field -= field.min(axis=(1, 2))[:, None, None]\n\n if win_fun is not None:\n tapering = utils.tapering.compute_window_function(\n field_shape[0], field_shape[1], win_fun\n )\n else:\n tapering = np.ones(field_shape)\n\n F = np.zeros(fft_shape, dtype=complex)\n for i in range(nr_fields):\n if use_full_fft:\n F += fft.fft2(field[i, :, :] * tapering)\n else:\n F += fft.rfft2(field[i, :, :] * tapering)\n F /= nr_fields\n\n # normalize the real and imaginary parts\n if donorm:\n if np.std(F.imag) > 0:\n F.imag = (F.imag - np.mean(F.imag)) / np.std(F.imag)\n if np.std(F.real) > 0:\n F.real = (F.real - np.mean(F.real)) / np.std(F.real)\n\n return {\n \"field\": np.abs(F),\n \"input_shape\": field.shape[1:],\n \"use_full_fft\": use_full_fft,\n }\n\n\ndef generate_noise_2d_fft_filter(\n F, randstate=None, seed=None, fft_method=None, domain=\"spatial\"\n):\n \"\"\"\n Produces a field of correlated noise using global Fourier filtering.\n\n Parameters\n ----------\n F: dict\n A filter object returned by\n :py:func:`pysteps.noise.fftgenerators.initialize_param_2d_fft_filter` or\n :py:func:`pysteps.noise.fftgenerators.initialize_nonparam_2d_fft_filter`.\n All values in the filter array are required to be finite.\n randstate: mtrand.RandomState\n Optional random generator to use. If set to None, use numpy.random.\n seed: int\n Value to set a seed for the generator. None will not set the seed.\n fft_method: str or tuple\n A string or a (function,kwargs) tuple defining the FFT method to use\n (see \"FFT methods\" in :py:func:`pysteps.utils.interface.get_method`).\n Defaults to \"numpy\".\n domain: {\"spatial\", \"spectral\"}\n The domain for the computations: If \"spatial\", the noise is generated\n in the spatial domain and transformed back to spatial domain after the\n Fourier filtering. If \"spectral\", the noise field is generated and kept\n in the spectral domain.\n\n Returns\n -------\n N: array-like\n A two-dimensional field of stationary correlated noise. The noise field\n is normalized to zero mean and unit variance.\n\n \"\"\"\n if domain not in [\"spatial\", \"spectral\"]:\n raise ValueError(\n \"invalid value %s for the 'domain' argument: must be 'spatial' or 'spectral'\"\n % str(domain)\n )\n\n input_shape = F[\"input_shape\"]\n use_full_fft = F[\"use_full_fft\"]\n F = F[\"field\"]\n\n if len(F.shape) != 2:\n raise ValueError(\"field is not two-dimensional array\")\n if np.any(~np.isfinite(F)):\n raise ValueError(\n \"field contains non-finite values, this typically happens when the input\\n\"\n + \"precipitation field provided to pysteps contains (mostly)zero values.\\n\"\n + \"To prevent this error please call pysteps.utils.check_norain first,\\n\"\n + \"using the same win_fun as used in this method (tukey by default)\\n\"\n + \"and then only call this method if that check fails.\"\n )\n\n if randstate is None:\n randstate = np.random\n\n # set the seed\n if seed is not None:\n randstate.seed(seed)\n\n if fft_method is None:\n fft = utils.get_method(\"numpy\", shape=input_shape)\n else:\n if type(fft_method) == str:\n fft = utils.get_method(fft_method, shape=input_shape)\n else:\n fft = fft_method\n\n # produce fields of white noise\n if domain == \"spatial\":\n N = randstate.randn(input_shape[0], input_shape[1])\n else:\n if use_full_fft:\n size = (input_shape[0], input_shape[1])\n else:\n size = (input_shape[0], int(input_shape[1] / 2) + 1)\n theta = randstate.uniform(low=0.0, high=2.0 * np.pi, size=size)\n if input_shape[0] % 2 == 0:\n theta[int(input_shape[0] / 2) + 1 :, 0] = -theta[\n 1 : int(input_shape[0] / 2), 0\n ][::-1]\n else:\n theta[int(input_shape[0] / 2) + 1 :, 0] = -theta[\n 1 : int(input_shape[0] / 2) + 1, 0\n ][::-1]\n N = np.cos(theta) + 1.0j * np.sin(theta)\n\n # apply the global Fourier filter to impose a correlation structure\n if domain == \"spatial\":\n if use_full_fft:\n fN = fft.fft2(N)\n else:\n fN = fft.rfft2(N)\n else:\n fN = N\n fN *= F\n if domain == \"spatial\":\n if use_full_fft:\n N = np.array(fft.ifft2(fN).real)\n else:\n N = np.array(fft.irfft2(fN))\n N = (N - N.mean()) / N.std()\n else:\n N = fN\n N[0, 0] = 0.0\n N /= utils.spectral.std(N, input_shape, use_full_fft=use_full_fft)\n\n return N\n\n\ndef initialize_nonparam_2d_ssft_filter(field, **kwargs):\n \"\"\"\n Function to compute the local Fourier filters using the Short-Space Fourier\n filtering approach.\n\n Parameters\n ----------\n field: array-like\n Two- or three-dimensional array containing one or more input fields.\n All values are required to be finite. If more than one field are passed,\n the average fourier filter is returned. It assumes that fields are stacked\n by the first axis: [nr_fields, y, x].\n\n Other Parameters\n ----------------\n win_size: int or two-element tuple of ints\n Size-length of the window to compute the SSFT (default (128, 128)).\n win_fun: {'hann', 'tukey', None}\n Optional tapering function to be applied to the input field, generated with\n :py:func:`pysteps.utils.tapering.compute_window_function`\n (default 'tukey').\n overlap: float [0,1[\n The proportion of overlap to be applied between successive windows\n (default 0.3).\n war_thr: float [0,1]\n Threshold for the minimum fraction of rain needed for computing the FFT\n (default 0.1).\n rm_rdisc: bool\n Whether or not to remove the rain/no-rain disconituity. It assumes no-rain\n pixels are assigned with lowest value.\n fft_method: str or tuple\n A string or a (function,kwargs) tuple defining the FFT method to use\n (see \"FFT methods\" in :py:func:`pysteps.utils.interface.get_method`).\n Defaults to \"numpy\".\n\n Returns\n -------\n field: array-like\n Four-dimensional array containing the 2d fourier filters distributed over\n a 2d spatial grid.\n It can be passed to\n :py:func:`pysteps.noise.fftgenerators.generate_noise_2d_ssft_filter`.\n\n References\n ----------\n :cite:`NBSG2017`\n \"\"\"\n\n if len(field.shape) < 2 or len(field.shape) > 3:\n raise ValueError(\"the input is not two- or three-dimensional array\")\n if np.any(np.isnan(field)):\n raise ValueError(\"field must not contain NaNs\")\n\n # defaults\n win_size = kwargs.get(\"win_size\", (128, 128))\n if type(win_size) == int:\n win_size = (win_size, win_size)\n win_fun = kwargs.get(\"win_fun\", \"tukey\")\n overlap = kwargs.get(\"overlap\", 0.3)\n war_thr = kwargs.get(\"war_thr\", 0.1)\n rm_rdisc = kwargs.get(\"rm_rdisc\", True)\n fft = kwargs.get(\"fft_method\", \"numpy\")\n if type(fft) == str:\n fft_shape = field.shape if len(field.shape) == 2 else field.shape[1:]\n fft = utils.get_method(fft, shape=fft_shape)\n\n field = field.copy()\n\n # remove rain/no-rain discontinuity\n if rm_rdisc:\n field[field > field.min()] -= field[field > field.min()].min() - field.min()\n\n # dims\n if len(field.shape) == 2:\n field = field[None, :, :]\n nr_fields = field.shape[0]\n dim = field.shape[1:]\n dim_x = dim[1]\n dim_y = dim[0]\n\n # make sure non-rainy pixels are set to zero\n field -= field.min(axis=(1, 2))[:, None, None]\n\n # SSFT algorithm\n\n # prepare indices\n idxi = np.zeros(2, dtype=int)\n idxj = np.zeros(2, dtype=int)\n\n # number of windows\n num_windows_y = np.ceil(float(dim_y) / win_size[0]).astype(int)\n num_windows_x = np.ceil(float(dim_x) / win_size[1]).astype(int)\n\n # domain fourier filter\n F0 = initialize_nonparam_2d_fft_filter(\n field, win_fun=win_fun, donorm=True, use_full_fft=True, fft_method=fft\n )[\"field\"]\n # and allocate it to the final grid\n F = np.zeros((num_windows_y, num_windows_x, F0.shape[0], F0.shape[1]))\n F += F0[np.newaxis, np.newaxis, :, :]\n\n # loop rows\n for i in range(F.shape[0]):\n # loop columns\n for j in range(F.shape[1]):\n # compute indices of local window\n idxi[0] = int(np.max((i * win_size[0] - overlap * win_size[0], 0)))\n idxi[1] = int(\n np.min((idxi[0] + win_size[0] + overlap * win_size[0], dim_y))\n )\n idxj[0] = int(np.max((j * win_size[1] - overlap * win_size[1], 0)))\n idxj[1] = int(\n np.min((idxj[0] + win_size[1] + overlap * win_size[1], dim_x))\n )\n\n # build localization mask\n # TODO: the 0.01 rain threshold must be improved\n mask = _get_mask(dim, idxi, idxj, win_fun)\n war = float(np.sum((field * mask[None, :, :]) > 0.01)) / (\n (idxi[1] - idxi[0]) * (idxj[1] - idxj[0]) * nr_fields\n )\n\n if war > war_thr:\n # the new filter\n F[i, j, :, :] = initialize_nonparam_2d_fft_filter(\n field * mask[None, :, :],\n win_fun=None,\n donorm=True,\n use_full_fft=True,\n fft_method=fft,\n )[\"field\"]\n\n return {\"field\": F, \"input_shape\": field.shape[1:], \"use_full_fft\": True}\n\n\ndef initialize_nonparam_2d_nested_filter(field, gridres=1.0, **kwargs):\n \"\"\"\n Function to compute the local Fourier filters using a nested approach.\n\n Parameters\n ----------\n field: array-like\n Two- or three-dimensional array containing one or more input fields.\n All values are required to be finite.\n If more than one field are passed, the average fourier filter is returned.\n It assumes that fields are stacked by the first axis: [nr_fields, y, x].\n gridres: float\n Grid resolution in km.\n\n Other Parameters\n ----------------\n max_level: int\n Localization parameter. 0: global noise, >0: increasing degree of\n localization (default 3).\n win_fun: {'hann', 'tukey', None}\n Optional tapering function to be applied to the input field, generated with\n :py:func:`pysteps.utils.tapering.compute_window_function`\n (default 'tukey').\n war_thr: float [0;1]\n Threshold for the minimum fraction of rain needed for computing the FFT\n (default 0.1).\n rm_rdisc: bool\n Whether or not to remove the rain/no-rain discontinuity. It assumes no-rain\n pixels are assigned with lowest value.\n fft_method: str or tuple\n A string or a (function,kwargs) tuple defining the FFT method to use\n (see \"FFT methods\" in :py:func:`pysteps.utils.interface.get_method`).\n Defaults to \"numpy\".\n\n Returns\n -------\n field: array-like\n Four-dimensional array containing the 2d fourier filters distributed over\n a 2d spatial grid.\n It can be passed to\n :py:func:`pysteps.noise.fftgenerators.generate_noise_2d_ssft_filter`.\n \"\"\"\n\n if len(field.shape) < 2 or len(field.shape) > 3:\n raise ValueError(\"the input is not two- or three-dimensional array\")\n if np.any(np.isnan(field)):\n raise ValueError(\"field must not contain NaNs\")\n\n # defaults\n max_level = kwargs.get(\"max_level\", 3)\n win_fun = kwargs.get(\"win_fun\", \"tukey\")\n war_thr = kwargs.get(\"war_thr\", 0.1)\n rm_rdisc = kwargs.get(\"rm_rdisc\", True)\n fft = kwargs.get(\"fft_method\", \"numpy\")\n if type(fft) == str:\n fft_shape = field.shape if len(field.shape) == 2 else field.shape[1:]\n fft = utils.get_method(fft, shape=fft_shape)\n\n field = field.copy()\n\n # remove rain/no-rain discontinuity\n if rm_rdisc:\n field[field > field.min()] -= field[field > field.min()].min() - field.min()\n\n # dims\n if len(field.shape) == 2:\n field = field[None, :, :]\n nr_fields = field.shape[0]\n dim = field.shape[1:]\n dim_x = dim[1]\n dim_y = dim[0]\n\n # make sure non-rainy pixels are set to zero\n field -= field.min(axis=(1, 2))[:, None, None]\n\n # Nested algorithm\n\n # prepare indices\n Idxi = np.array([[0, dim_y]])\n Idxj = np.array([[0, dim_x]])\n Idxipsd = np.array([[0, 2**max_level]])\n Idxjpsd = np.array([[0, 2**max_level]])\n\n # generate the FFT sample frequencies\n freqx = fft.fftfreq(dim_x, gridres)\n freqy = fft.fftfreq(dim_y, gridres)\n fx, fy = np.meshgrid(freqx, freqy)\n freq_grid = np.sqrt(fx**2 + fy**2)\n\n # domain fourier filter\n F0 = initialize_nonparam_2d_fft_filter(\n field, win_fun=win_fun, donorm=True, use_full_fft=True, fft_method=fft\n )[\"field\"]\n # and allocate it to the final grid\n F = np.zeros((2**max_level, 2**max_level, F0.shape[0], F0.shape[1]))\n F += F0[np.newaxis, np.newaxis, :, :]\n\n # now loop levels and build composite spectra\n level = 0\n while level < max_level:\n for m in range(len(Idxi)):\n # the indices of rainfall field\n Idxinext, Idxjnext = _split_field(Idxi[m, :], Idxj[m, :], 2)\n # the indices of the field of fourier filters\n Idxipsdnext, Idxjpsdnext = _split_field(Idxipsd[m, :], Idxjpsd[m, :], 2)\n\n for n in range(len(Idxinext)):\n mask = _get_mask(dim, Idxinext[n, :], Idxjnext[n, :], win_fun)\n war = np.sum((field * mask[None, :, :]) > 0.01) / float(\n (Idxinext[n, 1] - Idxinext[n, 0])\n * (Idxjnext[n, 1] - Idxjnext[n, 0])\n * nr_fields\n )\n\n if war > war_thr:\n # the new filter\n newfilter = initialize_nonparam_2d_fft_filter(\n field * mask[None, :, :],\n win_fun=None,\n donorm=True,\n use_full_fft=True,\n fft_method=fft,\n )[\"field\"]\n\n # compute logistic function to define weights as function of frequency\n # k controls the shape of the weighting function\n # TODO: optimize parameters\n k = 0.05\n x0 = (\n Idxinext[n, 1] - Idxinext[n, 0]\n ) / 2.0 # TODO: consider y dimension, too\n merge_weights = 1 / (\n 1 + np.exp(-k * (1 / freq_grid - x0 * gridres))\n )\n newfilter *= 1 - merge_weights\n\n # perform the weighted average of previous and new fourier filters\n F[\n Idxipsdnext[n, 0] : Idxipsdnext[n, 1],\n Idxjpsdnext[n, 0] : Idxjpsdnext[n, 1],\n :,\n :,\n ] *= merge_weights[np.newaxis, np.newaxis, :, :]\n F[\n Idxipsdnext[n, 0] : Idxipsdnext[n, 1],\n Idxjpsdnext[n, 0] : Idxjpsdnext[n, 1],\n :,\n :,\n ] += newfilter[np.newaxis, np.newaxis, :, :]\n\n # update indices\n level += 1\n Idxi, Idxj = _split_field((0, dim[0]), (0, dim[1]), 2**level)\n Idxipsd, Idxjpsd = _split_field((0, 2**max_level), (0, 2**max_level), 2**level)\n\n return {\"field\": F, \"input_shape\": field.shape[1:], \"use_full_fft\": True}\n\n\ndef generate_noise_2d_ssft_filter(F, randstate=None, seed=None, **kwargs):\n \"\"\"\n Function to compute the locally correlated noise using a nested approach.\n\n Parameters\n ----------\n F: array-like\n A filter object returned by\n :py:func:`pysteps.noise.fftgenerators.initialize_nonparam_2d_ssft_filter` or\n :py:func:`pysteps.noise.fftgenerators.initialize_nonparam_2d_nested_filter`.\n The filter is a four-dimensional array containing the 2d fourier filters\n distributed over a 2d spatial grid.\n randstate: mtrand.RandomState\n Optional random generator to use. If set to None, use numpy.random.\n seed: int\n Value to set a seed for the generator. None will not set the seed.\n\n Other Parameters\n ----------------\n overlap: float\n Percentage overlap [0-1] between successive windows (default 0.2).\n win_fun: {'hann', 'tukey', None}\n Optional tapering function to be applied to the input field, generated with\n :py:func:`pysteps.utils.tapering.compute_window_function`\n (default 'tukey').\n fft_method: str or tuple\n A string or a (function,kwargs) tuple defining the FFT method to use\n (see \"FFT methods\" in :py:func:`pysteps.utils.interface.get_method`).\n Defaults to \"numpy\".\n\n Returns\n -------\n N: array-like\n A two-dimensional numpy array of non-stationary correlated noise.\n \"\"\"\n input_shape = F[\"input_shape\"]\n use_full_fft = F[\"use_full_fft\"]\n F = F[\"field\"]\n\n if len(F.shape) != 4:\n raise ValueError(\"the input is not four-dimensional array\")\n if np.any(~np.isfinite(F)):\n raise ValueError(\n \"field contains non-finite values, this typically happens when the input\\n\"\n + \"precipitation field provided to pysteps contains (mostly) zero value.s\\n\"\n + \"To prevent this error please call pysteps.utils.check_norain first,\\n\"\n + \"using the same win_fun as used in this method (tukey by default)\\n\"\n + \"and then only call this method if that check fails.\"\n )\n\n if \"domain\" in kwargs.keys() and kwargs[\"domain\"] == \"spectral\":\n raise NotImplementedError(\n \"SSFT-based noise generator is not implemented in the spectral domain\"\n )\n\n # defaults\n overlap = kwargs.get(\"overlap\", 0.2)\n win_fun = kwargs.get(\"win_fun\", \"tukey\")\n fft = kwargs.get(\"fft_method\", \"numpy\")\n if type(fft) == str:\n fft = utils.get_method(fft, shape=input_shape)\n\n if randstate is None:\n randstate = np.random\n\n # set the seed\n if seed is not None:\n randstate.seed(seed)\n\n dim_y = F.shape[2]\n dim_x = F.shape[3]\n dim = (dim_y, dim_x)\n\n # produce fields of white noise\n N = randstate.randn(dim_y, dim_x)\n fN = fft.fft2(N)\n\n # initialize variables\n cN = np.zeros(dim)\n sM = np.zeros(dim)\n\n idxi = np.zeros(2, dtype=int)\n idxj = np.zeros(2, dtype=int)\n\n # get the window size\n win_size = (float(dim_y) / F.shape[0], float(dim_x) / F.shape[1])\n\n # loop the windows and build composite image of correlated noise\n\n # loop rows\n for i in range(F.shape[0]):\n # loop columns\n for j in range(F.shape[1]):\n # apply fourier filtering with local filter\n lF = F[i, j, :, :]\n flN = fN * lF\n flN = np.array(fft.ifft2(flN).real)\n\n # compute indices of local window\n idxi[0] = int(np.max((i * win_size[0] - overlap * win_size[0], 0)))\n idxi[1] = int(\n np.min((idxi[0] + win_size[0] + overlap * win_size[0], dim_y))\n )\n idxj[0] = int(np.max((j * win_size[1] - overlap * win_size[1], 0)))\n idxj[1] = int(\n np.min((idxj[0] + win_size[1] + overlap * win_size[1], dim_x))\n )\n\n # build mask and add local noise field to the composite image\n M = _get_mask(dim, idxi, idxj, win_fun)\n cN += flN * M\n sM += M\n\n # normalize the field\n cN[sM > 0] /= sM[sM > 0]\n cN = (cN - cN.mean()) / cN.std()\n\n return cN\n\n\ndef _split_field(idxi, idxj, Segments):\n \"\"\"Split domain field into a number of equally sapced segments.\"\"\"\n\n sizei = idxi[1] - idxi[0]\n sizej = idxj[1] - idxj[0]\n\n winsizei = int(sizei / Segments)\n winsizej = int(sizej / Segments)\n\n Idxi = np.zeros((Segments**2, 2))\n Idxj = np.zeros((Segments**2, 2))\n\n count = -1\n for i in range(Segments):\n for j in range(Segments):\n count += 1\n Idxi[count, 0] = idxi[0] + i * winsizei\n Idxi[count, 1] = np.min((Idxi[count, 0] + winsizei, idxi[1]))\n Idxj[count, 0] = idxj[0] + j * winsizej\n Idxj[count, 1] = np.min((Idxj[count, 0] + winsizej, idxj[1]))\n\n Idxi = np.array(Idxi).astype(int)\n Idxj = np.array(Idxj).astype(int)\n\n return Idxi, Idxj\n\n\ndef _get_mask(Size, idxi, idxj, win_fun):\n \"\"\"Compute a mask of zeros with a window at a given position.\"\"\"\n\n idxi = np.array(idxi).astype(int)\n idxj = np.array(idxj).astype(int)\n\n win_size = (idxi[1] - idxi[0], idxj[1] - idxj[0])\n if win_fun is not None:\n wind = utils.tapering.compute_window_function(win_size[0], win_size[1], win_fun)\n wind += 1e-6 # avoid zero values\n\n else:\n wind = np.ones(win_size)\n\n mask = np.zeros(Size)\n mask[idxi.item(0) : idxi.item(1), idxj.item(0) : idxj.item(1)] = wind\n\n return mask\n" }, { "path": "pysteps/noise/interface.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.noise.interface\n=======================\n\nInterface for the noise module.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nfrom pysteps.noise.fftgenerators import (\n initialize_param_2d_fft_filter,\n generate_noise_2d_fft_filter,\n initialize_nonparam_2d_fft_filter,\n initialize_nonparam_2d_ssft_filter,\n generate_noise_2d_ssft_filter,\n initialize_nonparam_2d_nested_filter,\n)\nfrom pysteps.noise.motion import initialize_bps, generate_bps\n\n_noise_methods = dict()\n\n_noise_methods[\"parametric\"] = (\n initialize_param_2d_fft_filter,\n generate_noise_2d_fft_filter,\n)\n\n_noise_methods[\"nonparametric\"] = (\n initialize_nonparam_2d_fft_filter,\n generate_noise_2d_fft_filter,\n)\n_noise_methods[\"ssft\"] = (\n initialize_nonparam_2d_ssft_filter,\n generate_noise_2d_ssft_filter,\n)\n\n_noise_methods[\"nested\"] = (\n initialize_nonparam_2d_nested_filter,\n generate_noise_2d_ssft_filter,\n)\n\n_noise_methods[\"bps\"] = (initialize_bps, generate_bps)\n\n\ndef get_method(name):\n \"\"\"\n Return two callable functions to initialize and generate 2d perturbations\n of precipitation or velocity fields.\\n\n\n Methods for precipitation fields:\n\n +-------------------+------------------------------------------------------+\n | Name | Description |\n +===================+======================================================+\n | parametric | this global generator uses parametric Fourier |\n | | filtering (power-law model) |\n +-------------------+------------------------------------------------------+\n | nonparametric | this global generator uses nonparametric Fourier |\n | | filtering |\n +-------------------+------------------------------------------------------+\n | ssft | this local generator uses the short-space Fourier |\n | | filtering |\n +-------------------+------------------------------------------------------+\n | nested | this local generator uses a nested Fourier filtering |\n +-------------------+------------------------------------------------------+\n\n Methods for velocity fields:\n\n +-------------------+------------------------------------------------------+\n | Name | Description |\n +===================+======================================================+\n | bps | The method described in :cite:`BPS2006`, where |\n | | time-dependent velocity perturbations are sampled |\n | | from the exponential distribution |\n +-------------------+------------------------------------------------------+\n\n \"\"\"\n if isinstance(name, str):\n name = name.lower()\n else:\n raise TypeError(\n \"Only strings supported for the method's names.\\n\"\n + \"Available names:\"\n + str(list(_noise_methods.keys()))\n ) from None\n\n try:\n return _noise_methods[name]\n except KeyError:\n raise ValueError(\n \"Unknown method {}\\n\".format(name)\n + \"The available methods are:\"\n + str(list(_noise_methods.keys()))\n ) from None\n" }, { "path": "pysteps/noise/motion.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.noise.motion\n====================\n\nMethods for generating perturbations of two-dimensional motion fields.\n\nThe methods in this module implement the following interface for\ninitialization::\n\n inizialize_xxx(V, pixelsperkm, timestep, optional arguments)\n\nwhere V (2,m,n) is the motion field and pixelsperkm and timestep describe the\nspatial and temporal resolution of the motion vectors.\nThe output of each initialization method is a dictionary containing the\nperturbator that can be supplied to generate_xxx.\n\nThe methods in this module implement the following interface for the generation\nof a motion perturbation field::\n\n generate_xxx(perturbator, t, randstate=np.random, seed=None)\n\nwhere perturbator is a dictionary returned by an initialize_xxx method.\nOptional random generator can be specified with the randstate and seed\narguments, respectively.\nThe output of each generator method is an array of shape (2,m,n) containing the\nx- and y-components of the motion vector perturbations, where m and n are\ndetermined from the perturbator.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_default_params_bps_par\n get_default_params_bps_perp\n initialize_bps\n generate_bps\n\"\"\"\n\nimport numpy as np\nfrom scipy import linalg\n\n\ndef get_default_params_bps_par():\n \"\"\"Return a tuple containing the default velocity perturbation parameters\n given in :cite:`BPS2006` for the parallel component.\"\"\"\n return (10.88, 0.23, -7.68)\n\n\ndef get_default_params_bps_perp():\n \"\"\"Return a tuple containing the default velocity perturbation parameters\n given in :cite:`BPS2006` for the perpendicular component.\"\"\"\n return (5.76, 0.31, -2.72)\n\n\ndef initialize_bps(\n V, pixelsperkm, timestep, p_par=None, p_perp=None, randstate=None, seed=None\n):\n \"\"\"\n Initialize the motion field perturbator described in :cite:`BPS2006`.\n For simplicity, the bias adjustment procedure described there has not been\n implemented. The perturbator generates a field whose magnitude increases\n with respect to lead time.\n\n Parameters\n ----------\n V: array_like\n Array of shape (2,m,n) containing the x- and y-components of the m*n\n motion field to perturb.\n p_par: tuple\n Tuple containing the parameters a,b and c for the standard deviation of\n the perturbations in the direction parallel to the motion vectors. The\n standard deviations are modeled by the function f_par(t) = a*t**b+c,\n where t is lead time. The default values are taken from :cite:`BPS2006`.\n p_perp: tuple\n Tuple containing the parameters a,b and c for the standard deviation of\n the perturbations in the direction perpendicular to the motion vectors.\n The standard deviations are modeled by the function f_par(t) = a*t**b+c,\n where t is lead time. The default values are taken from :cite:`BPS2006`.\n pixelsperkm: float\n Spatial resolution of the motion field (pixels/kilometer).\n timestep: float\n Time step for the motion vectors (minutes).\n randstate: mtrand.RandomState\n Optional random generator to use. If set to None, use numpy.random.\n seed: int\n Optional seed number for the random generator.\n\n Returns\n -------\n out: dict\n A dictionary containing the perturbator that can be supplied to\n generate_motion_perturbations_bps.\n\n See also\n --------\n pysteps.noise.motion.generate_bps\n\n \"\"\"\n if len(V.shape) != 3:\n raise ValueError(\"V is not a three-dimensional array\")\n if V.shape[0] != 2:\n raise ValueError(\"the first dimension of V is not 2\")\n\n if p_par is None:\n p_par = get_default_params_bps_par()\n if p_perp is None:\n p_perp = get_default_params_bps_perp()\n\n if len(p_par) != 3:\n raise ValueError(\"the length of p_par is not 3\")\n if len(p_perp) != 3:\n raise ValueError(\"the length of p_perp is not 3\")\n\n perturbator = {}\n if randstate is None:\n randstate = np.random\n\n if seed is not None:\n randstate.seed(seed)\n\n eps_par = randstate.laplace(scale=1.0 / np.sqrt(2))\n eps_perp = randstate.laplace(scale=1.0 / np.sqrt(2))\n\n # scale factor for converting the unit of the advection velocities\n # into km/h\n vsf = 60.0 / (timestep * pixelsperkm)\n\n N = linalg.norm(V, axis=0)\n mask = N > 1e-12\n V_n = np.empty(V.shape)\n V_n[:, mask] = V[:, mask] / np.stack([N[mask], N[mask]])\n V_n[:, ~mask] = 0.0\n\n perturbator[\"randstate\"] = randstate\n perturbator[\"vsf\"] = vsf\n perturbator[\"p_par\"] = p_par\n perturbator[\"p_perp\"] = p_perp\n perturbator[\"eps_par\"] = eps_par\n perturbator[\"eps_perp\"] = eps_perp\n perturbator[\"V_par\"] = V_n\n perturbator[\"V_perp\"] = np.stack([-V_n[1, :, :], V_n[0, :, :]])\n\n return perturbator\n\n\ndef generate_bps(perturbator, t):\n \"\"\"\n Generate a motion perturbation field by using the method described in\n :cite:`BPS2006`.\n\n Parameters\n ----------\n perturbator: dict\n A dictionary returned by initialize_motion_perturbations_bps.\n t: float\n Lead time for the perturbation field (minutes).\n\n Returns\n -------\n out: ndarray\n Array of shape (2,m,n) containing the x- and y-components of the motion\n vector perturbations, where m and n are determined from the perturbator.\n\n See also\n --------\n pysteps.noise.motion.initialize_bps\n\n \"\"\"\n vsf = perturbator[\"vsf\"]\n p_par = perturbator[\"p_par\"]\n p_perp = perturbator[\"p_perp\"]\n eps_par = perturbator[\"eps_par\"]\n eps_perp = perturbator[\"eps_perp\"]\n V_par = perturbator[\"V_par\"]\n V_perp = perturbator[\"V_perp\"]\n\n g_par = p_par[0] * pow(t, p_par[1]) + p_par[2]\n g_perp = p_perp[0] * pow(t, p_perp[1]) + p_perp[2]\n\n return (g_par * eps_par * V_par + g_perp * eps_perp * V_perp) / vsf\n" }, { "path": "pysteps/noise/utils.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.noise.utils\n===================\n\nMiscellaneous utility functions related to generation of stochastic perturbations.\n\n.. autosummary::\n :toctree: ../generated/\n\n compute_noise_stddev_adjs\n\"\"\"\n\nimport numpy as np\n\ntry:\n import dask\n\n dask_imported = True\nexcept ImportError:\n dask_imported = False\n\n\ndef compute_noise_stddev_adjs(\n R,\n R_thr_1,\n R_thr_2,\n F,\n decomp_method,\n noise_filter,\n noise_generator,\n num_iter,\n conditional=True,\n num_workers=1,\n seed=None,\n):\n \"\"\"Apply a scale-dependent adjustment factor to the noise fields used in STEPS.\n\n Simulates the effect of applying a precipitation mask to a Gaussian noise\n field obtained by the nonparametric filter method. The idea is to decompose\n the masked noise field into a cascade and compare the standard deviations\n of each level into those of the observed precipitation intensity field.\n This gives correction factors for the standard deviations :cite:`BPS2006`.\n The calculations are done for n realizations of the noise field, and the\n correction factors are calculated from the average values of the standard\n deviations.\n\n Parameters\n ----------\n R: array_like\n The input precipitation field, assumed to be in logarithmic units\n (dBR or reflectivity).\n R_thr_1: float\n Intensity threshold for precipitation/no precipitation.\n R_thr_2: float\n Intensity values below R_thr_1 are set to this value.\n F: dict\n A bandpass filter dictionary returned by a method defined in\n pysteps.cascade.bandpass_filters. This defines the filter to use and\n the number of cascade levels.\n decomp_method: function\n A function defined in pysteps.cascade.decomposition. Specifies the\n method to use for decomposing the observed precipitation field and\n noise field into different spatial scales.\n num_iter: int\n The number of noise fields to generate.\n conditional: bool\n If set to True, compute the statistics conditionally by excluding areas\n of no precipitation.\n num_workers: int\n The number of workers to use for parallel computation. Applicable if\n dask is installed.\n seed: int\n Optional seed number for the random generators.\n\n Returns\n -------\n out: list\n A list containing the standard deviation adjustment factor for each\n cascade level.\n \"\"\"\n\n MASK = R >= R_thr_1\n\n R = R.copy()\n R[~np.isfinite(R)] = R_thr_2\n R[~MASK] = R_thr_2\n if not conditional:\n mu, sigma = np.mean(R), np.std(R)\n else:\n mu, sigma = np.mean(R[MASK]), np.std(R[MASK])\n R -= mu\n\n MASK_ = MASK if conditional else None\n decomp_R = decomp_method(R, F, mask=MASK_)\n\n if dask_imported and num_workers > 1:\n res = []\n\n N_stds = [None] * num_iter\n randstates = []\n\n for k in range(num_iter):\n rs = np.random.RandomState(seed=seed)\n randstates.append(rs)\n seed = rs.randint(0, high=1e9)\n\n def worker(k):\n # generate Gaussian white noise field, filter it using the chosen\n # method, multiply it with the standard deviation of the observed\n # field and apply the precipitation mask\n N = noise_generator(noise_filter, randstate=randstates[k])\n N = N / np.std(N) * sigma + mu\n N[~MASK] = R_thr_2\n\n # subtract the mean and decompose the masked noise field into a\n # cascade\n N -= mu\n decomp_N = decomp_method(N, F, mask=MASK_)\n\n N_stds[k] = decomp_N[\"stds\"]\n\n if dask_imported and num_workers > 1:\n for k in range(num_iter):\n res.append(dask.delayed(worker)(k))\n dask.compute(*res, num_workers=num_workers)\n\n else:\n for k in range(num_iter):\n worker(k)\n\n # for each cascade level, compare the standard deviations between the\n # observed field and the masked noise field, which gives the correction\n # factors\n return decomp_R[\"stds\"] / np.mean(np.vstack(N_stds), axis=0)\n" }, { "path": "pysteps/nowcasts/__init__.py", "content": "\"\"\"Implementations of deterministic and ensemble nowcasting methods.\"\"\"\n\nfrom pysteps.nowcasts.interface import get_method\n" }, { "path": "pysteps/nowcasts/anvil.py", "content": "\"\"\"\npysteps.nowcasts.anvil\n======================\n\nImplementation of the autoregressive nowcasting using VIL (ANVIL) nowcasting\nmethod developed in :cite:`PCLH2020`. Compared to S-PROG, the main improvements\nare using an autoregressive integrated (ARI) model and the option to use\nvertically integrated liquid (VIL) as the input variable. Using the ARI model\navoids biasedness and loss of small-scale features in the forecast field, and\nno statistical post-processing is needed. In addition, the model allows\nlocalization of parameter estimates. It was shown in :cite:`PCLH2020` that due\nto the above improvements, ANVIL produces more reliable deterministic nowcasts\nthan S-PROG.\n\n.. autosummary::\n :toctree: ../generated/\n\n forecast\n\"\"\"\n\nimport time\nimport numpy as np\nfrom scipy.ndimage import gaussian_filter\nfrom pysteps import cascade, extrapolation\nfrom pysteps.nowcasts.utils import nowcast_main_loop\nfrom pysteps.timeseries import autoregression\nfrom pysteps import utils\n\ntry:\n import dask\n\n DASK_IMPORTED = True\nexcept ImportError:\n DASK_IMPORTED = False\n\n\ndef forecast(\n vil,\n velocity,\n timesteps,\n rainrate=None,\n n_cascade_levels=6,\n extrap_method=\"semilagrangian\",\n ar_order=2,\n ar_window_radius=50,\n r_vil_window_radius=3,\n fft_method=\"numpy\",\n apply_rainrate_mask=True,\n num_workers=1,\n extrap_kwargs=None,\n filter_kwargs=None,\n measure_time=False,\n):\n \"\"\"\n Generate a nowcast by using the autoregressive nowcasting using VIL\n (ANVIL) method. ANVIL is built on top of an extrapolation-based nowcast.\n The key features are:\n\n 1) Growth and decay: implemented by using a cascade decomposition and\n a multiscale autoregressive integrated ARI(p,1) model. Instead of the\n original time series, the ARI model is applied to the differenced one\n corresponding to time derivatives.\n 2) Originally designed for using integrated liquid (VIL) as the input data.\n In this case, the rain rate (R) is obtained from VIL via an empirical\n relation. This implementation is more general so that the input can be\n any two-dimensional precipitation field.\n 3) The parameters of the ARI model and the R(VIL) relation are allowed to\n be spatially variable. The estimation is done using a moving window.\n\n Parameters\n ----------\n vil: array_like\n Array of shape (ar_order+2,m,n) containing the input fields ordered by\n timestamp from oldest to newest. The inputs are expected to contain VIL\n or rain rate. The time steps between the inputs are assumed to be regular.\n velocity: array_like\n Array of shape (2,m,n) containing the x- and y-components of the\n advection field. The velocities are assumed to represent one time step\n between the inputs. All values are required to be finite.\n timesteps: int or list of floats\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed (relative to the input time step). The elements\n of the list are required to be in ascending order.\n rainrate: array_like\n Array of shape (m,n) containing the most recently observed rain rate\n field. If set to None, no R(VIL) conversion is done and the outputs\n are in the same units as the inputs.\n n_cascade_levels: int, optional\n The number of cascade levels to use. Defaults to 6, see issue #385\n on GitHub.\n extrap_method: str, optional\n Name of the extrapolation method to use. See the documentation of\n pysteps.extrapolation.interface.\n ar_order: int, optional\n The order of the autoregressive model to use. The recommended values\n are 1 or 2. Using a higher-order model is strongly discouraged because\n the stationarity of the AR process cannot be guaranteed.\n ar_window_radius: int, optional\n The radius of the window to use for determining the parameters of the\n autoregressive model. Set to None to disable localization.\n r_vil_window_radius: int, optional\n The radius of the window to use for determining the R(VIL) relation.\n Applicable if rainrate is not None.\n fft_method: str, optional\n A string defining the FFT method to use (see utils.fft.get_method).\n Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed,\n the recommended method is 'pyfftw'.\n apply_rainrate_mask: bool\n Apply mask to prevent producing precipitation to areas where it was not\n originally observed. Defaults to True. Disabling this may improve some\n verification metrics but increases the number of false alarms. Applicable\n if rainrate is None.\n num_workers: int, optional\n The number of workers to use for parallel computation. Applicable if\n dask is installed or pyFFTW is used for computing the FFT.\n When num_workers>1, it is advisable to disable OpenMP by setting\n the environment variable OMP_NUM_THREADS to 1.\n This avoids slowdown caused by too many simultaneous threads.\n extrap_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the extrapolation\n method. See the documentation of pysteps.extrapolation.\n filter_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the filter method.\n See the documentation of pysteps.cascade.bandpass_filters.py.\n measure_time: bool, optional\n If True, measure, print and return the computation time.\n\n Returns\n -------\n out: ndarray\n A three-dimensional array of shape (num_timesteps,m,n) containing a time\n series of forecast precipitation fields. The time series starts from\n t0+timestep, where timestep is taken from the input VIL/rain rate\n fields. If measure_time is True, the return value is a three-element\n tuple containing the nowcast array, the initialization time of the\n nowcast generator and the time used in the main loop (seconds).\n\n References\n ----------\n :cite:`PCLH2020`\n \"\"\"\n _check_inputs(vil, rainrate, velocity, timesteps, ar_order)\n\n if extrap_kwargs is None:\n extrap_kwargs = dict()\n else:\n extrap_kwargs = extrap_kwargs.copy()\n\n if filter_kwargs is None:\n filter_kwargs = dict()\n\n print(\"Computing ANVIL nowcast\")\n print(\"-----------------------\")\n print(\"\")\n\n print(\"Inputs\")\n print(\"------\")\n print(f\"input dimensions: {vil.shape[1]}x{vil.shape[2]}\")\n print(\"\")\n\n print(\"Methods\")\n print(\"-------\")\n print(f\"extrapolation: {extrap_method}\")\n print(f\"FFT: {fft_method}\")\n print(\"\")\n\n print(\"Parameters\")\n print(\"----------\")\n if isinstance(timesteps, int):\n print(f\"number of time steps: {timesteps}\")\n else:\n print(f\"time steps: {timesteps}\")\n print(f\"parallel threads: {num_workers}\")\n print(f\"number of cascade levels: {n_cascade_levels}\")\n print(f\"order of the ARI(p,1) model: {ar_order}\")\n if type(ar_window_radius) == int:\n print(f\"ARI(p,1) window radius: {ar_window_radius}\")\n else:\n print(\"ARI(p,1) window radius: none\")\n\n print(f\"R(VIL) window radius: {r_vil_window_radius}\")\n\n if measure_time:\n starttime_init = time.time()\n\n m, n = vil.shape[1:]\n vil = vil.copy()\n\n if rainrate is None and apply_rainrate_mask:\n rainrate_mask = vil[-1, :] < 0.1\n else:\n rainrate_mask = None\n\n if rainrate is not None:\n # determine the coefficients fields of the relation R=a*VIL+b by\n # localized linear regression\n r_vil_a, r_vil_b = _r_vil_regression(vil[-1, :], rainrate, r_vil_window_radius)\n else:\n r_vil_a, r_vil_b = None, None\n\n # transform the input fields to Lagrangian coordinates by extrapolation\n extrapolator = extrapolation.get_method(extrap_method)\n extrap_kwargs[\"allow_nonfinite_values\"] = (\n True if np.any(~np.isfinite(vil)) else False\n )\n\n res = list()\n\n def worker(vil, i):\n return (\n i,\n extrapolator(\n vil[i, :],\n velocity,\n vil.shape[0] - 1 - i,\n **extrap_kwargs,\n )[-1],\n )\n\n for i in range(vil.shape[0] - 1):\n if not DASK_IMPORTED or num_workers == 1:\n vil[i, :, :] = worker(vil, i)[1]\n else:\n res.append(dask.delayed(worker)(vil, i))\n\n if DASK_IMPORTED and num_workers > 1:\n num_workers_ = len(res) if num_workers > len(res) else num_workers\n vil_e = dask.compute(*res, num_workers=num_workers_)\n for i in range(len(vil_e)):\n vil[vil_e[i][0], :] = vil_e[i][1]\n\n # compute the final mask as the intersection of the masks of the advected\n # fields\n mask = np.isfinite(vil[0, :])\n for i in range(1, vil.shape[0]):\n mask = np.logical_and(mask, np.isfinite(vil[i, :]))\n\n if rainrate is None and apply_rainrate_mask:\n rainrate_mask = np.logical_and(rainrate_mask, mask)\n\n # apply cascade decomposition to the advected input fields\n bp_filter_method = cascade.get_method(\"gaussian\")\n bp_filter = bp_filter_method((m, n), n_cascade_levels, **filter_kwargs)\n\n fft = utils.get_method(fft_method, shape=vil.shape[1:], n_threads=num_workers)\n\n decomp_method, recomp_method = cascade.get_method(\"fft\")\n\n vil_dec = np.empty((n_cascade_levels, vil.shape[0], m, n))\n for i in range(vil.shape[0]):\n vil_ = vil[i, :].copy()\n vil_[~np.isfinite(vil_)] = 0.0\n vil_dec_i = decomp_method(vil_, bp_filter, fft_method=fft)\n for j in range(n_cascade_levels):\n vil_dec[j, i, :] = vil_dec_i[\"cascade_levels\"][j, :]\n\n # compute time-lagged correlation coefficients for the cascade levels of\n # the advected and differenced input fields\n gamma = np.empty((n_cascade_levels, ar_order, m, n))\n for i in range(n_cascade_levels):\n vil_diff = np.diff(vil_dec[i, :], axis=0)\n vil_diff[~np.isfinite(vil_diff)] = 0.0\n for j in range(ar_order):\n gamma[i, j, :] = _moving_window_corrcoef(\n vil_diff[-1, :], vil_diff[-(j + 2), :], ar_window_radius\n )\n\n if ar_order == 2:\n # if the order of the ARI model is 2, adjust the correlation coefficients\n # so that the resulting process is stationary\n for i in range(n_cascade_levels):\n gamma[i, 1, :] = autoregression.adjust_lag2_corrcoef2(\n gamma[i, 0, :], gamma[i, 1, :]\n )\n\n # estimate the parameters of the ARI models\n phi = []\n for i in range(n_cascade_levels):\n if ar_order > 2:\n phi_ = autoregression.estimate_ar_params_yw_localized(gamma[i, :], d=1)\n elif ar_order == 2:\n phi_ = _estimate_ar2_params(gamma[i, :])\n else:\n phi_ = _estimate_ar1_params(gamma[i, :])\n phi.append(phi_)\n\n vil_dec = vil_dec[:, -(ar_order + 1) :, :]\n\n if measure_time:\n init_time = time.time() - starttime_init\n\n print(\"Starting nowcast computation.\")\n\n rainrate_f = []\n\n extrap_kwargs[\"return_displacement\"] = True\n\n state = {\"vil_dec\": vil_dec}\n params = {\n \"apply_rainrate_mask\": apply_rainrate_mask,\n \"mask\": mask,\n \"n_cascade_levels\": n_cascade_levels,\n \"phi\": phi,\n \"rainrate\": rainrate,\n \"rainrate_mask\": rainrate_mask,\n \"recomp_method\": recomp_method,\n \"r_vil_a\": r_vil_a,\n \"r_vil_b\": r_vil_b,\n }\n\n rainrate_f = nowcast_main_loop(\n vil[-1, :],\n velocity,\n state,\n timesteps,\n extrap_method,\n _update,\n extrap_kwargs=extrap_kwargs,\n params=params,\n measure_time=measure_time,\n )\n if measure_time:\n rainrate_f, mainloop_time = rainrate_f\n\n if measure_time:\n return np.stack(rainrate_f), init_time, mainloop_time\n else:\n return np.stack(rainrate_f)\n\n\ndef _check_inputs(vil, rainrate, velocity, timesteps, ar_order):\n if vil.ndim != 3:\n raise ValueError(\n \"vil.shape = %s, but a three-dimensional array expected\" % str(vil.shape)\n )\n if rainrate is not None:\n if rainrate.ndim != 2:\n raise ValueError(\n \"rainrate.shape = %s, but a two-dimensional array expected\"\n % str(rainrate.shape)\n )\n if vil.shape[0] != ar_order + 2:\n raise ValueError(\n \"vil.shape[0] = %d, but vil.shape[0] = ar_order + 2 = %d required\"\n % (vil.shape[0], ar_order + 2)\n )\n if velocity.ndim != 3:\n raise ValueError(\n \"velocity.shape = %s, but a three-dimensional array expected\"\n % str(velocity.shape)\n )\n if isinstance(timesteps, list) and not sorted(timesteps) == timesteps:\n raise ValueError(\"timesteps is not in ascending order\")\n\n\n# optimized version of timeseries.autoregression.estimate_ar_params_yw_localized\n# for an ARI(1,1) model\ndef _estimate_ar1_params(gamma):\n phi = []\n phi.append(1 + gamma[0, :])\n phi.append(-gamma[0, :])\n phi.append(np.zeros(gamma[0, :].shape))\n\n return phi\n\n\n# optimized version of timeseries.autoregression.estimate_ar_params_yw_localized\n# for an ARI(2,1) model\ndef _estimate_ar2_params(gamma):\n phi_diff = []\n phi_diff.append(gamma[0, :] * (1 - gamma[1, :]) / (1 - gamma[0, :] * gamma[0, :]))\n phi_diff.append(\n (gamma[1, :] - gamma[0, :] * gamma[0, :]) / (1 - gamma[0, :] * gamma[0, :])\n )\n\n phi = []\n phi.append(1 + phi_diff[0])\n phi.append(-phi_diff[0] + phi_diff[1])\n phi.append(-phi_diff[1])\n phi.append(np.zeros(phi_diff[0].shape))\n\n return phi\n\n\n# Compute correlation coefficients of two 2d fields in a moving window with\n# a Gaussian weight function. See Section II.G of PCLH2020. Differently to the\n# standard formula for the Pearson correlation coefficient, the mean value of\n# the inputs is assumed to be zero.\ndef _moving_window_corrcoef(x, y, window_radius):\n mask = np.logical_and(np.isfinite(x), np.isfinite(y))\n x = x.copy()\n x[~mask] = 0.0\n y = y.copy()\n y[~mask] = 0.0\n mask = mask.astype(float)\n\n if window_radius is not None:\n n = gaussian_filter(mask, window_radius, mode=\"constant\")\n\n ssx = gaussian_filter(x**2, window_radius, mode=\"constant\")\n ssy = gaussian_filter(y**2, window_radius, mode=\"constant\")\n sxy = gaussian_filter(x * y, window_radius, mode=\"constant\")\n else:\n n = np.mean(mask)\n\n ssx = np.mean(x**2)\n ssy = np.mean(y**2)\n sxy = np.mean(x * y)\n\n stdx = np.sqrt(ssx / n)\n stdy = np.sqrt(ssy / n)\n cov = sxy / n\n\n mask = np.logical_and(stdx > 1e-8, stdy > 1e-8)\n mask = np.logical_and(mask, stdx * stdy > 1e-8)\n mask = np.logical_and(mask, n > 1e-3)\n corr = np.empty(x.shape)\n corr[mask] = cov[mask] / (stdx[mask] * stdy[mask])\n corr[~mask] = 0.0\n\n return corr\n\n\n# Determine the coefficients of the regression R=a*VIL+b.\n# See Section II.G of PCLH2020.\n# The parameters a and b are estimated in a localized fashion for each pixel\n# in the input grid. This is done using a window specified by window_radius.\n# Zero and non-finite values are not included. In addition, the regression is\n# done by using a Gaussian weight function depending on the distance to the\n# current grid point.\ndef _r_vil_regression(vil, r, window_radius):\n vil = vil.copy()\n vil[~np.isfinite(vil)] = 0.0\n\n r = r.copy()\n r[~np.isfinite(r)] = 0.0\n\n mask_vil = vil > 10.0\n mask_r = r > 0.1\n mask_obs = np.logical_and(mask_vil, mask_r)\n vil[~mask_obs] = 0.0\n r[~mask_obs] = 0.0\n\n n = gaussian_filter(mask_obs.astype(float), window_radius, mode=\"constant\")\n\n sx = gaussian_filter(vil, window_radius, mode=\"constant\")\n sx2 = gaussian_filter(vil * vil, window_radius, mode=\"constant\")\n sxy = gaussian_filter(vil * r, window_radius, mode=\"constant\")\n sy = gaussian_filter(r, window_radius, mode=\"constant\")\n\n rhs1 = sxy\n rhs2 = sy\n\n m1 = sx2\n m2 = sx\n m3 = sx\n m4 = n\n\n c = 1.0 / (m1 * m4 - m2 * m3)\n\n m_inv_11 = c * m4\n m_inv_12 = -c * m2\n m_inv_21 = -c * m3\n m_inv_22 = c * m1\n\n mask = np.abs(m1 * m4 - m2 * m3) > 1e-8\n mask = np.logical_and(mask, n > 0.01)\n a = np.empty(vil.shape)\n a[mask] = m_inv_11[mask] * rhs1[mask] + m_inv_12[mask] * rhs2[mask]\n a[~mask] = 0.0\n a[~mask_vil] = 0.0\n b = np.empty(vil.shape)\n b[mask] = m_inv_21[mask] * rhs1[mask] + m_inv_22[mask] * rhs2[mask]\n b[~mask] = 0.0\n b[~mask_vil] = 0.0\n\n return a, b\n\n\ndef _update(state, params):\n # iterate the ARI models for each cascade level\n for i in range(params[\"n_cascade_levels\"]):\n state[\"vil_dec\"][i, :] = autoregression.iterate_ar_model(\n state[\"vil_dec\"][i, :], params[\"phi\"][i]\n )\n\n # recompose the cascade to obtain the forecast field\n vil_dec_dict = {}\n vil_dec_dict[\"cascade_levels\"] = state[\"vil_dec\"][:, -1, :]\n vil_dec_dict[\"domain\"] = \"spatial\"\n vil_dec_dict[\"normalized\"] = False\n vil_f = params[\"recomp_method\"](vil_dec_dict)\n vil_f[~params[\"mask\"]] = np.nan\n\n if params[\"rainrate\"] is not None:\n # convert VIL to rain rate\n rainrate_f_new = params[\"r_vil_a\"] * vil_f + params[\"r_vil_b\"]\n else:\n rainrate_f_new = vil_f\n if params[\"apply_rainrate_mask\"]:\n rainrate_f_new[params[\"rainrate_mask\"]] = 0.0\n\n rainrate_f_new[rainrate_f_new < 0.0] = 0.0\n\n return rainrate_f_new, state\n" }, { "path": "pysteps/nowcasts/extrapolation.py", "content": "\"\"\"\npysteps.nowcasts.extrapolation\n==============================\n\nImplementation of extrapolation-based nowcasting methods.\n\n.. autosummary::\n :toctree: ../generated/\n\n forecast\n\"\"\"\n\nimport time\nimport numpy as np\n\nfrom pysteps import extrapolation\n\n\ndef forecast(\n precip,\n velocity,\n timesteps,\n extrap_method=\"semilagrangian\",\n extrap_kwargs=None,\n measure_time=False,\n):\n \"\"\"\n Generate a nowcast by applying a simple advection-based extrapolation to\n the given precipitation field.\n\n .. _ndarray: http://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\n\n Parameters\n ----------\n precip: array-like\n Two-dimensional array of shape (m,n) containing the input precipitation\n field.\n velocity: array-like\n Array of shape (2,m,n) containing the x- and y-components of the\n advection field. The velocities are assumed to represent one time step\n between the inputs.\n timesteps: int or list of floats\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed (relative to the input time step). The elements\n of the list are required to be in ascending order.\n extrap_method: str, optional\n Name of the extrapolation method to use. See the documentation of\n pysteps.extrapolation.interface.\n extrap_kwargs: dict, optional\n Optional dictionary that is expanded into keyword arguments for the\n extrapolation method.\n measure_time: bool, optional\n If True, measure, print, and return the computation time.\n\n Returns\n -------\n out: ndarray_\n Three-dimensional array of shape (num_timesteps, m, n) containing a time\n series of nowcast precipitation fields. The time series starts from\n t0 + timestep, where timestep is taken from the advection field velocity.\n If *measure_time* is True, the return value is a two-element tuple\n containing this array and the computation time (seconds).\n\n See also\n --------\n pysteps.extrapolation.interface\n \"\"\"\n\n _check_inputs(precip, velocity, timesteps)\n\n if extrap_kwargs is None:\n extrap_kwargs = dict()\n else:\n extrap_kwargs = extrap_kwargs.copy()\n\n extrap_kwargs[\"allow_nonfinite_values\"] = (\n True if np.any(~np.isfinite(precip)) else False\n )\n\n if measure_time:\n print(\n \"Computing extrapolation nowcast from a \"\n f\"{precip.shape[0]:d}x{precip.shape[1]:d} input grid... \",\n end=\"\",\n )\n\n if measure_time:\n start_time = time.time()\n\n extrapolation_method = extrapolation.get_method(extrap_method)\n\n precip_forecast = extrapolation_method(precip, velocity, timesteps, **extrap_kwargs)\n\n if measure_time:\n computation_time = time.time() - start_time\n print(f\"{computation_time:.2f} seconds.\")\n\n if measure_time:\n return precip_forecast, computation_time\n else:\n return precip_forecast\n\n\ndef _check_inputs(precip, velocity, timesteps):\n if precip.ndim != 2:\n raise ValueError(\"The input precipitation must be a \" \"two-dimensional array\")\n if velocity.ndim != 3:\n raise ValueError(\"Input velocity must be a three-dimensional array\")\n if precip.shape != velocity.shape[1:3]:\n raise ValueError(\n \"Dimension mismatch between \"\n \"input precipitation and velocity: \"\n + \"shape(precip)=%s, shape(velocity)=%s\"\n % (str(precip.shape), str(velocity.shape))\n )\n if isinstance(timesteps, list) and not sorted(timesteps) == timesteps:\n raise ValueError(\"timesteps is not in ascending order\")\n" }, { "path": "pysteps/nowcasts/interface.py", "content": "r\"\"\"\npysteps.nowcasts.interface\n==========================\n\nInterface for the nowcasts module. It returns a callable function for computing\nnowcasts.\n\nThe methods in the nowcasts module implement the following interface:\n\n ``forecast(precip, velocity, timesteps, **keywords)``\n\nwhere precip is a (m,n) array with input precipitation field to be advected and\nvelocity is a (2,m,n) array containing the x- and y-components of\nthe m x n advection field.\ntimesteps can be an integer or a list. An integer specifies the number of time\nsteps to forecast, where the output time step is taken from the inputs.\nIrregular time steps can be given in a list.\nThe interface accepts optional keyword arguments specific to the given method.\n\nThe output depends on the type of the method.\nFor deterministic methods, the output is a three-dimensional array of shape\n(num_timesteps,m,n) containing a time series of nowcast precipitation fields.\nFor stochastic methods that produce an ensemble, the output is a\nfour-dimensional array of shape (num_ensemble_members,num_timesteps,m,n).\nThe time step of the output is taken from the inputs.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nfrom pysteps.extrapolation.interface import eulerian_persistence\nfrom pysteps.nowcasts import (\n anvil,\n extrapolation,\n linda,\n sprog,\n steps,\n sseps,\n)\nfrom pysteps.nowcasts import lagrangian_probability\n\n_nowcast_methods = dict()\n_nowcast_methods[\"anvil\"] = anvil.forecast\n_nowcast_methods[\"eulerian\"] = eulerian_persistence\n_nowcast_methods[\"extrapolation\"] = extrapolation.forecast\n_nowcast_methods[\"lagrangian\"] = extrapolation.forecast\n_nowcast_methods[\"lagrangian_probability\"] = lagrangian_probability.forecast\n_nowcast_methods[\"linda\"] = linda.forecast\n_nowcast_methods[\"probability\"] = lagrangian_probability.forecast\n_nowcast_methods[\"sprog\"] = sprog.forecast\n_nowcast_methods[\"sseps\"] = sseps.forecast\n_nowcast_methods[\"steps\"] = steps.forecast\n\n\ndef get_method(name):\n r\"\"\"\n Return a callable function for computing nowcasts.\n\n Description:\n Return a callable function for computing deterministic or ensemble\n precipitation nowcasts.\n\n Implemented methods:\n\n +-----------------+-------------------------------------------------------+\n | Name | Description |\n +=================+=======================================================+\n | anvil | the autoregressive nowcasting using VIL (ANVIL) |\n | | nowcasting method developed in :cite:`PCLH2020` |\n +-----------------+-------------------------------------------------------+\n | eulerian | this approach keeps the last observation frozen |\n | | (Eulerian persistence) |\n +-----------------+-------------------------------------------------------+\n | lagrangian or | this approach extrapolates the last observation |\n | extrapolation | using the motion field (Lagrangian persistence) |\n +-----------------+-------------------------------------------------------+\n | linda | the LINDA method developed in :cite:`PCN2021` |\n +-----------------+-------------------------------------------------------+\n | lagrangian\\_ | this approach computes local lagrangian probability |\n | probability | forecasts of threshold exceedences |\n +-----------------+-------------------------------------------------------+\n | sprog | the S-PROG method described in :cite:`Seed2003` |\n +-----------------+-------------------------------------------------------+\n | steps | the STEPS stochastic nowcasting method described in |\n | | :cite:`Seed2003`, :cite:`BPS2006` and :cite:`SPN2013` |\n | | |\n +-----------------+-------------------------------------------------------+\n | sseps | short-space ensemble prediction system (SSEPS). |\n | | Essentially, this is a localization of STEPS |\n +-----------------+-------------------------------------------------------+\n \"\"\"\n if isinstance(name, str):\n name = name.lower()\n else:\n raise TypeError(\n \"Only strings supported for the method's names.\\n\"\n + \"Available names:\"\n + str(list(_nowcast_methods.keys()))\n ) from None\n\n try:\n return _nowcast_methods[name]\n except KeyError:\n raise ValueError(\n \"Unknown nowcasting method {}\\n\".format(name)\n + \"The available methods are:\"\n + str(list(_nowcast_methods.keys()))\n ) from None\n" }, { "path": "pysteps/nowcasts/lagrangian_probability.py", "content": "\"\"\"\npysteps.nowcasts.lagrangian_probability\n=======================================\n\nImplementation of the local Lagrangian probability nowcasting technique\ndescribed in :cite:`GZ2004`.\n\n.. autosummary::\n :toctree: ../generated/\n\n forecast\n\"\"\"\n\nimport numpy as np\nfrom scipy.signal import convolve\n\nfrom pysteps.nowcasts import extrapolation\n\n\ndef forecast(\n precip,\n velocity,\n timesteps,\n threshold,\n extrap_method=\"semilagrangian\",\n extrap_kwargs=None,\n slope=5,\n):\n \"\"\"\n Generate a probability nowcast by a local lagrangian approach. The ouput is\n the probability of exceeding a given intensity threshold, i.e.\n P(precip>=threshold).\n\n Parameters\n ----------\n precip: array_like\n Two-dimensional array of shape (m,n) containing the input precipitation\n field.\n velocity: array_like\n Array of shape (2,m,n) containing the x- and y-components of the\n advection field. The velocities are assumed to represent one time step\n between the inputs.\n timesteps: int or list of floats\n Number of time steps to forecast or a sorted list of time steps for which\n the forecasts are computed (relative to the input time step).\n The number of time steps has to be a positive integer.\n The elements of the list are required to be in ascending order.\n threshold: float\n Intensity threshold for which the exceedance probabilities are computed.\n slope: float, optional\n The slope of the relationship between optimum scale and lead time in\n pixels / timestep. Germann and Zawadzki (2004) found the optimal slope\n to be equal to 1 km / minute.\n\n Returns\n -------\n out: ndarray\n Three-dimensional array of shape (num_timesteps, m, n) containing a time\n series of nowcast exceedence probabilities. The time series starts from\n t0 + timestep, where timestep is taken from the advection field velocity.\n\n References\n ----------\n Germann, U. and I. Zawadzki, 2004:\n Scale Dependence of the Predictability of Precipitation from Continental\n Radar Images. Part II: Probability Forecasts.\n Journal of Applied Meteorology, 43(1), 74-89.\n \"\"\"\n # Compute deterministic extrapolation forecast\n if isinstance(timesteps, int) and timesteps > 0:\n timesteps = np.arange(1, timesteps + 1)\n elif not isinstance(timesteps, list):\n raise ValueError(f\"invalid value for argument 'timesteps': {timesteps}\")\n precip_forecast = extrapolation.forecast(\n precip,\n velocity,\n timesteps,\n extrap_method,\n extrap_kwargs,\n )\n\n # Ignore missing values\n nanmask = np.isnan(precip_forecast)\n precip_forecast[nanmask] = threshold - 1\n valid_pixels = (~nanmask).astype(float)\n\n # Compute exceedance probabilities using a neighborhood approach\n precip_forecast = (precip_forecast >= threshold).astype(float)\n for i, timestep in enumerate(timesteps):\n scale = int(timestep * slope)\n if scale == 0:\n continue\n kernel = _get_kernel(scale)\n kernel_sum = convolve(\n valid_pixels[i, ...],\n kernel,\n mode=\"same\",\n )\n precip_forecast[i, ...] = convolve(\n precip_forecast[i, ...],\n kernel,\n mode=\"same\",\n )\n precip_forecast[i, ...] /= kernel_sum\n precip_forecast = np.clip(precip_forecast, 0, 1)\n precip_forecast[nanmask] = np.nan\n return precip_forecast\n\n\ndef _get_kernel(size):\n \"\"\"\n Generate a circular kernel.\n\n Parameters\n ----------\n size : int\n Size of the circular kernel (its diameter). For size < 5, the kernel is\n a square instead of a circle.\n\n Returns\n -------\n 2-D array with kernel values\n \"\"\"\n middle = max((int(size / 2), 1))\n if size < 5:\n return np.ones((size, size), dtype=np.float32)\n else:\n xx, yy = np.mgrid[:size, :size]\n circle = (xx - middle) ** 2 + (yy - middle) ** 2\n return np.asarray(circle <= (middle**2), dtype=np.float32)\n" }, { "path": "pysteps/nowcasts/linda.py", "content": "\"\"\"\npysteps.nowcasts.linda\n======================\n\nThis module implements the Lagrangian INtegro-Difference equation model with\nAutoregression (LINDA). The model combines extrapolation, S-PROG, STEPS, ANVIL,\nintegro-difference equation (IDE) and cell tracking methods. It can produce\nboth deterministic and probabilistic nowcasts. LINDA is specifically designed\nfor nowcasting intense localized rainfall. For this purpose, it is expected to\ngive better forecast skill than S-PROG or STEPS.\n\nThe model consists of the following components:\n\n1. feature detection to identify rain cells\n2. advection-based extrapolation\n3. autoregressive integrated ARI(p,1) process for growth and decay of rainfall\n4. convolution to account for loss of predictability\n5. stochastic perturbations to simulate forecast errors\n\nLINDA utilizes a sparse feature-based representation of the input rain rate\nfields. This allows localization to cells containing intense rainfall. Building\non extrapolation nowcast, the temporal evolution of rainfall is modeled in the\nLagrangian coordinates. Using the ARI process is adapted from ANVIL\n:cite:`PCLH2020`, and the convolution is adapted from the integro-difference\nequation (IDE) models proposed in :cite:`FW2005` and :cite:`XWF2005`. The\ncombination of these two approaches essentially replaces the cascade-based\nautoregressive process used in S-PROG and STEPS. Using the convolution gives\nseveral advantages such as the ability to handle anisotropic structure, domain\nboundaries and missing data. Based on the marginal distribution and covariance\nstructure of forecast errors, localized perturbations are generated by adapting\nthe short-space Fourier transform (SSFT) methodology developed in\n:cite:`NBSG2017`.\n\n.. autosummary::\n :toctree: ../generated/\n\n forecast\n\"\"\"\n\nimport time\nimport warnings\n\nfrom pysteps.utils.check_norain import check_norain\n\ntry:\n import dask\n\n DASK_IMPORTED = True\nexcept ImportError:\n DASK_IMPORTED = False\nimport numpy as np\nfrom scipy import optimize as opt\nfrom scipy import stats\nfrom scipy.integrate import nquad\nfrom scipy.interpolate import interp1d\nfrom scipy.signal import convolve\n\nfrom pysteps import extrapolation, feature, noise\nfrom pysteps.nowcasts.utils import nowcast_main_loop, zero_precipitation_forecast\n\n\ndef forecast(\n precip,\n velocity,\n timesteps,\n feature_method=\"blob\",\n max_num_features=25,\n feature_kwargs=None,\n ari_order=1,\n kernel_type=\"anisotropic\",\n localization_window_radius=None,\n errdist_window_radius=None,\n acf_window_radius=None,\n extrap_method=\"semilagrangian\",\n extrap_kwargs=None,\n add_perturbations=True,\n pert_thrs=(0.5, 1.0),\n n_ens_members=10,\n vel_pert_method=\"bps\",\n vel_pert_kwargs=None,\n kmperpixel=None,\n timestep=None,\n seed=None,\n num_workers=1,\n use_multiprocessing=False,\n measure_time=False,\n callback=None,\n return_output=True,\n):\n \"\"\"\n Generate a deterministic or ensemble nowcast by using the Lagrangian\n INtegro-Difference equation model with Autoregression (LINDA) model.\n\n Parameters\n ----------\n precip: array_like\n Array of shape (ari_order + 2, m, n) containing the input rain rate\n or reflectivity fields (in linear scale) ordered by timestamp from\n oldest to newest. The time steps between the inputs are assumed to be\n regular.\n velocity: array_like\n Array of shape (2, m, n) containing the x- and y-components of the\n advection field. The velocities are assumed to represent one time step\n between the inputs.\n timesteps: int or list of floats\n Number of time steps to forecast or a list of time steps. If a list is\n given, the values are assumed to be relative to the input time step and\n in ascending order.\n feature_method: {'blob', 'domain' 'shitomasi'}\n Feature detection method:\n\n +-------------------+-----------------------------------------------------+\n | Method name | Description |\n +===================+=====================================================+\n | blob | Laplacian of Gaussian (LoG) blob detector |\n | | implemented in scikit-image |\n +-------------------+-----------------------------------------------------+\n | domain | no feature detection, the model is applied over the |\n | | whole domain without localization |\n +-------------------+-----------------------------------------------------+\n | shitomasi | Shi-Tomasi corner detector implemented in OpenCV |\n +-------------------+-----------------------------------------------------+\n\n Default: 'blob'\n max_num_features: int, optional\n Maximum number of features to use. It is recommended to set this between\n 20 and 50, which gives a good tradeoff between localization and\n computation time. Default: 25\n feature_kwargs: dict, optional\n Keyword arguments that are passed as ``**kwargs`` for the feature detector.\n See :py:mod:`pysteps.feature.blob` and :py:mod:`pysteps.feature.shitomasi`.\n ari_order: {1, 2}, optional\n The order of the ARI(p, 1) model. Default: 1\n kernel_type: {\"anisotropic\", \"isotropic\"}, optional\n The type of the kernel. Default: 'anisotropic'\n localization_window_radius: float, optional\n The standard deviation of the Gaussian localization window.\n Default: 0.2 * min(m, n)\n errdist_window_radius: float, optional\n The standard deviation of the Gaussian window for estimating the\n forecast error distribution. Default: 0.15 * min(m, n)\n acf_window_radius: float, optional\n The standard deviation of the Gaussian window for estimating the\n forecast error ACF. Default: 0.25 * min(m, n)\n extrap_method: str, optional\n The extrapolation method to use. See the documentation of\n :py:mod:`pysteps.extrapolation.interface`. Default: 'semilagrangian'\n extrap_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the extrapolation\n method. See :py:mod:`pysteps.extrapolation.interface`.\n add_perturbations: bool\n Set to False to disable perturbations and generate a single\n deterministic nowcast. Default: True\n pert_thrs: float\n Two-element tuple containing the threshold values for estimating the\n perturbation parameters (mm/h). Default: (0.5, 1.0)\n n_ens_members: int, optional\n The number of ensemble members to generate. Default: 10\n vel_pert_method: {'bps', None}, optional\n Name of the generator to use for perturbing the advection field. See\n :py:mod:`pysteps.noise.interface`. Default: 'bps'\n vel_pert_kwargs: dict, optional\n Optional dictionary containing keyword arguments 'p_par' and 'p_perp'\n for the initializer of the velocity perturbator. The choice of the\n optimal parameters depends on the domain and the used optical flow\n method. For the default values and parameters optimized for different\n domains, see :py:func:`pysteps.nowcasts.steps.forecast`.\n kmperpixel: float, optional\n Spatial resolution of the input data (kilometers/pixel). Required if\n vel_pert_method is not None.\n timestep: float, optional\n Time step of the motion vectors (minutes). Required if vel_pert_method\n is not None.\n seed: int, optional\n Optional seed for the random generators.\n num_workers: int, optional\n The number of workers to use for parallel computations. Applicable if\n dask is installed. Default: 1\n use_multiprocessing: bool, optional\n Set to True to improve the performance of certain parallelized parts of\n the code. If set to True, the main script calling linda.forecast must\n be enclosed within the 'if __name__ == \"__main__\":' block.\n Default: False\n measure_time: bool, optional\n If set to True, measure, print and return the computation time.\n Default: False\n callback: function, optional\n Optional function that is called after computation of each time step of\n the nowcast. The function takes one argument: a three-dimensional array\n of shape (n_ens_members,h,w), where h and w are the height and width\n of the input precipitation fields, respectively. This can be used, for\n instance, writing the outputs into files. Default: None\n return_output: bool, optional\n Set to False to disable returning the outputs as numpy arrays. This can\n save memory if the intermediate results are written to output files\n using the callback function. Default: True\n\n Returns\n -------\n out: numpy.ndarray\n A four-dimensional array of shape (n_ens_members, len(timesteps), m, n)\n containing a time series of forecast precipitation fields for each\n ensemble member. If add_perturbations is False, the first dimension is\n dropped. The time series starts from t0 + timestep, where timestep is\n taken from the input fields. If measure_time is True, the return value\n is a three-element tuple containing the nowcast array, the initialization\n time of the nowcast generator and the time used in the main loop\n (seconds). If return_output is set to False, a single None value is\n returned instead.\n\n Notes\n -----\n It is recommended to choose the feature detector parameters so that the\n number of features is around 20-40. This gives a good tradeoff between\n localization and computation time.\n\n It is highly recommented to set num_workers>1 to reduce computation time.\n In this case, it is advisable to disable OpenMP by setting the environment\n variable OMP_NUM_THREADS to 1. This avoids slowdown caused by too many\n simultaneous threads.\n \"\"\"\n _check_inputs(precip, velocity, timesteps, ari_order)\n\n if feature_kwargs is None:\n feature_kwargs = dict()\n if extrap_kwargs is None:\n extrap_kwargs = dict()\n else:\n extrap_kwargs = extrap_kwargs.copy()\n\n if localization_window_radius is None:\n localization_window_radius = 0.2 * np.min(precip.shape[1:])\n if add_perturbations:\n if errdist_window_radius is None:\n errdist_window_radius = 0.15 * min(precip.shape[1], precip.shape[2])\n if acf_window_radius is None:\n acf_window_radius = 0.25 * min(precip.shape[1], precip.shape[2])\n if vel_pert_method is not None:\n if kmperpixel is None:\n raise ValueError(\"vel_pert_method is set but kmperpixel is None\")\n if timestep is None:\n raise ValueError(\"vel_pert_method is set but timestep is None\")\n if vel_pert_kwargs is None:\n vel_pert_kwargs = dict()\n\n print(\"Computing LINDA nowcast\")\n print(\"-----------------------\")\n print(\"\")\n\n print(\"Inputs\")\n print(\"------\")\n print(f\"dimensions: {precip.shape[1]}x{precip.shape[2]}\")\n print(f\"number of time steps: {precip.shape[0]}\")\n print(\"\")\n\n print(\"Methods\")\n print(\"-------\")\n nowcast_type = \"ensemble\" if add_perturbations else \"deterministic\"\n print(f\"nowcast type: {nowcast_type}\")\n print(f\"feature detector: {feature_method}\")\n print(f\"extrapolator: {extrap_method}\")\n print(f\"kernel type: {kernel_type}\")\n if add_perturbations and vel_pert_method is not None:\n print(f\"velocity perturbator: {vel_pert_method}\")\n print(\"\")\n\n print(\"Parameters\")\n print(\"----------\")\n if isinstance(timesteps, int):\n print(f\"number of time steps: {timesteps}\")\n else:\n print(f\"time steps: {timesteps}\")\n print(f\"ARI model order: {ari_order}\")\n print(f\"localization window radius: {localization_window_radius}\")\n if add_perturbations:\n print(f\"error dist. window radius: {errdist_window_radius}\")\n print(f\"error ACF window radius: {acf_window_radius}\")\n print(f\"ensemble size: {n_ens_members}\")\n print(f\"parallel workers: {num_workers}\")\n print(f\"seed: {seed}\")\n if vel_pert_method == \"bps\":\n vp_par = vel_pert_kwargs.get(\n \"p_par\", noise.motion.get_default_params_bps_par()\n )\n vp_perp = vel_pert_kwargs.get(\n \"p_perp\", noise.motion.get_default_params_bps_perp()\n )\n print(\n f\"velocity perturbations, parallel: {vp_par[0]:.2f}, {vp_par[1]:.2f}, {vp_par[2]:.2f}\"\n )\n print(\n f\"velocity perturbations, perpendicular: {vp_perp[0]:.2f}, {vp_perp[1]:.2f}, {vp_perp[2]:.2f}\"\n )\n vel_pert_kwargs = vel_pert_kwargs.copy()\n vel_pert_kwargs[\"vp_par\"] = vp_par\n vel_pert_kwargs[\"vp_perp\"] = vp_perp\n\n extrap_kwargs[\"allow_nonfinite_values\"] = (\n True if np.any(~np.isfinite(precip)) else False\n )\n\n starttime_init = time.time()\n\n if check_norain(precip, 0.0, 0.0, None):\n return zero_precipitation_forecast(\n n_ens_members if nowcast_type == \"ensemble\" else None,\n timesteps,\n precip,\n callback,\n return_output,\n measure_time,\n starttime_init,\n )\n\n forecast_gen = _linda_deterministic_init(\n precip,\n velocity,\n feature_method,\n max_num_features,\n feature_kwargs,\n ari_order,\n kernel_type,\n localization_window_radius,\n extrap_method,\n extrap_kwargs,\n add_perturbations,\n num_workers,\n measure_time,\n )\n if measure_time:\n forecast_gen, precip_lagr_diff, init_time = forecast_gen\n else:\n forecast_gen, precip_lagr_diff = forecast_gen\n\n if add_perturbations:\n pert_gen = _linda_perturbation_init(\n precip,\n precip_lagr_diff,\n velocity,\n forecast_gen,\n pert_thrs,\n localization_window_radius,\n errdist_window_radius,\n acf_window_radius,\n vel_pert_method,\n vel_pert_kwargs,\n kmperpixel,\n timestep,\n num_workers,\n use_multiprocessing,\n measure_time,\n )\n if measure_time:\n precip_pert_gen, velocity_pert_gen, pert_init_time = pert_gen\n init_time += pert_init_time\n else:\n precip_pert_gen, velocity_pert_gen = pert_gen\n else:\n precip_pert_gen = None\n velocity_pert_gen = None\n\n precip_forecast = _linda_forecast(\n precip,\n precip_lagr_diff[1:],\n timesteps,\n forecast_gen,\n precip_pert_gen,\n velocity_pert_gen,\n n_ens_members,\n seed,\n measure_time,\n True,\n return_output,\n callback,\n )\n\n if return_output:\n if measure_time:\n return precip_forecast[0], init_time, precip_forecast[1]\n else:\n return precip_forecast\n else:\n return None\n\n\ndef _check_inputs(precip, velocity, timesteps, ari_order):\n if ari_order not in [1, 2]:\n raise ValueError(f\"ari_order {ari_order} given, 1 or 2 required\")\n if len(precip.shape) != 3:\n raise ValueError(\"precip must be a three-dimensional array\")\n if precip.shape[0] < ari_order + 2:\n raise ValueError(\"precip.shape[0] < ari_order+2\")\n if len(velocity.shape) != 3:\n raise ValueError(\"velocity must be a three-dimensional array\")\n if precip.shape[1:3] != velocity.shape[1:3]:\n raise ValueError(\n f\"dimension mismatch between precip and velocity: precip.shape={precip.shape}, velocity.shape={velocity.shape}\"\n )\n if isinstance(timesteps, list) and not sorted(timesteps) == timesteps:\n raise ValueError(\"timesteps must be in ascending order\")\n\n\ndef _composite_convolution(field, kernels, weights):\n \"\"\"\n Compute a localized convolution by applying a set of kernels with the\n given spatial weights. The weights are assumed to be normalized.\n \"\"\"\n n = len(kernels)\n field_c = 0.0\n\n for i in range(n):\n field_c += weights[i] * _masked_convolution(field, kernels[i])\n\n return field_c\n\n\ndef _compute_ellipse_bbox(phi, sigma1, sigma2, cutoff):\n \"\"\"Compute the bounding box of an ellipse.\"\"\"\n r1 = cutoff * sigma1\n r2 = cutoff * sigma2\n phi_r = phi / 180.0 * np.pi\n\n if np.abs(phi_r - np.pi / 2) > 1e-6 and np.abs(phi_r - 3 * np.pi / 2) > 1e-6:\n alpha = np.arctan(-r2 * np.sin(phi_r) / (r1 * np.cos(phi_r)))\n w = r1 * np.cos(alpha) * np.cos(phi_r) - r2 * np.sin(alpha) * np.sin(phi_r)\n\n alpha = np.arctan(r2 * np.cos(phi_r) / (r1 * np.sin(phi_r)))\n h = r1 * np.cos(alpha) * np.sin(phi_r) + r2 * np.sin(alpha) * np.cos(phi_r)\n else:\n w = sigma2 * cutoff\n h = sigma1 * cutoff\n\n return -abs(h), -abs(w), abs(h), abs(w)\n\n\ndef _compute_inverse_acf_mapping(target_dist, target_dist_params, n_intervals=10):\n \"\"\"Compute the inverse ACF mapping between two distributions.\"\"\"\n phi = (\n lambda x1, x2, rho: 1.0\n / (2 * np.pi * np.sqrt(1 - rho**2))\n * np.exp(-(x1**2 + x2**2 - 2 * rho * x1 * x2) / (2 * (1 - rho**2)))\n )\n\n rho_1 = np.linspace(-0.9, 0.9, n_intervals)\n rho_2 = np.empty(len(rho_1))\n\n mu = target_dist.mean(*target_dist_params)\n sigma = target_dist.std(*target_dist_params)\n\n cdf_trans = lambda x: target_dist.ppf(stats.norm.cdf(x), *target_dist_params)\n int_range = (-6, 6)\n\n for i, rho_1_ in enumerate(rho_1):\n f = (\n lambda x1, x2: (cdf_trans(x1) - mu)\n * (cdf_trans(x2) - mu)\n * phi(x1, x2, rho_1_)\n )\n opts = {\"epsabs\": 1e-8, \"epsrel\": 1e-8, \"limit\": 1}\n rho_2[i] = nquad(f, (int_range, int_range), opts=opts)[0] / (sigma * sigma)\n\n return interp1d(rho_2, rho_1, fill_value=\"extrapolate\")\n\n\ndef _compute_kernel_anisotropic(params, cutoff=6.0):\n \"\"\"Compute anisotropic Gaussian convolution kernel.\"\"\"\n phi, sigma1, sigma2 = params\n\n phi_r = phi / 180.0 * np.pi\n rot_inv = np.array(\n [[np.cos(phi_r), np.sin(phi_r)], [-np.sin(phi_r), np.cos(phi_r)]]\n )\n\n bb_y1, bb_x1, bb_y2, bb_x2 = _compute_ellipse_bbox(phi, sigma1, sigma2, cutoff)\n\n x = np.arange(int(bb_x1), int(bb_x2) + 1).astype(float)\n if len(x) % 2 == 0:\n x = np.arange(int(bb_x1) - 1, int(bb_x2) + 1).astype(float)\n y = np.arange(int(bb_y1), int(bb_y2) + 1).astype(float)\n if len(y) % 2 == 0:\n y = np.arange(int(bb_y1) - 1, int(bb_y2) + 1).astype(float)\n\n x_grid, y_grid = np.meshgrid(x, y)\n xy_grid = np.vstack([x_grid.flatten(), y_grid.flatten()])\n xy_grid = np.dot(rot_inv, xy_grid)\n\n x2 = xy_grid[0, :] * xy_grid[0, :]\n y2 = xy_grid[1, :] * xy_grid[1, :]\n result = np.exp(-(x2 / sigma1**2 + y2 / sigma2**2))\n\n return np.reshape(result / np.sum(result), x_grid.shape)\n\n\ndef _compute_kernel_isotropic(sigma, cutoff=6.0):\n \"\"\"Compute isotropic Gaussian convolution kernel.\"\"\"\n bb_y1, bb_x1, bb_y2, bb_x2 = (\n -sigma * cutoff,\n -sigma * cutoff,\n sigma * cutoff,\n sigma * cutoff,\n )\n\n x = np.arange(int(bb_x1), int(bb_x2) + 1).astype(float)\n if len(x) % 2 == 0:\n x = np.arange(int(bb_x1) - 1, int(bb_x2) + 1).astype(float)\n y = np.arange(int(bb_y1), int(bb_y2) + 1).astype(float)\n if len(y) % 2 == 0:\n y = np.arange(int(bb_y1) - 1, int(bb_y2) + 1).astype(float)\n\n x_grid, y_grid = np.meshgrid(x / sigma, y / sigma)\n\n r2 = x_grid * x_grid + y_grid * y_grid\n result = np.exp(-0.5 * r2)\n\n return result / np.sum(result)\n\n\ndef _compute_parametric_acf(params, m, n):\n \"\"\"Compute parametric ACF.\"\"\"\n c, phi, sigma1, sigma2 = params\n\n phi_r = phi / 180.0 * np.pi\n rot_inv = np.array(\n [[np.cos(phi_r), np.sin(phi_r)], [-np.sin(phi_r), np.cos(phi_r)]]\n )\n\n if n % 2 == 0:\n n_max = int(n / 2)\n else:\n n_max = int(n / 2) + 1\n x = np.fft.ifftshift(np.arange(-int(n / 2), n_max))\n if m % 2 == 0:\n m_max = int(m / 2)\n else:\n m_max = int(m / 2) + 1\n y = np.fft.ifftshift(np.arange(-int(m / 2), m_max))\n\n grid_x, grid_y = np.meshgrid(x, y)\n grid_xy = np.vstack([grid_x.flatten(), grid_y.flatten()])\n grid_xy = np.dot(rot_inv, grid_xy)\n\n grid_xy[0, :] = grid_xy[0, :] / sigma1\n grid_xy[1, :] = grid_xy[1, :] / sigma2\n\n r2 = np.reshape(\n grid_xy[0, :] * grid_xy[0, :] + grid_xy[1, :] * grid_xy[1, :], grid_x.shape\n )\n result = np.exp(-np.sqrt(r2))\n\n return c * result\n\n\ndef _compute_sample_acf(field):\n \"\"\"Compute sample ACF from FFT.\"\"\"\n # TODO: let user choose the FFT method\n field_fft = np.fft.rfft2((field - np.mean(field)) / np.std(field))\n fft_abs = np.abs(field_fft * np.conj(field_fft))\n\n return np.fft.irfft2(fft_abs, s=field.shape) / (field.shape[0] * field.shape[1])\n\n\ndef _compute_window_weights(coords, grid_height, grid_width, window_radius):\n \"\"\"Compute interpolation weights.\"\"\"\n coords = coords.astype(float).copy()\n num_features = coords.shape[0]\n\n coords[:, 0] /= grid_height\n coords[:, 1] /= grid_width\n\n window_radius_1 = window_radius / grid_height\n window_radius_2 = window_radius / grid_width\n\n grid_x = (np.arange(grid_width) + 0.5) / grid_width\n grid_y = (np.arange(grid_height) + 0.5) / grid_height\n\n grid_x, grid_y = np.meshgrid(grid_x, grid_y)\n\n w = np.empty((num_features, grid_x.shape[0], grid_x.shape[1]))\n\n if coords.shape[0] > 1:\n for i, c in enumerate(coords):\n dy = c[0] - grid_y\n dx = c[1] - grid_x\n\n w[i, :] = np.exp(\n -dy * dy / (2 * window_radius_1**2) - dx * dx / (2 * window_radius_2**2)\n )\n else:\n w[0, :] = np.ones((grid_height, grid_width))\n\n return w\n\n\ndef _estimate_ar1_params(\n field_src, field_dst, estim_weights, interp_weights, num_workers=1\n):\n \"\"\"Constrained optimization of AR(1) parameters.\"\"\"\n\n def objf(p, *args):\n i = args[0]\n field_ar = p * field_src\n return np.nansum(estim_weights[i] * (field_dst - field_ar) ** 2.0)\n\n bounds = (-0.98, 0.98)\n\n def worker(i):\n return opt.minimize_scalar(objf, method=\"bounded\", bounds=bounds, args=(i,)).x\n\n if DASK_IMPORTED and num_workers > 1:\n res = []\n for i in range(len(estim_weights)):\n res.append(dask.delayed(worker)(i))\n\n psi = dask.compute(*res, num_workers=num_workers, scheduler=\"threads\")\n else:\n psi = []\n for i in range(len(estim_weights)):\n psi.append(worker(i))\n\n return [np.sum([psi_ * interp_weights[i] for i, psi_ in enumerate(psi)], axis=0)]\n\n\ndef _estimate_ar2_params(\n field_src, field_dst, estim_weights, interp_weights, num_workers=1\n):\n \"\"\"Constrained optimization of AR(2) parameters.\"\"\"\n\n def objf(p, *args):\n i = args[0]\n field_ar = p[0] * field_src[1] + p[1] * field_src[0]\n return np.nansum(estim_weights[i] * (field_dst - field_ar) ** 2.0)\n\n bounds = [(-1.98, 1.98), (-0.98, 0.98)]\n constraints = [\n opt.LinearConstraint(\n np.array([(1, 1), (-1, 1)]),\n (-np.inf, -np.inf),\n (0.98, 0.98),\n keep_feasible=True,\n )\n ]\n\n def worker(i):\n return opt.minimize(\n objf,\n (0.8, 0.0),\n method=\"trust-constr\",\n bounds=bounds,\n constraints=constraints,\n args=(i,),\n ).x\n\n if DASK_IMPORTED and num_workers > 1:\n res = []\n for i in range(len(estim_weights)):\n res.append(dask.delayed(worker)(i))\n\n psi = dask.compute(*res, num_workers=num_workers, scheduler=\"threads\")\n else:\n psi = []\n for i in range(len(estim_weights)):\n psi.append(worker(i))\n\n psi_out = []\n for i in range(2):\n psi_out.append(\n np.sum([psi[j][i] * interp_weights[j] for j in range(len(psi))], axis=0)\n )\n\n return psi_out\n\n\ndef _estimate_convol_params(\n field_src,\n field_dst,\n weights,\n mask,\n kernel_type=\"anisotropic\",\n kernel_params=None,\n num_workers=1,\n):\n \"\"\"Estimation of convolution kernel.\"\"\"\n if kernel_params is None:\n kernel_params = {}\n masks = []\n for weight in weights:\n masks.append(np.logical_and(mask, weight > 1e-3))\n\n def objf_aniso(p, *args):\n i = args[0]\n p = _get_anisotropic_kernel_params(p)\n kernel = _compute_kernel_anisotropic(p, **kernel_params)\n\n field_src_c = _masked_convolution(field_src, kernel)\n fval = np.sqrt(weights[i][masks[i]]) * (\n field_dst[masks[i]] - field_src_c[masks[i]]\n )\n\n return fval\n\n def objf_iso(p, *args):\n i = args[0]\n kernel = _compute_kernel_isotropic(p, **kernel_params)\n\n field_src_c = _masked_convolution(field_src, kernel)\n fval = np.sum(\n weights[i][masks[i]] * (field_dst[masks[i]] - field_src_c[masks[i]]) ** 2\n )\n\n return fval\n\n def worker(i):\n if kernel_type == \"anisotropic\":\n bounds = ((-np.inf, 0.1, 0.2), (np.inf, 10.0, 5.0))\n p_opt = opt.least_squares(\n objf_aniso,\n np.array((0.0, 1.0, 1.0)),\n bounds=bounds,\n method=\"trf\",\n ftol=1e-6,\n xtol=1e-4,\n gtol=1e-6,\n args=(i,),\n )\n p_opt = _get_anisotropic_kernel_params(p_opt.x)\n\n return _compute_kernel_anisotropic(p_opt, **kernel_params)\n else:\n p_opt = opt.minimize_scalar(\n objf_iso, bounds=[0.01, 10.0], method=\"bounded\", args=(i,)\n )\n p_opt = p_opt.x\n\n return _compute_kernel_isotropic(p_opt, **kernel_params)\n\n if DASK_IMPORTED and num_workers > 1:\n res = []\n for i in range(len(weights)):\n res.append(dask.delayed(worker)(i))\n kernels = dask.compute(*res, num_workers=num_workers, scheduler=\"threads\")\n else:\n kernels = []\n for i in range(len(weights)):\n kernels.append(worker(i))\n\n return kernels\n\n\ndef _estimate_perturbation_params(\n forecast_err,\n forecast_gen,\n errdist_window_radius,\n acf_window_radius,\n interp_window_radius,\n measure_time,\n num_workers,\n use_multiprocessing,\n):\n \"\"\"\n Estimate perturbation generator parameters from forecast errors.\"\"\"\n pert_gen = {}\n pert_gen[\"m\"] = forecast_err.shape[0]\n pert_gen[\"n\"] = forecast_err.shape[1]\n\n feature_coords = forecast_gen[\"feature_coords\"]\n\n print(\"Estimating perturbation parameters... \", end=\"\", flush=True)\n\n if measure_time:\n starttime = time.time()\n\n mask_finite = np.isfinite(forecast_err)\n\n forecast_err = forecast_err.copy()\n forecast_err[~mask_finite] = 1.0\n\n weights_dist = _compute_window_weights(\n feature_coords,\n forecast_err.shape[0],\n forecast_err.shape[1],\n errdist_window_radius,\n )\n\n acf_winfunc = _window_tukey if feature_coords.shape[0] > 1 else _window_uniform\n\n def worker(i):\n weights_acf = acf_winfunc(\n forecast_err.shape[0],\n forecast_err.shape[1],\n feature_coords[i, 0],\n feature_coords[i, 1],\n acf_window_radius,\n acf_window_radius,\n )\n\n mask = np.logical_and(mask_finite, weights_dist[i] > 0.1)\n if np.sum(mask) > 10 and np.sum(np.abs(forecast_err[mask] - 1.0) >= 1e-3) > 10:\n distpar = _fit_dist(forecast_err, stats.lognorm, weights_dist[i], mask)\n inv_acf_mapping = _compute_inverse_acf_mapping(stats.lognorm, distpar)\n mask_acf = weights_acf > 1e-4\n std = _weighted_std(forecast_err[mask_acf], weights_dist[i][mask_acf])\n if np.isfinite(std):\n acf = inv_acf_mapping(\n _compute_sample_acf(weights_acf * (forecast_err - 1.0) / std)\n )\n acf = _fit_acf(acf)\n\n valid_data = True\n else:\n valid_data = False\n else:\n valid_data = False\n\n if valid_data:\n return distpar, std, np.sqrt(np.abs(np.fft.rfft2(acf)))\n else:\n return (\n (1e-10, 1e-10),\n 1e-10,\n np.ones((weights_acf.shape[0], int(weights_acf.shape[1] / 2) + 1))\n * 1e-10,\n )\n\n dist_params = []\n stds = []\n acf_fft_ampl = []\n\n if DASK_IMPORTED and num_workers > 1:\n res = []\n for i in range(feature_coords.shape[0]):\n res.append(dask.delayed(worker)(i))\n scheduler = \"threads\" if not use_multiprocessing else \"multiprocessing\"\n res = dask.compute(*res, num_workers=num_workers, scheduler=scheduler)\n for r in res:\n dist_params.append(r[0])\n stds.append(r[1])\n acf_fft_ampl.append(r[2])\n else:\n for i in range(feature_coords.shape[0]):\n r = worker(i)\n dist_params.append(r[0])\n stds.append(r[1])\n acf_fft_ampl.append(r[2])\n\n pert_gen[\"dist_param\"] = dist_params\n pert_gen[\"std\"] = stds\n pert_gen[\"acf_fft_ampl\"] = acf_fft_ampl\n\n weights = _compute_window_weights(\n feature_coords,\n forecast_err.shape[0],\n forecast_err.shape[1],\n interp_window_radius,\n )\n pert_gen[\"weights\"] = weights / np.sum(weights, axis=0)\n\n if measure_time:\n print(f\"{time.time() - starttime:.2f} seconds.\")\n else:\n print(\"done.\")\n\n return pert_gen\n\n\ndef _fit_acf(acf):\n \"\"\"\n Fit a parametric ACF to the given sample estimate.\"\"\"\n\n def objf(p, *args):\n p = _get_acf_params(p)\n fitted_acf = _compute_parametric_acf(p, acf.shape[0], acf.shape[1])\n\n return (acf - fitted_acf).flatten()\n\n bounds = ((0.01, -np.inf, 0.1, 0.2), (10.0, np.inf, 10.0, 5.0))\n p_opt = opt.least_squares(\n objf,\n np.array((1.0, 0.0, 1.0, 1.0)),\n bounds=bounds,\n method=\"trf\",\n ftol=1e-6,\n xtol=1e-4,\n gtol=1e-6,\n )\n\n return _compute_parametric_acf(_get_acf_params(p_opt.x), acf.shape[0], acf.shape[1])\n\n\ndef _fit_dist(err, dist, wf, mask):\n \"\"\"\n Fit a lognormal distribution by maximizing the log-likelihood function\n with the constraint that the mean value is one.\"\"\"\n func = lambda p: -np.sum(np.log(stats.lognorm.pdf(err[mask], p, -0.5 * p**2)))\n p_opt = opt.minimize_scalar(func, bounds=(1e-3, 20.0), method=\"Bounded\")\n\n return (p_opt.x, -0.5 * p_opt.x**2)\n\n\n# TODO: restrict the perturbation generation inside the radar mask\ndef _generate_perturbations(pert_gen, num_workers, seed):\n \"\"\"Generate perturbations based on the estimated forecast error statistics.\"\"\"\n m, n = pert_gen[\"m\"], pert_gen[\"n\"]\n dist_param = pert_gen[\"dist_param\"]\n std = pert_gen[\"std\"]\n acf_fft_ampl = pert_gen[\"acf_fft_ampl\"]\n weights = pert_gen[\"weights\"]\n\n perturb = stats.norm.rvs(size=(m, n), random_state=seed)\n perturb_fft = np.fft.rfft2(perturb)\n\n out = np.zeros((m, n))\n\n def worker(i):\n if std[i] > 0.0:\n filtered_noise = np.fft.irfft2(acf_fft_ampl[i] * perturb_fft, s=(m, n))\n filtered_noise /= np.std(filtered_noise)\n filtered_noise = stats.lognorm.ppf(\n stats.norm.cdf(filtered_noise), *dist_param[i]\n )\n else:\n filtered_noise = np.ones(weights[i].shape)\n\n return weights[i] * filtered_noise\n\n if DASK_IMPORTED and num_workers > 1:\n res = []\n for i in range(weights.shape[0]):\n res.append(dask.delayed(worker)(i))\n res = dask.compute(*res, num_workers=num_workers, scheduler=\"threads\")\n for r in res:\n out += r\n else:\n for i in range(weights.shape[0]):\n out += worker(i)\n\n return out\n\n\ndef _get_acf_params(p):\n \"\"\"Get ACF parameters from the given parameter vector.\"\"\"\n return p[0], p[1], p[2], p[3] * p[2]\n\n\ndef _get_anisotropic_kernel_params(p):\n \"\"\"Get anisotropic convolution kernel parameters from the given parameter\n vector.\"\"\"\n return p[0], p[1], p[2] * p[1]\n\n\n# TODO: use the method implemented in pysteps.timeseries.autoregression\ndef _iterate_ar_model(input_fields, psi):\n \"\"\"Iterate autoregressive process.\"\"\"\n input_field_new = 0.0\n\n for i, psi_ in enumerate(psi):\n input_field_new += psi_ * input_fields[-(i + 1), :]\n\n return np.concatenate([input_fields[1:, :], input_field_new[np.newaxis, :]])\n\n\ndef _linda_forecast(\n precip,\n precip_lagr_diff,\n timesteps,\n forecast_gen,\n precip_pert_gen,\n velocity_pert_gen,\n n_ensemble_members,\n seed,\n measure_time,\n print_info,\n return_output,\n callback,\n):\n \"\"\"Compute LINDA nowcast.\"\"\"\n # compute convolved difference fields\n precip_lagr_diff = precip_lagr_diff.copy()\n\n for i in range(precip_lagr_diff.shape[0]):\n for _ in range(forecast_gen[\"ari_order\"] - i):\n precip_lagr_diff[i] = _composite_convolution(\n precip_lagr_diff[i],\n forecast_gen[\"kernels_1\"],\n forecast_gen[\"interp_weights\"],\n )\n\n # initialize the random generators\n if precip_pert_gen is not None:\n rs_precip_pert = []\n np.random.seed(seed)\n for _ in range(n_ensemble_members):\n rs = np.random.RandomState(seed)\n rs_precip_pert.append(rs)\n seed = rs.randint(0, high=1e9)\n else:\n rs_precip_pert = None\n\n if velocity_pert_gen is not None:\n velocity_perturbators = []\n np.random.seed(seed)\n for _ in range(n_ensemble_members):\n vp = velocity_pert_gen[\"init_func\"](seed)\n velocity_perturbators.append(\n lambda t, vp=vp: velocity_pert_gen[\"gen_func\"](\n vp, t * velocity_pert_gen[\"timestep\"]\n )\n )\n seed = np.random.RandomState(seed).randint(0, high=1e9)\n else:\n velocity_perturbators = None\n\n state = {\n \"precip_forecast\": [precip[-1].copy() for _ in range(n_ensemble_members)],\n \"precip_lagr_diff\": [\n precip_lagr_diff.copy() for _ in range(n_ensemble_members)\n ],\n \"rs_precip_pert\": rs_precip_pert,\n }\n params = {\n \"interp_weights\": forecast_gen[\"interp_weights\"],\n \"kernels_1\": forecast_gen[\"kernels_1\"],\n \"kernels_2\": forecast_gen[\"kernels_2\"],\n \"mask_adv\": forecast_gen[\"mask_adv\"],\n \"num_ens_members\": n_ensemble_members,\n \"num_workers\": forecast_gen[\"num_workers\"],\n \"num_ensemble_workers\": min(n_ensemble_members, forecast_gen[\"num_workers\"]),\n \"precip_pert_gen\": precip_pert_gen,\n \"psi\": forecast_gen[\"psi\"],\n }\n\n precip_forecast = nowcast_main_loop(\n precip[-1],\n forecast_gen[\"velocity\"],\n state,\n timesteps,\n forecast_gen[\"extrap_method\"],\n _update,\n extrap_kwargs=forecast_gen[\"extrap_kwargs\"],\n velocity_pert_gen=velocity_perturbators,\n params=params,\n ensemble=True,\n num_ensemble_members=n_ensemble_members,\n callback=callback,\n return_output=return_output,\n num_workers=forecast_gen[\"num_workers\"],\n measure_time=measure_time,\n )\n if measure_time:\n precip_forecast, mainloop_time = precip_forecast\n\n if return_output:\n if not forecast_gen[\"add_perturbations\"]:\n precip_forecast = precip_forecast[0]\n if measure_time:\n return precip_forecast, mainloop_time\n else:\n return precip_forecast\n else:\n return None\n\n\ndef _linda_deterministic_init(\n precip,\n velocity,\n feature_method,\n max_num_features,\n feature_kwargs,\n ari_order,\n kernel_type,\n localization_window_radius,\n extrap_method,\n extrap_kwargs,\n add_perturbations,\n num_workers,\n measure_time,\n):\n \"\"\"Initialize the deterministic LINDA nowcast model.\"\"\"\n forecast_gen = {}\n forecast_gen[\"velocity\"] = velocity\n forecast_gen[\"extrap_method\"] = extrap_method\n forecast_gen[\"ari_order\"] = ari_order\n forecast_gen[\"add_perturbations\"] = add_perturbations\n forecast_gen[\"num_workers\"] = num_workers\n forecast_gen[\"measure_time\"] = measure_time\n\n precip = precip[-(ari_order + 2) :]\n input_length = precip.shape[0]\n\n starttime_init = time.time()\n\n extrapolator = extrapolation.get_method(extrap_method)\n extrap_kwargs = extrap_kwargs.copy()\n extrap_kwargs[\"allow_nonfinite_values\"] = True\n forecast_gen[\"extrapolator\"] = extrapolator\n forecast_gen[\"extrap_kwargs\"] = extrap_kwargs\n\n # detect features from the most recent input field\n if feature_method in {\"blob\", \"shitomasi\"}:\n precip_ = precip[-1].copy()\n precip_[~np.isfinite(precip_)] = 0.0\n feature_detector = feature.get_method(feature_method)\n\n if measure_time:\n starttime = time.time()\n\n feature_kwargs = feature_kwargs.copy()\n feature_kwargs[\"max_num_features\"] = max_num_features\n\n feature_coords = np.fliplr(feature_detector(precip_, **feature_kwargs)[:, :2])\n\n feature_type = \"blobs\" if feature_method == \"blob\" else \"corners\"\n print(\"\")\n print(\"Detecting features... \", end=\"\", flush=True)\n if measure_time:\n print(\n f\"found {feature_coords.shape[0]} {feature_type} in {time.time() - starttime:.2f} seconds.\"\n )\n else:\n print(f\"found {feature_coords.shape[0]} {feature_type}.\")\n\n if len(feature_coords) == 0:\n raise ValueError(\n \"no features found, check input data and feature detector configuration\"\n )\n elif feature_method == \"domain\":\n feature_coords = np.zeros((1, 2), dtype=int)\n else:\n raise NotImplementedError(\n \"feature detector '%s' not implemented\" % feature_method\n )\n forecast_gen[\"feature_coords\"] = feature_coords\n\n # compute interpolation weights\n interp_weights = _compute_window_weights(\n feature_coords,\n precip.shape[1],\n precip.shape[2],\n localization_window_radius,\n )\n interp_weights /= np.sum(interp_weights, axis=0)\n forecast_gen[\"interp_weights\"] = interp_weights\n\n # transform the input fields to the Lagrangian coordinates\n precip_lagr = np.empty(precip.shape)\n\n def worker(i):\n precip_lagr[i, :] = extrapolator(\n precip[i, :],\n velocity,\n input_length - 1 - i,\n \"min\",\n **extrap_kwargs,\n )[-1]\n\n if DASK_IMPORTED and num_workers > 1:\n res = []\n\n print(\"Transforming to Lagrangian coordinates... \", end=\"\", flush=True)\n\n if measure_time:\n starttime = time.time()\n\n for i in range(precip.shape[0] - 1):\n if DASK_IMPORTED and num_workers > 1:\n res.append(dask.delayed(worker)(i))\n else:\n worker(i)\n\n if DASK_IMPORTED and num_workers > 1:\n dask.compute(*res, num_workers=min(num_workers, len(res)), scheduler=\"threads\")\n precip_lagr[-1] = precip[-1]\n\n if measure_time:\n print(f\"{time.time() - starttime:.2f} seconds.\")\n else:\n print(\"done.\")\n\n # compute advection mask and set nan to pixels, where one or more of the\n # advected input fields has a nan value\n mask_adv = np.all(np.isfinite(precip_lagr), axis=0)\n forecast_gen[\"mask_adv\"] = mask_adv\n for i in range(precip_lagr.shape[0]):\n precip_lagr[i, ~mask_adv] = np.nan\n\n # compute differenced input fields in the Lagrangian coordinates\n precip_lagr_diff = np.diff(precip_lagr, axis=0)\n\n # estimate parameters of the deterministic model (i.e. the convolution and\n # the ARI process)\n\n print(\"Estimating the first convolution kernel... \", end=\"\", flush=True)\n\n if measure_time:\n starttime = time.time()\n\n # estimate convolution kernel for the differenced component\n convol_weights = _compute_window_weights(\n feature_coords,\n precip.shape[1],\n precip.shape[2],\n localization_window_radius,\n )\n\n kernels_1 = _estimate_convol_params(\n precip_lagr_diff[-2],\n precip_lagr_diff[-1],\n convol_weights,\n mask_adv,\n kernel_type=kernel_type,\n num_workers=num_workers,\n )\n forecast_gen[\"kernels_1\"] = kernels_1\n\n if measure_time:\n print(f\"{time.time() - starttime:.2f} seconds.\")\n else:\n print(\"done.\")\n\n # compute convolved difference fields\n precip_lagr_diff_c = precip_lagr_diff[:-1].copy()\n for i in range(precip_lagr_diff_c.shape[0]):\n for _ in range(ari_order - i):\n precip_lagr_diff_c[i] = _composite_convolution(\n precip_lagr_diff_c[i],\n kernels_1,\n interp_weights,\n )\n\n print(\"Estimating the ARI(p,1) parameters... \", end=\"\", flush=True)\n\n if measure_time:\n starttime = time.time()\n\n # estimate ARI(p,1) parameters\n weights = _compute_window_weights(\n feature_coords,\n precip.shape[1],\n precip.shape[2],\n localization_window_radius,\n )\n\n if ari_order == 1:\n psi = _estimate_ar1_params(\n precip_lagr_diff_c[-1],\n precip_lagr_diff[-1],\n weights,\n interp_weights,\n num_workers=num_workers,\n )\n else:\n psi = _estimate_ar2_params(\n precip_lagr_diff_c[-2:],\n precip_lagr_diff[-1],\n weights,\n interp_weights,\n num_workers=num_workers,\n )\n forecast_gen[\"psi\"] = psi\n\n if measure_time:\n print(f\"{time.time() - starttime:.2f} seconds.\")\n else:\n print(\"done.\")\n\n # apply the ARI(p,1) model and integrate the differences\n precip_lagr_diff_c = _iterate_ar_model(precip_lagr_diff_c, psi)\n precip_forecast = precip_lagr[-2] + precip_lagr_diff_c[-1]\n precip_forecast[precip_forecast < 0.0] = 0.0\n\n print(\"Estimating the second convolution kernel... \", end=\"\", flush=True)\n\n if measure_time:\n starttime = time.time()\n\n # estimate the second convolution kernels based on the forecast field\n # computed above\n kernels_2 = _estimate_convol_params(\n precip_forecast,\n precip[-1],\n convol_weights,\n mask_adv,\n kernel_type=kernel_type,\n num_workers=num_workers,\n )\n forecast_gen[\"kernels_2\"] = kernels_2\n\n if measure_time:\n print(f\"{time.time() - starttime:.2f} seconds.\")\n else:\n print(\"done.\")\n\n if measure_time:\n return forecast_gen, precip_lagr_diff, time.time() - starttime_init\n else:\n return forecast_gen, precip_lagr_diff\n\n\ndef _linda_perturbation_init(\n precip,\n precip_lagr_diff,\n velocity,\n forecast_gen,\n pert_thrs,\n localization_window_radius,\n errdist_window_radius,\n acf_window_radius,\n vel_pert_method,\n vel_pert_kwargs,\n kmperpixel,\n timestep,\n num_workers,\n use_multiprocessing,\n measure_time,\n):\n \"\"\"Initialize the LINDA perturbation generator.\"\"\"\n if measure_time:\n starttime = time.time()\n\n print(\"Estimating forecast errors... \", end=\"\", flush=True)\n\n forecast_gen = forecast_gen.copy()\n forecast_gen[\"add_perturbations\"] = False\n forecast_gen[\"num_ens_members\"] = 1\n\n precip_forecast_det = _linda_forecast(\n precip[:-1],\n precip_lagr_diff[:-1],\n 1,\n forecast_gen,\n None,\n None,\n 1,\n None,\n False,\n False,\n True,\n None,\n )\n\n # compute multiplicative forecast errors\n err = precip_forecast_det[-1] / precip[-1]\n\n # mask small precipitation intensities\n mask = np.logical_or(\n np.logical_and(\n precip_forecast_det[-1] >= pert_thrs[1], precip[-1] >= pert_thrs[0]\n ),\n np.logical_and(\n precip_forecast_det[-1] >= pert_thrs[0], precip[-1] >= pert_thrs[1]\n ),\n )\n err[~mask] = np.nan\n\n if measure_time:\n print(f\"{time.time() - starttime:.2f} seconds.\")\n else:\n print(\"done.\")\n\n pert_gen = _estimate_perturbation_params(\n err,\n forecast_gen,\n errdist_window_radius,\n acf_window_radius,\n localization_window_radius,\n measure_time,\n num_workers,\n use_multiprocessing,\n )\n\n if vel_pert_method == \"bps\":\n init_vel_noise, generate_vel_noise = noise.get_method(\"bps\")\n\n vp_par = vel_pert_kwargs[\"vp_par\"]\n vp_perp = vel_pert_kwargs[\"vp_perp\"]\n\n kwargs = {\n \"p_par\": vp_par,\n \"p_perp\": vp_perp,\n }\n velocity_pert_gen = {\n \"gen_func\": generate_vel_noise,\n \"init_func\": lambda seed: init_vel_noise(\n velocity, 1.0 / kmperpixel, timestep, seed=seed, **kwargs\n ),\n \"timestep\": timestep,\n }\n else:\n velocity_pert_gen = None\n\n if measure_time:\n return pert_gen, velocity_pert_gen, time.time() - starttime\n else:\n return pert_gen, velocity_pert_gen\n\n\ndef _masked_convolution(field, kernel):\n \"\"\"Compute \"masked\" convolution where non-finite values are ignored.\"\"\"\n mask = np.isfinite(field)\n\n field = field.copy()\n field[~mask] = 0.0\n\n field_c = np.ones(field.shape) * np.nan\n field_c[mask] = convolve(field, kernel, mode=\"same\")[mask]\n field_c[mask] /= convolve(mask.astype(float), kernel, mode=\"same\")[mask]\n\n return field_c\n\n\ndef _update(state, params):\n def worker(j):\n state[\"precip_lagr_diff\"][j] = _iterate_ar_model(\n state[\"precip_lagr_diff\"][j], params[\"psi\"]\n )\n state[\"precip_forecast\"][j] += state[\"precip_lagr_diff\"][j][-1]\n for i in range(state[\"precip_lagr_diff\"][j].shape[0]):\n state[\"precip_lagr_diff\"][j][i] = _composite_convolution(\n state[\"precip_lagr_diff\"][j][i],\n params[\"kernels_1\"],\n params[\"interp_weights\"],\n )\n state[\"precip_forecast\"][j] = _composite_convolution(\n state[\"precip_forecast\"][j], params[\"kernels_2\"], params[\"interp_weights\"]\n )\n\n out = state[\"precip_forecast\"][j].copy()\n out[out < 0.0] = 0.0\n out[~params[\"mask_adv\"]] = np.nan\n\n # apply perturbations\n if params[\"precip_pert_gen\"] is not None:\n seed = state[\"rs_precip_pert\"][j].randint(0, high=1e9)\n perturb = _generate_perturbations(\n params[\"precip_pert_gen\"], params[\"num_workers\"], seed\n )\n out *= perturb\n\n return out\n\n out = []\n if DASK_IMPORTED and params[\"num_workers\"] > 1 and params[\"num_ens_members\"] > 1:\n res = []\n for j in range(params[\"num_ens_members\"]):\n res.append(dask.delayed(worker)(j))\n out = dask.compute(\n *res, num_workers=params[\"num_ensemble_workers\"], scheduler=\"threads\"\n )\n else:\n for j in range(params[\"num_ens_members\"]):\n out.append(worker(j))\n\n return np.stack(out), state\n\n\ndef _weighted_std(f, w):\n \"\"\"\n Compute standard deviation of forecast errors with spatially varying weights.\n Values close to zero are omitted.\n \"\"\"\n mask = np.abs(f - 1.0) > 1e-4\n n = np.count_nonzero(mask)\n if n > 0:\n c = (w[mask].size - 1.0) / n\n return np.sqrt(np.sum(w[mask] * (f[mask] - 1.0) ** 2.0) / (c * np.sum(w[mask])))\n else:\n return np.nan\n\n\ndef _window_tukey(m, n, ci, cj, ri, rj, alpha=0.5):\n \"\"\"Tukey window function centered at the given coordinates.\"\"\"\n j, i = np.meshgrid(np.arange(n), np.arange(m))\n\n di = np.abs(i - ci)\n dj = np.abs(j - cj)\n\n mask1 = np.logical_and(di <= ri, dj <= rj)\n\n w1 = np.zeros(di.shape)\n mask2 = di <= alpha * ri\n mask12 = np.logical_and(mask1, ~mask2)\n w1[mask12] = 0.5 * (\n 1.0 + np.cos(np.pi * (di[mask12] - alpha * ri) / ((1.0 - alpha) * ri))\n )\n w1[np.logical_and(mask1, mask2)] = 1.0\n\n w2 = np.zeros(dj.shape)\n mask2 = dj <= alpha * rj\n mask12 = np.logical_and(mask1, ~mask2)\n w2[mask12] = 0.5 * (\n 1.0 + np.cos(np.pi * (dj[mask12] - alpha * rj) / ((1.0 - alpha) * rj))\n )\n w2[np.logical_and(mask1, mask2)] = 1.0\n\n weights = np.zeros((m, n))\n weights[mask1] = w1[mask1] * w2[mask1]\n\n return weights\n\n\ndef _window_uniform(m, n, ci, cj, ri, rj):\n \"\"\"Uniform window function with all values set to one.\"\"\"\n return np.ones((m, n))\n" }, { "path": "pysteps/nowcasts/sprog.py", "content": "\"\"\"\npysteps.nowcasts.sprog\n======================\n\nImplementation of the S-PROG method described in :cite:`Seed2003`\n\n.. autosummary::\n :toctree: ../generated/\n\n forecast\n\"\"\"\n\nimport time\n\nimport numpy as np\n\nfrom pysteps import cascade, extrapolation, utils\nfrom pysteps.nowcasts import utils as nowcast_utils\nfrom pysteps.nowcasts.utils import compute_percentile_mask, nowcast_main_loop\nfrom pysteps.postprocessing import probmatching\nfrom pysteps.timeseries import autoregression, correlation\nfrom pysteps.utils.check_norain import check_norain\n\ntry:\n import dask\n\n DASK_IMPORTED = True\nexcept ImportError:\n DASK_IMPORTED = False\n\n\ndef forecast(\n precip,\n velocity,\n timesteps,\n precip_thr=None,\n norain_thr=0.0,\n n_cascade_levels=6,\n extrap_method=\"semilagrangian\",\n decomp_method=\"fft\",\n bandpass_filter_method=\"gaussian\",\n ar_order=2,\n conditional=False,\n probmatching_method=\"cdf\",\n num_workers=1,\n fft_method=\"numpy\",\n domain=\"spatial\",\n extrap_kwargs=None,\n filter_kwargs=None,\n measure_time=False,\n):\n \"\"\"\n Generate a nowcast by using the Spectral Prognosis (S-PROG) method.\n\n Parameters\n ----------\n precip: array-like\n Array of shape (ar_order+1,m,n) containing the input precipitation fields\n ordered by timestamp from oldest to newest. The time steps between\n the inputs are assumed to be regular.\n velocity: array-like\n Array of shape (2,m,n) containing the x- and y-components of the\n advection field.\n The velocities are assumed to represent one time step between the\n inputs. All values are required to be finite.\n timesteps: int or list of floats\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed (relative to the input time step). The elements\n of the list are required to be in ascending order.\n precip_thr: float, required\n The threshold value for minimum observable precipitation intensity.\n norain_thr: float\n Specifies the threshold value for the fraction of rainy (see above) pixels\n in the radar rainfall field below which we consider there to be no rain.\n Depends on the amount of clutter typically present.\n Standard set to 0.0\n n_cascade_levels: int, optional\n The number of cascade levels to use. Defaults to 6, see issue #385\n on GitHub.\n extrap_method: str, optional\n Name of the extrapolation method to use. See the documentation of\n pysteps.extrapolation.interface.\n decomp_method: {'fft'}, optional\n Name of the cascade decomposition method to use. See the documentation\n of pysteps.cascade.interface.\n bandpass_filter_method: {'gaussian', 'uniform'}, optional\n Name of the bandpass filter method to use with the cascade decomposition.\n See the documentation of pysteps.cascade.interface.\n ar_order: int, optional\n The order of the autoregressive model to use. Must be >= 1.\n conditional: bool, optional\n If set to True, compute the statistics of the precipitation field\n conditionally by excluding pixels where the values are\n below the threshold precip_thr.\n probmatching_method: {'cdf','mean',None}, optional\n Method for matching the conditional statistics of the forecast field\n (areas with precipitation intensity above the threshold precip_thr) with\n those of the most recently observed one. 'cdf'=map the forecast CDF to the\n observed one, 'mean'=adjust only the mean value,\n None=no matching applied.\n num_workers: int, optional\n The number of workers to use for parallel computation. Applicable if dask\n is enabled or pyFFTW is used for computing the FFT.\n When num_workers>1, it is advisable to disable OpenMP by setting\n the environment variable OMP_NUM_THREADS to 1.\n This avoids slowdown caused by too many simultaneous threads.\n fft_method: str, optional\n A string defining the FFT method to use (see utils.fft.get_method).\n Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed,\n the recommended method is 'pyfftw'.\n domain: {\"spatial\", \"spectral\"}\n If \"spatial\", all computations are done in the spatial domain (the\n classical S-PROG model). If \"spectral\", the AR(2) models are applied\n directly in the spectral domain to reduce memory footprint and improve\n performance :cite:`PCH2019a`.\n extrap_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the extrapolation\n method. See the documentation of pysteps.extrapolation.\n filter_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the filter method.\n See the documentation of pysteps.cascade.bandpass_filters.py.\n measure_time: bool\n If set to True, measure, print and return the computation time.\n\n Returns\n -------\n out: ndarray\n A three-dimensional array of shape (num_timesteps,m,n) containing a time\n series of forecast precipitation fields. The time series starts from\n t0+timestep, where timestep is taken from the input precipitation fields\n precip. If measure_time is True, the return value is a three-element\n tuple containing the nowcast array, the initialization time of the\n nowcast generator and the time used in the main loop (seconds).\n\n See also\n --------\n pysteps.extrapolation.interface, pysteps.cascade.interface\n\n References\n ----------\n :cite:`Seed2003`, :cite:`PCH2019a`\n \"\"\"\n\n _check_inputs(precip, velocity, timesteps, ar_order)\n\n if extrap_kwargs is None:\n extrap_kwargs = dict()\n\n if filter_kwargs is None:\n filter_kwargs = dict()\n\n if np.any(~np.isfinite(velocity)):\n raise ValueError(\"velocity contains non-finite values\")\n\n if precip_thr is None:\n raise ValueError(\"precip_thr required but not specified\")\n\n print(\"Computing S-PROG nowcast\")\n print(\"------------------------\")\n print(\"\")\n\n print(\"Inputs\")\n print(\"------\")\n print(f\"input dimensions: {precip.shape[1]}x{precip.shape[2]}\")\n print(\"\")\n\n print(\"Methods\")\n print(\"-------\")\n print(f\"extrapolation: {extrap_method}\")\n print(f\"bandpass filter: {bandpass_filter_method}\")\n print(f\"decomposition: {decomp_method}\")\n print(\"conditional statistics: {}\".format(\"yes\" if conditional else \"no\"))\n print(f\"probability matching: {probmatching_method}\")\n print(f\"FFT method: {fft_method}\")\n print(f\"domain: {domain}\")\n print(\"\")\n\n print(\"Parameters\")\n print(\"----------\")\n if isinstance(timesteps, int):\n print(f\"number of time steps: {timesteps}\")\n else:\n print(f\"time steps: {timesteps}\")\n print(f\"parallel threads: {num_workers}\")\n print(f\"number of cascade levels: {n_cascade_levels}\")\n print(f\"order of the AR(p) model: {ar_order}\")\n print(f\"precip. intensity threshold: {precip_thr}\")\n\n if measure_time:\n starttime_init = time.time()\n else:\n starttime_init = None\n\n fft = utils.get_method(fft_method, shape=precip.shape[1:], n_threads=num_workers)\n\n m, n = precip.shape[1:]\n\n # initialize the band-pass filter\n filter_method = cascade.get_method(bandpass_filter_method)\n bp_filter = filter_method((m, n), n_cascade_levels, **filter_kwargs)\n\n decomp_method, recomp_method = cascade.get_method(decomp_method)\n\n extrapolator_method = extrapolation.get_method(extrap_method)\n\n precip = precip[-(ar_order + 1) :, :, :].copy()\n precip_min = np.nanmin(precip)\n\n # determine the domain mask from non-finite values\n domain_mask = np.logical_or.reduce(\n [~np.isfinite(precip[i, :]) for i in range(precip.shape[0])]\n )\n\n if check_norain(precip, precip_thr, norain_thr, None):\n return nowcast_utils.zero_precipitation_forecast(\n None, timesteps, precip, None, True, measure_time, starttime_init\n )\n\n # determine the precipitation threshold mask\n if conditional:\n mask_thr = np.logical_and.reduce(\n [precip[i, :, :] >= precip_thr for i in range(precip.shape[0])]\n )\n else:\n mask_thr = None\n\n # initialize the extrapolator\n x_values, y_values = np.meshgrid(\n np.arange(precip.shape[2]), np.arange(precip.shape[1])\n )\n\n xy_coords = np.stack([x_values, y_values])\n\n extrap_kwargs = extrap_kwargs.copy()\n extrap_kwargs[\"xy_coords\"] = xy_coords\n extrap_kwargs[\"allow_nonfinite_values\"] = (\n True if np.any(~np.isfinite(precip)) else False\n )\n\n # advect the previous precipitation fields to the same position with the\n # most recent one (i.e. transform them into the Lagrangian coordinates)\n res = list()\n\n def f(precip, i):\n return extrapolator_method(\n precip[i, :], velocity, ar_order - i, \"min\", **extrap_kwargs\n )[-1]\n\n for i in range(ar_order):\n if not DASK_IMPORTED:\n precip[i, :, :] = f(precip, i)\n else:\n res.append(dask.delayed(f)(precip, i))\n\n if DASK_IMPORTED:\n num_workers_ = len(res) if num_workers > len(res) else num_workers\n precip = np.stack(\n list(dask.compute(*res, num_workers=num_workers_)) + [precip[-1, :, :]]\n )\n\n # replace non-finite values with the minimum value\n precip = precip.copy()\n for i in range(precip.shape[0]):\n precip[i, ~np.isfinite(precip[i, :])] = np.nanmin(precip[i, :])\n\n # compute the cascade decompositions of the input precipitation fields\n precip_decomp = []\n for i in range(ar_order + 1):\n precip_ = decomp_method(\n precip[i, :, :],\n bp_filter,\n mask=mask_thr,\n fft_method=fft,\n output_domain=domain,\n normalize=True,\n compute_stats=True,\n compact_output=True,\n )\n precip_decomp.append(precip_)\n\n # rearrange the cascade levels into a four-dimensional array of shape\n # (n_cascade_levels,ar_order+1,m,n) for the autoregressive model\n precip_cascades = nowcast_utils.stack_cascades(\n precip_decomp, n_cascade_levels, convert_to_full_arrays=True\n )\n\n # compute lag-l temporal autocorrelation coefficients for each cascade level\n gamma = np.empty((n_cascade_levels, ar_order))\n for i in range(n_cascade_levels):\n if domain == \"spatial\":\n gamma[i, :] = correlation.temporal_autocorrelation(\n precip_cascades[i], mask=mask_thr\n )\n else:\n gamma[i, :] = correlation.temporal_autocorrelation(\n precip_cascades[i], domain=\"spectral\", x_shape=precip.shape[1:]\n )\n\n precip_cascades = nowcast_utils.stack_cascades(\n precip_decomp, n_cascade_levels, convert_to_full_arrays=False\n )\n\n precip_decomp = precip_decomp[-1]\n\n nowcast_utils.print_corrcoefs(gamma)\n\n if ar_order == 2:\n # adjust the lag-2 correlation coefficient to ensure that the AR(p)\n # process is stationary\n for i in range(n_cascade_levels):\n gamma[i, 1] = autoregression.adjust_lag2_corrcoef2(gamma[i, 0], gamma[i, 1])\n\n # estimate the parameters of the AR(p) model from the autocorrelation\n # coefficients\n phi = np.empty((n_cascade_levels, ar_order + 1))\n for i in range(n_cascade_levels):\n phi[i, :] = autoregression.estimate_ar_params_yw(gamma[i, :])\n\n nowcast_utils.print_ar_params(phi)\n\n # discard all except the p-1 last cascades because they are not needed for\n # the AR(p) model\n precip_cascades = [precip_cascades[i][-ar_order:] for i in range(n_cascade_levels)]\n\n if probmatching_method == \"mean\":\n mu_0 = np.mean(precip[-1, :, :][precip[-1, :, :] >= precip_thr])\n else:\n mu_0 = None\n\n # compute precipitation mask and wet area ratio\n mask_p = precip[-1, :, :] >= precip_thr\n war = 1.0 * np.sum(mask_p) / (precip.shape[1] * precip.shape[2])\n\n if measure_time:\n init_time = time.time() - starttime_init\n\n precip = precip[-1, :, :]\n\n print(\"Starting nowcast computation.\")\n\n precip_forecast = []\n\n state = {\"precip_cascades\": precip_cascades, \"precip_decomp\": precip_decomp}\n params = {\n \"domain\": domain,\n \"domain_mask\": domain_mask,\n \"fft\": fft,\n \"mu_0\": mu_0,\n \"n_cascade_levels\": n_cascade_levels,\n \"phi\": phi,\n \"precip_0\": precip,\n \"precip_min\": precip_min,\n \"probmatching_method\": probmatching_method,\n \"recomp_method\": recomp_method,\n \"war\": war,\n }\n\n precip_forecast = nowcast_main_loop(\n precip,\n velocity,\n state,\n timesteps,\n extrap_method,\n _update,\n extrap_kwargs=extrap_kwargs,\n params=params,\n measure_time=measure_time,\n )\n if measure_time:\n precip_forecast, mainloop_time = precip_forecast\n\n precip_forecast = np.stack(precip_forecast)\n\n if measure_time:\n return precip_forecast, init_time, mainloop_time\n else:\n return precip_forecast\n\n\ndef _check_inputs(precip, velocity, timesteps, ar_order):\n if precip.ndim != 3:\n raise ValueError(\"precip must be a three-dimensional array\")\n if precip.shape[0] < ar_order + 1:\n raise ValueError(\"precip.shape[0] < ar_order+1\")\n if velocity.ndim != 3:\n raise ValueError(\"velocity must be a three-dimensional array\")\n if precip.shape[1:3] != velocity.shape[1:3]:\n raise ValueError(\n \"dimension mismatch between precip and velocity: shape(precip)=%s, shape(velocity)=%s\"\n % (str(precip.shape), str(velocity.shape))\n )\n if isinstance(timesteps, list) and not sorted(timesteps) == timesteps:\n raise ValueError(\"timesteps is not in ascending order\")\n\n\ndef _update(state, params):\n for i in range(params[\"n_cascade_levels\"]):\n state[\"precip_cascades\"][i] = autoregression.iterate_ar_model(\n state[\"precip_cascades\"][i], params[\"phi\"][i, :]\n )\n\n state[\"precip_decomp\"][\"cascade_levels\"] = [\n state[\"precip_cascades\"][i][-1, :] for i in range(params[\"n_cascade_levels\"])\n ]\n if params[\"domain\"] == \"spatial\":\n state[\"precip_decomp\"][\"cascade_levels\"] = np.stack(\n state[\"precip_decomp\"][\"cascade_levels\"]\n )\n\n precip_forecast_recomp = params[\"recomp_method\"](state[\"precip_decomp\"])\n\n if params[\"domain\"] == \"spectral\":\n precip_forecast_recomp = params[\"fft\"].irfft2(precip_forecast_recomp)\n\n mask = compute_percentile_mask(precip_forecast_recomp, params[\"war\"])\n precip_forecast_recomp[~mask] = params[\"precip_min\"]\n\n if params[\"probmatching_method\"] == \"cdf\":\n # adjust the CDF of the forecast to match the most recently\n # observed precipitation field\n precip_forecast_recomp = probmatching.nonparam_match_empirical_cdf(\n precip_forecast_recomp, params[\"precip_0\"]\n )\n elif params[\"probmatching_method\"] == \"mean\":\n mu_fct = np.mean(precip_forecast_recomp[mask])\n precip_forecast_recomp[mask] = (\n precip_forecast_recomp[mask] - mu_fct + params[\"mu_0\"]\n )\n\n precip_forecast_recomp[params[\"domain_mask\"]] = np.nan\n\n return precip_forecast_recomp, state\n" }, { "path": "pysteps/nowcasts/sseps.py", "content": "\"\"\"\npysteps.nowcasts.sseps\n======================\n\nImplementation of the Short-space ensemble prediction system (SSEPS) method.\nEssentially, SSEPS is a localized version of STEPS.\n\nFor localization we intend the use of a subset of the observations in order to\nestimate model parameters that are distributed in space. The short-space\napproach used in :cite:`NBSG2017` is generalized to the whole nowcasting system.\nThis essentially boils down to a moving window localization of the nowcasting\nprocedure, whereby all parameters are estimated over a subdomain of prescribed\nsize.\n\n.. autosummary::\n :toctree: ../generated/\n\n forecast\n\"\"\"\n\nimport time\n\nimport numpy as np\nfrom scipy.ndimage import generate_binary_structure, iterate_structure\n\nfrom pysteps import cascade, extrapolation, noise\nfrom pysteps.nowcasts import utils as nowcast_utils\nfrom pysteps.postprocessing import probmatching\nfrom pysteps.timeseries import autoregression, correlation\nfrom pysteps.utils.check_norain import check_norain\n\ntry:\n import dask\n\n dask_imported = True\nexcept ImportError:\n dask_imported = False\n\n\ndef forecast(\n precip,\n metadata,\n velocity,\n timesteps,\n n_ens_members=24,\n n_cascade_levels=6,\n win_size=256,\n overlap=0.1,\n war_thr=0.1,\n extrap_method=\"semilagrangian\",\n decomp_method=\"fft\",\n bandpass_filter_method=\"gaussian\",\n noise_method=\"ssft\",\n ar_order=2,\n vel_pert_method=None,\n probmatching_method=\"cdf\",\n mask_method=\"incremental\",\n callback=None,\n fft_method=\"numpy\",\n return_output=True,\n seed=None,\n num_workers=1,\n extrap_kwargs=None,\n filter_kwargs=None,\n noise_kwargs=None,\n vel_pert_kwargs=None,\n mask_kwargs=None,\n measure_time=False,\n):\n \"\"\"\n Generate a nowcast ensemble by using the Short-space ensemble prediction\n system (SSEPS) method.\n This is an experimental version of STEPS which allows for localization\n by means of a window function.\n\n Parameters\n ----------\n precip: array-like\n Array of shape (ar_order+1,m,n) containing the input precipitation fields\n ordered by timestamp from oldest to newest. The time steps between the inputs\n are assumed to be regular, and the inputs are required to have finite values.\n metadata: dict\n Metadata dictionary containing the accutime, xpixelsize, threshold and\n zerovalue attributes as described in the documentation of\n :py:mod:`pysteps.io.importers`. xpixelsize is assumed to be in meters.\n velocity: array-like\n Array of shape (2,m,n) containing the x- and y-components of the advection\n field. The velocities are assumed to represent one time step between the\n inputs. All values are required to be finite.\n win_size: int or two-element sequence of ints\n Size-length of the localization window.\n overlap: float [0,1[\n A float between 0 and 1 prescribing the level of overlap between\n successive windows. If set to 0, no overlap is used.\n war_thr: float\n Threshold for the minimum fraction of rain in a given window.\n timesteps: int or list of floats\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed (relative to the input time step). The elements\n of the list are required to be in ascending order.\n n_ens_members: int\n The number of ensemble members to generate.\n n_cascade_levels: int\n The number of cascade levels to use. Defaults to 6, see issue #385\n on GitHub.\n extrap_method: {'semilagrangian'}\n Name of the extrapolation method to use. See the documentation of\n pysteps.extrapolation.interface.\n decomp_method: {'fft'}\n Name of the cascade decomposition method to use. See the documentation\n of pysteps.cascade.interface.\n bandpass_filter_method: {'gaussian', 'uniform'}\n Name of the bandpass filter method to use with the cascade\n decomposition.\n noise_method: {'parametric','nonparametric','ssft','nested',None}\n Name of the noise generator to use for perturbating the precipitation\n field. See the documentation of pysteps.noise.interface. If set to None,\n no noise is generated.\n ar_order: int\n The order of the autoregressive model to use. Must be >= 1.\n vel_pert_method: {'bps',None}\n Name of the noise generator to use for perturbing the advection field.\n See the documentation of pysteps.noise.interface. If set to None,\n the advection field is not perturbed.\n mask_method: {'incremental', None}\n The method to use for masking no precipitation areas in the forecast\n field. The masked pixels are set to the minimum value of the\n observations. 'incremental' = iteratively buffer the mask with a\n certain rate (currently it is 1 km/min), None=no masking.\n probmatching_method: {'cdf', None}\n Method for matching the statistics of the forecast field with those of\n the most recently observed one. 'cdf'=map the forecast CDF to the\n observed one, None=no matching applied. Using 'mean' requires\n that mask_method is not None.\n callback: function\n Optional function that is called after computation of each time step of\n the nowcast. The function takes one argument: a three-dimensional array\n of shape (n_ens_members,h,w), where h and w are the height and width\n of the input field precip, respectively. This can be used, for instance,\n writing the outputs into files.\n return_output: bool\n Set to False to disable returning the outputs as numpy arrays. This can\n save memory if the intermediate results are written to output files\n using the callback function.\n seed: int\n Optional seed number for the random generators.\n num_workers: int\n The number of workers to use for parallel computation. Applicable if\n dask is enabled or pyFFTW is used for computing the FFT.\n When num_workers>1, it is advisable to disable OpenMP by setting the\n environment variable OMP_NUM_THREADS to 1.\n This avoids slowdown caused by too many simultaneous threads.\n fft_method: str\n A string defining the FFT method to use (see utils.fft.get_method).\n Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed,\n the recommended method is 'pyfftw'.\n extrap_kwargs: dict\n Optional dictionary containing keyword arguments for the extrapolation\n method. See the documentation of pysteps.extrapolation.\n filter_kwargs: dict\n Optional dictionary containing keyword arguments for the filter method.\n See the documentation of pysteps.cascade.bandpass_filters.py.\n noise_kwargs: dict\n Optional dictionary containing keyword arguments for the initializer of\n the noise generator. See the documentation of\n pysteps.noise.fftgenerators.\n vel_pert_kwargs: dict\n Optional dictionary containing keyword arguments \"p_pert_par\" and\n \"p_pert_perp\" for the initializer of the velocity perturbator.\n See the documentation of pysteps.noise.motion.\n mask_kwargs: dict\n Optional dictionary containing mask keyword arguments 'mask_f' and\n 'mask_rim', the factor defining the the mask increment and the rim size,\n respectively.\n The mask increment is defined as mask_f*timestep/kmperpixel.\n measure_time: bool\n If set to True, measure, print and return the computation time.\n\n Returns\n -------\n out: ndarray\n If return_output is True, a four-dimensional array of shape\n (n_ens_members,num_timesteps,m,n) containing a time series of forecast\n precipitation fields for each ensemble member. Otherwise, a None value\n is returned. The time series starts from t0+timestep, where timestep is\n taken from the input precipitation fields.\n\n See also\n --------\n pysteps.extrapolation.interface, pysteps.cascade.interface,\n pysteps.noise.interface, pysteps.noise.utils.compute_noise_stddev_adjs\n\n Notes\n -----\n Please be aware that this represents a (very) experimental implementation.\n\n References\n ----------\n :cite:`Seed2003`, :cite:`BPS2006`, :cite:`SPN2013`, :cite:`NBSG2017`\n \"\"\"\n\n _check_inputs(precip, velocity, timesteps, ar_order)\n\n if extrap_kwargs is None:\n extrap_kwargs = dict()\n else:\n extrap_kwargs = extrap_kwargs.copy()\n\n if filter_kwargs is None:\n filter_kwargs = dict()\n\n if noise_kwargs is None:\n noise_kwargs = {\"win_fun\": \"tukey\"}\n\n if vel_pert_kwargs is None:\n vel_pert_kwargs = dict()\n\n if mask_kwargs is None:\n mask_kwargs = dict()\n\n if np.any(~np.isfinite(precip)):\n raise ValueError(\"precip contains non-finite values\")\n\n if np.any(~np.isfinite(velocity)):\n raise ValueError(\"velocity contains non-finite values\")\n\n if mask_method not in [\"incremental\", None]:\n raise ValueError(\n \"unknown mask method %s: must be 'incremental' or None\" % mask_method\n )\n\n if np.isscalar(win_size):\n win_size = (int(win_size), int(win_size))\n else:\n win_size = tuple([int(win_size[i]) for i in range(2)])\n\n timestep = metadata[\"accutime\"]\n kmperpixel = metadata[\"xpixelsize\"] / 1000\n\n print(\"Computing SSEPS nowcast\")\n print(\"-----------------------\")\n print(\"\")\n\n print(\"Inputs\")\n print(\"------\")\n print(\"input dimensions: %dx%d\" % (precip.shape[1], precip.shape[2]))\n print(f\"km/pixel: {kmperpixel}\")\n print(f\"time step: {timestep} minutes\")\n print(\"\")\n\n print(\"Methods\")\n print(\"-------\")\n print(f\"extrapolation: {extrap_method}\")\n print(f\"bandpass filter: {bandpass_filter_method}\")\n print(f\"decomposition: {decomp_method}\")\n print(f\"noise generator: {noise_method}\")\n print(f\"velocity perturbator: {vel_pert_method}\")\n print(f\"precip. mask method: {mask_method}\")\n print(f\"probability matching: {probmatching_method}\")\n print(f\"FFT method: {fft_method}\")\n print(\"\")\n\n print(\"Parameters\")\n print(\"----------\")\n print(f\"localization window: {win_size[0]}x{win_size[1]}\")\n print(f\"overlap: {overlap:.1f}\")\n print(f\"war thr: {war_thr:.2f}\")\n if isinstance(timesteps, int):\n print(f\"number of time steps: {timesteps}\")\n else:\n print(f\"time steps: {timesteps}\")\n print(f\"ensemble size: {n_ens_members}\")\n print(f\"number of cascade levels: {n_cascade_levels}\")\n print(f\"order of the AR(p) model: {ar_order}\")\n print(\"dask imported: {}\".format((\"yes\" if dask_imported else \"no\")))\n print(f\"num workers: {num_workers}\")\n\n if vel_pert_method == \"bps\":\n vp_par = vel_pert_kwargs.get(\n \"p_pert_par\", noise.motion.get_default_params_bps_par()\n )\n vp_perp = vel_pert_kwargs.get(\n \"p_pert_perp\", noise.motion.get_default_params_bps_perp()\n )\n print(\n f\"velocity perturbations, parallel: {vp_par[0]},{vp_par[1]},{vp_par[2]}\"\n )\n print(\n f\"velocity perturbations, perpendicular: {vp_perp[0]},{vp_perp[1]},{vp_perp[2]}\"\n )\n\n precip_thr = metadata[\"threshold\"]\n precip_min = metadata[\"zerovalue\"]\n\n num_ensemble_workers = n_ens_members if num_workers > n_ens_members else num_workers\n\n if measure_time:\n starttime_init = time.time()\n else:\n starttime_init = None\n\n # get methods\n extrapolator_method = extrapolation.get_method(extrap_method)\n\n x_values, y_values = np.meshgrid(\n np.arange(precip.shape[2]), np.arange(precip.shape[1])\n )\n\n xy_coords = np.stack([x_values, y_values])\n\n decomp_method, __ = cascade.get_method(decomp_method)\n filter_method = cascade.get_method(bandpass_filter_method)\n if noise_method is not None:\n init_noise, generate_noise = noise.get_method(noise_method)\n\n if check_norain(\n precip,\n precip_thr,\n war_thr,\n noise_kwargs[\"win_fun\"],\n ):\n return nowcast_utils.zero_precipitation_forecast(\n n_ens_members,\n timesteps,\n precip,\n callback,\n return_output,\n measure_time,\n starttime_init,\n )\n\n # advect the previous precipitation fields to the same position with the\n # most recent one (i.e. transform them into the Lagrangian coordinates)\n precip = precip[-(ar_order + 1) :, :, :].copy()\n extrap_kwargs = extrap_kwargs.copy()\n extrap_kwargs[\"xy_coords\"] = xy_coords\n extrap_kwargs[\"allow_nonfinite_values\"] = (\n True if np.any(~np.isfinite(precip)) else False\n )\n\n res = []\n extrapolate = lambda precip, i: extrapolator_method(\n precip[i, :, :], velocity, ar_order - i, \"min\", **extrap_kwargs\n )[-1]\n for i in range(ar_order):\n if not dask_imported:\n precip[i, :, :] = extrapolate(precip, i)\n else:\n res.append(dask.delayed(extrapolate)(precip, i))\n\n if dask_imported:\n num_workers_ = len(res) if num_workers > len(res) else num_workers\n precip = np.stack(\n list(dask.compute(*res, num_workers=num_workers_)) + [precip[-1, :, :]]\n )\n\n if mask_method == \"incremental\":\n # get mask parameters\n mask_rim = mask_kwargs.get(\"mask_rim\", 10)\n mask_f = mask_kwargs.get(\"mask_f\", 1.0)\n # initialize the structuring element\n struct = generate_binary_structure(2, 1)\n # iterate it to expand it nxn\n n = mask_f * timestep / kmperpixel\n struct = iterate_structure(struct, int((n - 1) / 2.0))\n\n noise_kwargs.update(\n {\n \"win_size\": win_size,\n \"overlap\": overlap,\n \"war_thr\": war_thr,\n \"rm_rdisc\": True,\n \"donorm\": True,\n }\n )\n\n print(\"Estimating nowcast parameters...\", end=\"\")\n\n def estimator(precip, parsglob=None, idxm=None, idxn=None):\n pars = {}\n\n # initialize the perturbation generator for the precipitation field\n if noise_method is not None and parsglob is None:\n pert_gen = init_noise(precip, fft_method=fft_method, **noise_kwargs)\n else:\n pert_gen = None\n pars[\"pert_gen\"] = pert_gen\n\n # initialize the band-pass filter\n if parsglob is None:\n bp_filter = filter_method(\n precip.shape[1:], n_cascade_levels, **filter_kwargs\n )\n pars[\"filter\"] = bp_filter\n else:\n pars[\"filter\"] = None\n\n # compute the cascade decompositions of the input precipitation fields\n if parsglob is None:\n precip_decomp = []\n for i in range(ar_order + 1):\n precip_decomp_ = decomp_method(\n precip[i, :, :],\n bp_filter,\n fft_method=fft_method,\n normalize=True,\n compute_stats=True,\n )\n precip_decomp.append(precip_decomp_)\n precip_decomp_ = None\n\n # normalize the cascades and rearrange them into a four-dimensional array\n # of shape (n_cascade_levels,ar_order+1,m,n) for the autoregressive model\n if parsglob is None:\n precip_cascades = nowcast_utils.stack_cascades(\n precip_decomp, n_cascade_levels\n )\n mu = precip_decomp[-1][\"means\"]\n sigma = precip_decomp[-1][\"stds\"]\n precip_decomp = None\n\n else:\n precip_cascades = parsglob[\"precip_cascades\"][0][\n :, :, idxm.item(0) : idxm.item(1), idxn.item(0) : idxn.item(1)\n ].copy()\n mu = np.mean(precip_cascades, axis=(2, 3))\n sigma = np.std(precip_cascades, axis=(2, 3))\n\n precip_cascades = (precip_cascades - mu[:, :, None, None]) / sigma[\n :, :, None, None\n ]\n\n mu = mu[:, -1]\n sigma = sigma[:, -1]\n\n pars[\"mu\"] = mu\n pars[\"sigma\"] = sigma\n\n # compute lag-l temporal autocorrelation coefficients for each cascade level\n gamma = np.empty((n_cascade_levels, ar_order))\n for i in range(n_cascade_levels):\n precip_cascades_ = np.stack(\n [precip_cascades[i, j, :, :] for j in range(ar_order + 1)]\n )\n gamma[i, :] = correlation.temporal_autocorrelation(precip_cascades_)\n precip_cascades_ = None\n\n if ar_order == 2:\n # adjust the local lag-2 correlation coefficient to ensure that the AR(p)\n # process is stationary\n for i in range(n_cascade_levels):\n gamma[i, 1] = autoregression.adjust_lag2_corrcoef2(\n gamma[i, 0], gamma[i, 1]\n )\n\n # estimate the parameters of the AR(p) model from the autocorrelation\n # coefficients\n phi = np.empty((n_cascade_levels, ar_order + 1))\n for i in range(n_cascade_levels):\n phi[i, :] = autoregression.estimate_ar_params_yw(gamma[i, :])\n pars[\"phi\"] = phi\n\n # stack the cascades into a five-dimensional array containing all ensemble\n # members\n precip_cascades = [precip_cascades.copy() for _ in range(n_ens_members)]\n pars[\"precip_cascades\"] = precip_cascades\n\n if mask_method is not None and parsglob is None:\n mask_prec = precip[-1, :, :] >= precip_thr\n if mask_method == \"incremental\":\n # initialize precip mask for each member\n mask_prec = nowcast_utils.compute_dilated_mask(\n mask_prec, struct, mask_rim\n )\n mask_prec = [mask_prec.copy() for _ in range(n_ens_members)]\n else:\n mask_prec = None\n pars[\"mask_prec\"] = mask_prec\n\n return pars\n\n # prepare windows\n M, N = precip.shape[1:]\n n_windows_M = np.ceil(1.0 * M / win_size[0]).astype(int)\n n_windows_N = np.ceil(1.0 * N / win_size[1]).astype(int)\n idxm = np.zeros(2, dtype=int)\n idxn = np.zeros(2, dtype=int)\n\n if measure_time:\n starttime = time.time()\n\n # compute global parameters to be used as defaults\n parsglob = estimator(precip)\n\n # loop windows\n if n_windows_M > 1 or n_windows_N > 1:\n war = np.empty((n_windows_M, n_windows_N))\n phi = np.empty((n_windows_M, n_windows_N, n_cascade_levels, ar_order + 1))\n mu = np.empty((n_windows_M, n_windows_N, n_cascade_levels))\n sigma = np.empty((n_windows_M, n_windows_N, n_cascade_levels))\n ff = []\n rc = []\n pp = []\n mm = []\n for m in range(n_windows_M):\n ff_ = []\n pp_ = []\n rc_ = []\n mm_ = []\n for n in range(n_windows_N):\n # compute indices of local window\n idxm[0] = int(np.max((m * win_size[0] - overlap * win_size[0], 0)))\n idxm[1] = int(\n np.min((idxm[0] + win_size[0] + overlap * win_size[0], M))\n )\n idxn[0] = int(np.max((n * win_size[1] - overlap * win_size[1], 0)))\n idxn[1] = int(\n np.min((idxn[0] + win_size[1] + overlap * win_size[1], N))\n )\n\n mask = np.zeros((M, N), dtype=bool)\n mask[idxm.item(0) : idxm.item(1), idxn.item(0) : idxn.item(1)] = True\n\n precip_ = precip[\n :, idxm.item(0) : idxm.item(1), idxn.item(0) : idxn.item(1)\n ]\n\n war[m, n] = (\n np.sum(precip_[-1, :, :] >= precip_thr) / precip_[-1, :, :].size\n )\n if war[m, n] > war_thr:\n # estimate local parameters\n pars = estimator(precip, parsglob, idxm, idxn)\n ff_.append(pars[\"filter\"])\n pp_.append(pars[\"pert_gen\"])\n rc_.append(pars[\"precip_cascades\"])\n mm_.append(pars[\"mask_prec\"])\n mu[m, n, :] = pars[\"mu\"]\n sigma[m, n, :] = pars[\"sigma\"]\n phi[m, n, :, :] = pars[\"phi\"]\n\n else:\n # dry window\n ff_.append(None)\n pp_.append(None)\n rc_.append(None)\n mm_.append(None)\n\n ff.append(ff_)\n pp.append(pp_)\n rc.append(rc_)\n mm.append(mm_)\n\n # remove unnecessary variables\n ff_ = None\n pp_ = None\n rc_ = None\n mm_ = None\n pars = None\n\n if measure_time:\n print(f\"{time.time() - starttime:.2f} seconds.\")\n else:\n print(\" done.\")\n\n # initialize the random generators\n if noise_method is not None:\n randgen_prec = []\n randgen_motion = []\n np.random.seed(seed)\n for _ in range(n_ens_members):\n rs = np.random.RandomState(seed)\n randgen_prec.append(rs)\n seed = rs.randint(0, high=1e9)\n rs = np.random.RandomState(seed)\n randgen_motion.append(rs)\n seed = rs.randint(0, high=1e9)\n\n if vel_pert_method is not None:\n init_vel_noise, generate_vel_noise = noise.get_method(vel_pert_method)\n\n # initialize the perturbation generators for the motion field\n velocity_perturbators = []\n for j in range(n_ens_members):\n kwargs = {\n \"randstate\": randgen_motion[j],\n \"p_par\": vp_par,\n \"p_perp\": vp_perp,\n }\n vp = init_vel_noise(velocity, 1.0 / kmperpixel, timestep, **kwargs)\n velocity_perturbators.append(vp)\n\n D = [None for _ in range(n_ens_members)]\n precip_forecast = [[] for _ in range(n_ens_members)]\n\n if measure_time:\n init_time = time.time() - starttime_init\n\n precip = precip[-1, :, :]\n\n print(\"Starting nowcast computation.\")\n\n if measure_time:\n starttime_mainloop = time.time()\n\n if isinstance(timesteps, int):\n timesteps = range(timesteps + 1)\n timestep_type = \"int\"\n else:\n original_timesteps = [0] + list(timesteps)\n timesteps = nowcast_utils.binned_timesteps(original_timesteps)\n timestep_type = \"list\"\n\n extrap_kwargs[\"return_displacement\"] = True\n precip_forecast_prev = [precip for _ in range(n_ens_members)]\n t_prev = [0.0 for _ in range(n_ens_members)]\n t_total = [0.0 for _ in range(n_ens_members)]\n\n # iterate each time step\n for t, subtimestep_idx in enumerate(timesteps):\n if timestep_type == \"list\":\n subtimesteps = [original_timesteps[t_] for t_ in subtimestep_idx]\n else:\n subtimesteps = [t]\n\n if (timestep_type == \"list\" and subtimesteps) or (\n timestep_type == \"int\" and t > 0\n ):\n is_nowcast_time_step = True\n else:\n is_nowcast_time_step = False\n\n if is_nowcast_time_step:\n print(\n f\"Computing nowcast for time step {t}... \",\n end=\"\",\n flush=True,\n )\n\n if measure_time:\n starttime = time.time()\n\n # iterate each ensemble member\n def worker(j):\n # first the global step\n\n if noise_method is not None:\n # generate noise field\n EPS = generate_noise(\n parsglob[\"pert_gen\"],\n randstate=randgen_prec[j],\n fft_method=fft_method,\n )\n # decompose the noise field into a cascade\n EPS_d = decomp_method(\n EPS,\n parsglob[\"filter\"],\n fft_method=fft_method,\n normalize=True,\n compute_stats=True,\n )\n else:\n EPS_d = None\n\n # iterate the AR(p) model for each cascade level\n precip_cascades = parsglob[\"precip_cascades\"][j].copy()\n if precip_cascades.shape[1] >= ar_order:\n precip_cascades = precip_cascades[:, -ar_order:, :, :].copy()\n for i in range(n_cascade_levels):\n # normalize the noise cascade\n if EPS_d is not None:\n EPS_ = (\n EPS_d[\"cascade_levels\"][i, :, :] - EPS_d[\"means\"][i]\n ) / EPS_d[\"stds\"][i]\n else:\n EPS_ = None\n # apply AR(p) process to cascade level\n precip_cascades[i, :, :, :] = autoregression.iterate_ar_model(\n precip_cascades[i, :, :, :], parsglob[\"phi\"][i, :], eps=EPS_\n )\n EPS_ = None\n parsglob[\"precip_cascades\"][j] = precip_cascades.copy()\n EPS = None\n\n # compute the recomposed precipitation field(s) from the cascades\n # obtained from the AR(p) model(s)\n precip_forecast_new = _recompose_cascade(\n precip_cascades, parsglob[\"mu\"], parsglob[\"sigma\"]\n )\n precip_cascades = None\n\n # then the local steps\n if n_windows_M > 1 or n_windows_N > 1:\n idxm = np.zeros(2, dtype=int)\n idxn = np.zeros(2, dtype=int)\n precip_l = np.zeros((M, N), dtype=float)\n M_s = np.zeros((M, N), dtype=float)\n for m in range(n_windows_M):\n for n in range(n_windows_N):\n # compute indices of local window\n idxm[0] = int(\n np.max((m * win_size[0] - overlap * win_size[0], 0))\n )\n idxm[1] = int(\n np.min((idxm[0] + win_size[0] + overlap * win_size[0], M))\n )\n idxn[0] = int(\n np.max((n * win_size[1] - overlap * win_size[1], 0))\n )\n idxn[1] = int(\n np.min((idxn[0] + win_size[1] + overlap * win_size[1], N))\n )\n\n # build localization mask\n mask = _get_mask((M, N), idxm, idxn)\n mask_l = mask[\n idxm.item(0) : idxm.item(1), idxn.item(0) : idxn.item(1)\n ]\n M_s += mask\n\n # skip if dry\n if war[m, n] > war_thr:\n precip_cascades = rc[m][n][j].copy()\n if precip_cascades.shape[1] >= ar_order:\n precip_cascades = precip_cascades[:, -ar_order:, :, :]\n if noise_method is not None:\n # extract noise field\n EPS_d_l = EPS_d[\"cascade_levels\"][\n :,\n idxm.item(0) : idxm.item(1),\n idxn.item(0) : idxn.item(1),\n ].copy()\n mu_ = np.mean(EPS_d_l, axis=(1, 2))\n sigma_ = np.std(EPS_d_l, axis=(1, 2))\n else:\n EPS_d_l = None\n\n # iterate the AR(p) model for each cascade level\n for i in range(n_cascade_levels):\n # normalize the noise cascade\n if EPS_d_l is not None:\n EPS_ = (\n EPS_d_l[i, :, :] - mu_[i, None, None]\n ) / sigma_[i, None, None]\n else:\n EPS_ = None\n # apply AR(p) process to cascade level\n precip_cascades[i, :, :, :] = (\n autoregression.iterate_ar_model(\n precip_cascades[i, :, :, :],\n phi[m, n, i, :],\n eps=EPS_,\n )\n )\n EPS_ = None\n rc[m][n][j] = precip_cascades.copy()\n EPS_d_l = mu_ = sigma_ = None\n\n # compute the recomposed precipitation field(s) from the cascades\n # obtained from the AR(p) model(s)\n mu_ = mu[m, n, :]\n sigma_ = sigma[m, n, :]\n precip_cascades = [\n ((precip_cascades[i, -1, :, :] * sigma_[i]) + mu_[i])\n * parsglob[\"sigma\"][i]\n + parsglob[\"mu\"][i]\n for i in range(len(mu_))\n ]\n precip_l_ = np.sum(np.stack(precip_cascades), axis=0)\n precip_cascades = mu_ = sigma_ = None\n # precip_l_ = _recompose_cascade(precip_cascades[:, :, :], mu[m, n, :], sigma[m, n, :])\n else:\n precip_l_ = precip_forecast_new[\n idxm.item(0) : idxm.item(1), idxn.item(0) : idxn.item(1)\n ].copy()\n\n if probmatching_method == \"cdf\":\n # adjust the CDF of the forecast to match the most recently\n # observed precipitation field\n precip_ = precip[\n idxm.item(0) : idxm.item(1), idxn.item(0) : idxn.item(1)\n ].copy()\n precip_l_ = probmatching.nonparam_match_empirical_cdf(\n precip_l_, precip_\n )\n precip_ = None\n\n precip_l[\n idxm.item(0) : idxm.item(1), idxn.item(0) : idxn.item(1)\n ] += (precip_l_ * mask_l)\n precip_l_ = None\n\n ind = M_s > 0\n precip_l[ind] *= 1 / M_s[ind]\n precip_l[~ind] = precip_min\n\n precip_forecast_new = precip_l.copy()\n precip_l = None\n\n if probmatching_method == \"cdf\":\n # adjust the CDF of the forecast to match the most recently\n # observed precipitation field\n precip_forecast_new[precip_forecast_new < precip_thr] = precip_min\n precip_forecast_new = probmatching.nonparam_match_empirical_cdf(\n precip_forecast_new, precip\n )\n\n if mask_method is not None:\n # apply the precipitation mask to prevent generation of new\n # precipitation into areas where it was not originally\n # observed\n if mask_method == \"incremental\":\n mask_prec = parsglob[\"mask_prec\"][j].copy()\n precip_forecast_new = (\n precip_forecast_new.min()\n + (precip_forecast_new - precip_forecast_new.min()) * mask_prec\n )\n mask_prec = None\n\n if mask_method == \"incremental\":\n parsglob[\"mask_prec\"][j] = nowcast_utils.compute_dilated_mask(\n precip_forecast_new >= precip_thr, struct, mask_rim\n )\n\n precip_forecast_out = []\n extrap_kwargs_ = extrap_kwargs.copy()\n extrap_kwargs_[\"xy_coords\"] = xy_coords\n extrap_kwargs_[\"return_displacement\"] = True\n\n V_pert = velocity\n\n # advect the recomposed precipitation field to obtain the forecast for\n # the current time step (or subtimesteps if non-integer time steps are\n # given)\n for t_sub in subtimesteps:\n if t_sub > 0:\n t_diff_prev_int = t_sub - int(t_sub)\n if t_diff_prev_int > 0.0:\n precip_forecast_ip = (\n 1.0 - t_diff_prev_int\n ) * precip_forecast_prev[\n j\n ] + t_diff_prev_int * precip_forecast_new\n else:\n precip_forecast_ip = precip_forecast_prev[j]\n\n t_diff_prev = t_sub - t_prev[j]\n t_total[j] += t_diff_prev\n\n # compute the perturbed motion field\n if vel_pert_method is not None:\n V_pert = velocity + generate_vel_noise(\n velocity_perturbators[j], t_total[j] * timestep\n )\n\n extrap_kwargs_[\"displacement_prev\"] = D[j]\n precip_forecast_ep, D[j] = extrapolator_method(\n precip_forecast_ip,\n V_pert,\n [t_diff_prev],\n **extrap_kwargs_,\n )\n precip_forecast_ep[0][\n precip_forecast_ep[0] < precip_thr\n ] = precip_min\n precip_forecast_out.append(precip_forecast_ep[0])\n t_prev[j] = t_sub\n\n # advect the forecast field by one time step if no subtimesteps in the\n # current interval were found\n if not subtimesteps:\n t_diff_prev = t + 1 - t_prev[j]\n t_total[j] += t_diff_prev\n\n # compute the perturbed motion field\n if vel_pert_method is not None:\n V_pert = velocity + generate_vel_noise(\n velocity_perturbators[j], t_total[j] * timestep\n )\n\n extrap_kwargs_[\"displacement_prev\"] = D[j]\n _, D[j] = extrapolator_method(\n None,\n V_pert,\n [t_diff_prev],\n **extrap_kwargs_,\n )\n t_prev[j] = t + 1\n\n precip_forecast_prev[j] = precip_forecast_new\n\n return precip_forecast_out\n\n res = []\n for j in range(n_ens_members):\n if not dask_imported or n_ens_members == 1:\n res.append(worker(j))\n else:\n res.append(dask.delayed(worker)(j))\n\n precip_forecast_ = (\n dask.compute(*res, num_workers=num_ensemble_workers)\n if dask_imported and n_ens_members > 1\n else res\n )\n res = None\n\n if is_nowcast_time_step:\n if measure_time:\n print(f\"{time.time() - starttime:.2f} seconds.\")\n else:\n print(\"done.\")\n\n if callback is not None:\n precip_forecast_stacked = np.stack(precip_forecast_)\n if precip_forecast_stacked.shape[1] > 0:\n callback(precip_forecast_stacked.squeeze())\n precip_forecast_ = None\n\n if return_output:\n for j in range(n_ens_members):\n precip_forecast[j].extend(precip_forecast_[j])\n\n if measure_time:\n mainloop_time = time.time() - starttime_mainloop\n\n if return_output:\n outarr = np.stack([np.stack(precip_forecast[j]) for j in range(n_ens_members)])\n if measure_time:\n return outarr, init_time, mainloop_time\n else:\n return outarr\n else:\n return None\n\n\ndef _check_inputs(precip, velocity, timesteps, ar_order):\n if precip.ndim != 3:\n raise ValueError(\"precip must be a three-dimensional array\")\n if precip.shape[0] < ar_order + 1:\n raise ValueError(\"precip.shape[0] < ar_order+1\")\n if velocity.ndim != 3:\n raise ValueError(\"velocity must be a three-dimensional array\")\n if precip.shape[1:3] != velocity.shape[1:3]:\n raise ValueError(\n \"dimension mismatch between precip and velocity: precip.shape=%s, velocity.shape=%s\"\n % (str(precip.shape), str(precip.shape))\n )\n if isinstance(timesteps, list) and not sorted(timesteps) == timesteps:\n raise ValueError(\"timesteps is not in ascending order\")\n\n\n# TODO: Use the recomponse_cascade method in the cascade.decomposition module\ndef _recompose_cascade(precip, mu, sigma):\n precip_rc = [(precip[i, -1, :, :] * sigma[i]) + mu[i] for i in range(len(mu))]\n precip_rc = np.sum(np.stack(precip_rc), axis=0)\n\n return precip_rc\n\n\ndef _build_2D_tapering_function(win_size, win_type=\"flat-hanning\"):\n \"\"\"\n Produces two-dimensional tapering function for rectangular fields.\n\n Parameters\n ----------\n win_size: tuple of int\n Size of the tapering window as two-element tuple of integers.\n win_type: str\n Name of the tapering window type (hanning, flat-hanning)\n\n Returns\n -------\n w2d: array-like\n A two-dimensional numpy array containing the 2D tapering function.\n \"\"\"\n\n if len(win_size) != 2:\n raise ValueError(\"win_size is not a two-element tuple\")\n\n if win_type == \"hanning\":\n w1dr = np.hanning(win_size[0])\n w1dc = np.hanning(win_size[1])\n\n elif win_type == \"flat-hanning\":\n T = win_size[0] / 4.0\n W = win_size[0] / 2.0\n B = np.linspace(-W, W, int(2 * W))\n R = np.abs(B) - T\n R[R < 0] = 0.0\n A = 0.5 * (1.0 + np.cos(np.pi * R / T))\n A[np.abs(B) > (2 * T)] = 0.0\n w1dr = A\n\n T = win_size[1] / 4.0\n W = win_size[1] / 2.0\n B = np.linspace(-W, W, int(2 * W))\n R = np.abs(B) - T\n R[R < 0] = 0.0\n A = 0.5 * (1.0 + np.cos(np.pi * R / T))\n A[np.abs(B) > (2 * T)] = 0.0\n w1dc = A\n\n elif win_type == \"rectangular\":\n w1dr = np.ones(win_size[0])\n w1dc = np.ones(win_size[1])\n\n else:\n raise ValueError(\"unknown win_type %s\" % win_type)\n\n # Expand to 2-D\n # w2d = np.sqrt(np.outer(w1dr,w1dc))\n w2d = np.outer(w1dr, w1dc)\n\n # Set nans to zero\n if np.any(np.isnan(w2d)):\n w2d[np.isnan(w2d)] = np.min(w2d[w2d > 0])\n\n w2d[w2d < 1e-3] = 1e-3\n\n return w2d\n\n\ndef _get_mask(Size, idxi, idxj, win_type=\"flat-hanning\"):\n \"\"\"Compute a mask of zeros with a window at a given position.\"\"\"\n\n idxi = np.array(idxi).astype(int)\n idxj = np.array(idxj).astype(int)\n\n win_size = (idxi[1] - idxi[0], idxj[1] - idxj[0])\n wind = _build_2D_tapering_function(win_size, win_type)\n\n mask = np.zeros(Size)\n mask[idxi.item(0) : idxi.item(1), idxj.item(0) : idxj.item(1)] = wind\n\n return mask\n" }, { "path": "pysteps/nowcasts/steps.py", "content": "\"\"\"\npysteps.nowcasts.steps\n======================\n\nImplementation of the STEPS stochastic nowcasting method as described in\n:cite:`Seed2003`, :cite:`BPS2006` and :cite:`SPN2013`.\n\n.. autosummary::\n :toctree: ../generated/\n\n forecast\n\"\"\"\n\nimport time\nfrom copy import deepcopy\nfrom dataclasses import dataclass, field\nfrom typing import Any, Callable\n\nimport numpy as np\nfrom scipy.ndimage import generate_binary_structure, iterate_structure\n\nfrom pysteps import cascade, extrapolation, noise, utils\nfrom pysteps.nowcasts import utils as nowcast_utils\nfrom pysteps.nowcasts.utils import (\n compute_percentile_mask,\n nowcast_main_loop,\n zero_precipitation_forecast,\n)\nfrom pysteps.postprocessing import probmatching\nfrom pysteps.timeseries import autoregression, correlation\nfrom pysteps.utils.check_norain import check_norain\n\ntry:\n import dask\n\n DASK_IMPORTED = True\nexcept ImportError:\n DASK_IMPORTED = False\n\n\n@dataclass(frozen=True)\nclass StepsNowcasterConfig:\n \"\"\"\n Parameters\n ----------\n\n n_ens_members: int, optional\n The number of ensemble members to generate.\n n_cascade_levels: int, optional\n The number of cascade levels to use. Defaults to 6, see issue #385\n on GitHub.\n precip_threshold: float, optional\n Specifies the threshold value for minimum observable precipitation\n intensity. Required if mask_method is not None or conditional is True.\n norain_threshold: float\n Specifies the threshold value for the fraction of rainy (see above) pixels\n in the radar rainfall field below which we consider there to be no rain.\n Depends on the amount of clutter typically present.\n Standard set to 0.0\n kmperpixel: float, optional\n Spatial resolution of the input data (kilometers/pixel). Required if\n vel_pert_method is not None or mask_method is 'incremental'.\n timestep: float, optional\n Time step of the motion vectors (minutes). Required if vel_pert_method is\n not None or mask_method is 'incremental'.\n extrapolation_method: str, optional\n Name of the extrapolation method to use. See the documentation of\n pysteps.extrapolation.interface.\n decomposition_method: {'fft'}, optional\n Name of the cascade decomposition method to use. See the documentation\n of pysteps.cascade.interface.\n bandpass_filter_method: {'gaussian', 'uniform'}, optional\n Name of the bandpass filter method to use with the cascade decomposition.\n See the documentation of pysteps.cascade.interface.\n noise_method: {'parametric','nonparametric','ssft','nested',None}, optional\n Name of the noise generator to use for perturbating the precipitation\n field. See the documentation of pysteps.noise.interface. If set to None,\n no noise is generated.\n noise_stddev_adj: {'auto','fixed',None}, optional\n Optional adjustment for the standard deviations of the noise fields added\n to each cascade level. This is done to compensate incorrect std. dev.\n estimates of casace levels due to presence of no-rain areas. 'auto'=use\n the method implemented in pysteps.noise.utils.compute_noise_stddev_adjs.\n 'fixed'= use the formula given in :cite:`BPS2006` (eq. 6), None=disable\n noise std. dev adjustment.\n ar_order: int, optional\n The order of the autoregressive model to use. Must be >= 1.\n velocity_perturbation_method: {'bps',None}, optional\n Name of the noise generator to use for perturbing the advection field. See\n the documentation of pysteps.noise.interface. If set to None, the advection\n field is not perturbed.\n conditional: bool, optional\n If set to True, compute the statistics of the precipitation field\n conditionally by excluding pixels where the values are below the\n threshold precip_thr.\n mask_method: {'obs','sprog','incremental',None}, optional\n The method to use for masking no precipitation areas in the forecast\n field. The masked pixels are set to the minimum value of the observations.\n 'obs' = apply precip_thr to the most recently observed precipitation\n intensity field, 'sprog' = use the smoothed forecast field from S-PROG,\n where the AR(p) model has been applied, 'incremental' = iteratively\n buffer the mask with a certain rate (currently it is 1 km/min),\n None=no masking.\n probmatching_method: {'cdf','mean',None}, optional\n Method for matching the statistics of the forecast field with those of\n the most recently observed one. 'cdf'=map the forecast CDF to the observed\n one, 'mean'=adjust only the conditional mean value of the forecast field\n in precipitation areas, None=no matching applied. Using 'mean' requires\n that precip_thr and mask_method are not None.\n seed: int, optional\n Optional seed number for the random generators.\n num_workers: int, optional\n The number of workers to use for parallel computation. Applicable if dask\n is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it\n is advisable to disable OpenMP by setting the environment variable\n OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous\n threads.\n fft_method: str, optional\n A string defining the FFT method to use (see utils.fft.get_method).\n Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed,\n the recommended method is 'pyfftw'.\n domain: {\"spatial\", \"spectral\"}\n If \"spatial\", all computations are done in the spatial domain (the\n classical STEPS model). If \"spectral\", the AR(2) models and stochastic\n perturbations are applied directly in the spectral domain to reduce\n memory footprint and improve performance :cite:`PCH2019b`.\n extrapolation_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the extrapolation\n method. See the documentation of pysteps.extrapolation.\n filter_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the filter method.\n See the documentation of pysteps.cascade.bandpass_filters.py.\n noise_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the initializer of\n the noise generator. See the documentation of pysteps.noise.fftgenerators.\n velocity_perturbation_kwargs: dict, optional\n Optional dictionary containing keyword arguments 'p_par' and 'p_perp' for\n the initializer of the velocity perturbator. The choice of the optimal\n parameters depends on the domain and the used optical flow method.\n\n Default parameters from :cite:`BPS2006`:\n p_par = [10.88, 0.23, -7.68]\n p_perp = [5.76, 0.31, -2.72]\n\n Parameters fitted to the data (optical flow/domain):\n\n darts/fmi:\n p_par = [13.71259667, 0.15658963, -16.24368207]\n p_perp = [8.26550355, 0.17820458, -9.54107834]\n\n darts/mch:\n p_par = [24.27562298, 0.11297186, -27.30087471]\n p_perp = [-7.80797846e+01, -3.38641048e-02, 7.56715304e+01]\n\n darts/fmi+mch:\n p_par = [16.55447057, 0.14160448, -19.24613059]\n p_perp = [14.75343395, 0.11785398, -16.26151612]\n\n lucaskanade/fmi:\n p_par = [2.20837526, 0.33887032, -2.48995355]\n p_perp = [2.21722634, 0.32359621, -2.57402761]\n\n lucaskanade/mch:\n p_par = [2.56338484, 0.3330941, -2.99714349]\n p_perp = [1.31204508, 0.3578426, -1.02499891]\n\n lucaskanade/fmi+mch:\n p_par = [2.31970635, 0.33734287, -2.64972861]\n p_perp = [1.90769947, 0.33446594, -2.06603662]\n\n vet/fmi:\n p_par = [0.25337388, 0.67542291, 11.04895538]\n p_perp = [0.02432118, 0.99613295, 7.40146505]\n\n vet/mch:\n p_par = [0.5075159, 0.53895212, 7.90331791]\n p_perp = [0.68025501, 0.41761289, 4.73793581]\n\n vet/fmi+mch:\n p_par = [0.29495222, 0.62429207, 8.6804131 ]\n p_perp = [0.23127377, 0.59010281, 5.98180004]\n\n fmi=Finland, mch=Switzerland, fmi+mch=both pooled into the same data set\n\n The above parameters have been fitten by using run_vel_pert_analysis.py\n and fit_vel_pert_params.py located in the scripts directory.\n\n See pysteps.noise.motion for additional documentation.\n mask_kwargs: dict\n Optional dictionary containing mask keyword arguments 'mask_f' and\n 'mask_rim', the factor defining the the mask increment and the rim size,\n respectively.\n The mask increment is defined as mask_f*timestep/kmperpixel.\n measure_time: bool\n If set to True, measure, print and return the computation time.\n callback: function, optional\n Optional function that is called after computation of each time step of\n the nowcast. The function takes one argument: a three-dimensional array\n of shape (n_ens_members,h,w), where h and w are the height and width\n of the input precipitation fields, respectively. This can be used, for\n instance, writing the outputs into files.\n return_output: bool, optional\n Set to False to disable returning the outputs as numpy arrays. This can\n save memory if the intermediate results are written to output files using\n the callback function.\n \"\"\"\n\n n_ens_members: int = 24\n n_cascade_levels: int = 6\n precip_threshold: float | None = None\n norain_threshold: float = 0.0\n kmperpixel: float | None = None\n timestep: float | None = None\n extrapolation_method: str = \"semilagrangian\"\n decomposition_method: str = \"fft\"\n bandpass_filter_method: str = \"gaussian\"\n noise_method: str | None = \"nonparametric\"\n noise_stddev_adj: str | None = None\n ar_order: int = 2\n velocity_perturbation_method: str | None = \"bps\"\n conditional: bool = False\n probmatching_method: str | None = \"cdf\"\n mask_method: str | None = \"incremental\"\n seed: int | None = None\n num_workers: int = 1\n fft_method: str = \"numpy\"\n domain: str = \"spatial\"\n extrapolation_kwargs: dict[str, Any] = field(default_factory=dict)\n filter_kwargs: dict[str, Any] = field(default_factory=dict)\n noise_kwargs: dict[str, Any] = field(default_factory=dict)\n velocity_perturbation_kwargs: dict[str, Any] = field(default_factory=dict)\n mask_kwargs: dict[str, Any] = field(default_factory=dict)\n measure_time: bool = False\n callback: Callable[[Any], None] | None = None\n return_output: bool = True\n\n\n@dataclass\nclass StepsNowcasterParams:\n fft: Any = None\n bandpass_filter: Any = None\n extrapolation_method: Any = None\n decomposition_method: Any = None\n recomposition_method: Any = None\n noise_generator: Callable | None = None\n perturbation_generator: Callable | None = None\n noise_std_coefficients: np.ndarray | None = None\n ar_model_coefficients: np.ndarray | None = None # Corresponds to phi\n autocorrelation_coefficients: np.ndarray | None = None # Corresponds to gamma\n domain_mask: np.ndarray | None = None\n structuring_element: np.ndarray | None = None\n precipitation_mean: float | None = None\n wet_area_ratio: float | None = None\n mask_rim: int | None = None\n num_ensemble_workers: int = 1\n xy_coordinates: np.ndarray | None = None\n velocity_perturbation_parallel: list[float] | None = None\n velocity_perturbation_perpendicular: list[float] | None = None\n filter_kwargs: dict | None = None\n noise_kwargs: dict | None = None\n velocity_perturbation_kwargs: dict | None = None\n mask_kwargs: dict | None = None\n\n\n@dataclass\nclass StepsNowcasterState:\n precip_forecast: list[Any] | None = field(default_factory=list)\n precip_cascades: list[list[np.ndarray]] | None = field(default_factory=list)\n precip_decomposed: list[dict[str, Any]] | None = field(default_factory=list)\n # The observation mask (where the radar can observe the precipitation)\n precip_mask: list[Any] | None = field(default_factory=list)\n precip_mask_decomposed: dict[str, Any] | None = field(default_factory=dict)\n # The mask around the precipitation fields (to get only non-zero values)\n mask_precip: np.ndarray | None = None\n mask_threshold: np.ndarray | None = None\n random_generator_precip: list[np.random.RandomState] | None = field(\n default_factory=list\n )\n random_generator_motion: list[np.random.RandomState] | None = field(\n default_factory=list\n )\n velocity_perturbations: list[Callable] | None = field(default_factory=list)\n fft_objects: list[Any] | None = field(default_factory=list)\n extrapolation_kwargs: dict[str, Any] | None = field(default_factory=dict)\n\n\nclass StepsNowcaster:\n def __init__(\n self, precip, velocity, time_steps, steps_config: StepsNowcasterConfig\n ):\n # Store inputs and optional parameters\n self.__precip = precip\n self.__velocity = velocity\n self.__time_steps = time_steps\n\n # Store the config data:\n self.__config = steps_config\n\n # Store the state and params data:\n self.__state = StepsNowcasterState()\n self.__params = StepsNowcasterParams()\n\n # Additional variables for time measurement\n self.__start_time_init = None\n self.__init_time = None\n self.__mainloop_time = None\n\n def compute_forecast(self):\n \"\"\"\n Generate a nowcast ensemble by using the Short-Term Ensemble Prediction\n System (STEPS) method.\n\n Parameters\n ----------\n precip: array-like\n Array of shape (ar_order+1,m,n) containing the input precipitation fields\n ordered by timestamp from oldest to newest. The time steps between the\n inputs are assumed to be regular.\n velocity: array-like\n Array of shape (2,m,n) containing the x- and y-components of the advection\n field. The velocities are assumed to represent one time step between the\n inputs. All values are required to be finite.\n timesteps: int or list of floats\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed (relative to the input time step). The elements\n of the list are required to be in ascending order.\n config: StepsNowcasterConfig\n Provides a set of configuration parameters for the nowcast ensemble generation.\n\n Returns\n -------\n out: ndarray\n If return_output is True, a four-dimensional array of shape\n (n_ens_members,num_timesteps,m,n) containing a time series of forecast\n precipitation fields for each ensemble member. Otherwise, a None value\n is returned. The time series starts from t0+timestep, where timestep is\n taken from the input precipitation fields. If measure_time is True, the\n return value is a three-element tuple containing the nowcast array, the\n initialization time of the nowcast generator and the time used in the\n main loop (seconds).\n\n See also\n --------\n pysteps.extrapolation.interface, pysteps.cascade.interface,\n pysteps.noise.interface, pysteps.noise.utils.compute_noise_stddev_adjs\n\n References\n ----------\n :cite:`Seed2003`, :cite:`BPS2006`, :cite:`SPN2013`, :cite:`PCH2019b`\n \"\"\"\n self.__check_inputs()\n self.__print_forecast_info()\n # Measure time for initialization\n if self.__config.measure_time:\n self.__start_time_init = time.time()\n\n # Slice the precipitation field to only use the last ar_order + 1 fields\n self.__precip = self.__precip[-(self.__config.ar_order + 1) :, :, :].copy()\n self.__initialize_nowcast_components()\n if check_norain(\n self.__precip,\n self.__config.precip_threshold,\n self.__config.norain_threshold,\n self.__params.noise_kwargs[\"win_fun\"],\n ):\n return zero_precipitation_forecast(\n self.__config.n_ens_members,\n self.__time_steps,\n self.__precip,\n self.__config.callback,\n self.__config.return_output,\n self.__config.measure_time,\n self.__start_time_init,\n )\n\n self.__perform_extrapolation()\n self.__apply_noise_and_ar_model()\n self.__initialize_velocity_perturbations()\n self.__initialize_precipitation_mask()\n self.__initialize_fft_objects()\n # Measure and print initialization time\n if self.__config.measure_time:\n self.__init_time = self.__measure_time(\n \"Initialization\", self.__start_time_init\n )\n\n # Run the main nowcast loop\n self.__nowcast_main()\n\n # Unstack nowcast output if return_output is True\n if self.__config.measure_time:\n (\n self.__state.precip_forecast,\n self.__mainloop_time,\n ) = self.__state.precip_forecast\n\n # Stack and return the forecast output\n if self.__config.return_output:\n self.__state.precip_forecast = np.stack(\n [\n np.stack(self.__state.precip_forecast[j])\n for j in range(self.__config.n_ens_members)\n ]\n )\n if self.__config.measure_time:\n return (\n self.__state.precip_forecast,\n self.__init_time,\n self.__mainloop_time,\n )\n else:\n return self.__state.precip_forecast\n else:\n return None\n\n def __nowcast_main(self):\n \"\"\"\n Main nowcast loop that iterates through the ensemble members and time steps\n to generate forecasts.\n \"\"\"\n # Isolate the last time slice of observed precipitation\n precip = self.__precip[\n -1, :, :\n ] # Extract the last available precipitation field\n\n # Prepare state and params dictionaries, these need to be formatted a specific way for the nowcast_main_loop\n state = self.__return_state_dict()\n params = self.__return_params_dict(precip)\n\n print(\"Starting nowcast computation.\")\n\n # Run the nowcast main loop\n self.__state.precip_forecast = nowcast_main_loop(\n precip,\n self.__velocity,\n state,\n self.__time_steps,\n self.__config.extrapolation_method,\n self.__update_state, # Reference to the update function\n extrap_kwargs=self.__state.extrapolation_kwargs,\n velocity_pert_gen=self.__state.velocity_perturbations,\n params=params,\n ensemble=True,\n num_ensemble_members=self.__config.n_ens_members,\n callback=self.__config.callback,\n return_output=self.__config.return_output,\n num_workers=self.__params.num_ensemble_workers,\n measure_time=self.__config.measure_time,\n )\n\n def __check_inputs(self):\n \"\"\"\n Validate the inputs to ensure consistency and correct shapes.\n \"\"\"\n if self.__precip.ndim != 3:\n raise ValueError(\"precip must be a three-dimensional array\")\n if self.__precip.shape[0] < self.__config.ar_order + 1:\n raise ValueError(\n f\"precip.shape[0] must be at least ar_order+1, \"\n f\"but found {self.__precip.shape[0]}\"\n )\n if self.__velocity.ndim != 3:\n raise ValueError(\"velocity must be a three-dimensional array\")\n if self.__precip.shape[1:3] != self.__velocity.shape[1:3]:\n raise ValueError(\n f\"Dimension mismatch between precip and velocity: \"\n f\"shape(precip)={self.__precip.shape}, shape(velocity)={self.__velocity.shape}\"\n )\n if (\n isinstance(self.__time_steps, list)\n and not sorted(self.__time_steps) == self.__time_steps\n ):\n raise ValueError(\"timesteps must be in ascending order\")\n if np.any(~np.isfinite(self.__velocity)):\n raise ValueError(\"velocity contains non-finite values\")\n if self.__config.mask_method not in [\"obs\", \"sprog\", \"incremental\", None]:\n raise ValueError(\n f\"Unknown mask method '{self.__config.mask_method}'. \"\n \"Must be 'obs', 'sprog', 'incremental', or None.\"\n )\n if self.__config.precip_threshold is None:\n if self.__config.conditional:\n raise ValueError(\"conditional=True but precip_thr is not specified.\")\n if self.__config.mask_method is not None:\n raise ValueError(\"mask_method is set but precip_thr is not specified.\")\n if self.__config.probmatching_method == \"mean\":\n raise ValueError(\n \"probmatching_method='mean' but precip_thr is not specified.\"\n )\n if (\n self.__config.noise_method is not None\n and self.__config.noise_stddev_adj == \"auto\"\n ):\n raise ValueError(\n \"noise_stddev_adj='auto' but precip_thr is not specified.\"\n )\n if self.__config.noise_stddev_adj not in [\"auto\", \"fixed\", None]:\n raise ValueError(\n f\"Unknown noise_stddev_adj method '{self.__config.noise_stddev_adj}'. \"\n \"Must be 'auto', 'fixed', or None.\"\n )\n if self.__config.kmperpixel is None:\n if self.__config.velocity_perturbation_method is not None:\n raise ValueError(\"vel_pert_method is set but kmperpixel=None\")\n if self.__config.mask_method == \"incremental\":\n raise ValueError(\"mask_method='incremental' but kmperpixel=None\")\n if self.__config.timestep is None:\n if self.__config.velocity_perturbation_method is not None:\n raise ValueError(\"vel_pert_method is set but timestep=None\")\n if self.__config.mask_method == \"incremental\":\n raise ValueError(\"mask_method='incremental' but timestep=None\")\n\n # Handle None values for various kwargs\n if self.__config.extrapolation_kwargs is None:\n self.__state.extrapolation_kwargs = dict()\n else:\n self.__state.extrapolation_kwargs = deepcopy(\n self.__config.extrapolation_kwargs\n )\n\n if self.__config.filter_kwargs is None:\n self.__params.filter_kwargs = dict()\n else:\n self.__params.filter_kwargs = deepcopy(self.__config.filter_kwargs)\n\n if self.__config.noise_kwargs is None:\n self.__params.noise_kwargs = {\"win_fun\": \"tukey\"}\n else:\n self.__params.noise_kwargs = deepcopy(self.__config.noise_kwargs)\n\n if self.__config.velocity_perturbation_kwargs is None:\n self.__params.velocity_perturbation_kwargs = dict()\n else:\n self.__params.velocity_perturbation_kwargs = deepcopy(\n self.__config.velocity_perturbation_kwargs\n )\n\n if self.__config.mask_kwargs is None:\n self.__params.mask_kwargs = dict()\n else:\n self.__params.mask_kwargs = deepcopy(self.__config.mask_kwargs)\n\n print(\"Inputs validated and initialized successfully.\")\n\n def __print_forecast_info(self):\n \"\"\"\n Print information about the forecast setup, including inputs, methods, and parameters.\n \"\"\"\n print(\"Computing STEPS nowcast\")\n print(\"-----------------------\")\n print(\"\")\n\n print(\"Inputs\")\n print(\"------\")\n print(f\"input dimensions: {self.__precip.shape[1]}x{self.__precip.shape[2]}\")\n if self.__config.kmperpixel is not None:\n print(f\"km/pixel: {self.__config.kmperpixel}\")\n if self.__config.timestep is not None:\n print(f\"time step: {self.__config.timestep} minutes\")\n print(\"\")\n\n print(\"Methods\")\n print(\"-------\")\n print(f\"extrapolation: {self.__config.extrapolation_method}\")\n print(f\"bandpass filter: {self.__config.bandpass_filter_method}\")\n print(f\"decomposition: {self.__config.decomposition_method}\")\n print(f\"noise generator: {self.__config.noise_method}\")\n print(\n \"noise adjustment: {}\".format(\n (\"yes\" if self.__config.noise_stddev_adj else \"no\")\n )\n )\n print(f\"velocity perturbator: {self.__config.velocity_perturbation_method}\")\n print(\n \"conditional statistics: {}\".format(\n (\"yes\" if self.__config.conditional else \"no\")\n )\n )\n print(f\"precip. mask method: {self.__config.mask_method}\")\n print(f\"probability matching: {self.__config.probmatching_method}\")\n print(f\"FFT method: {self.__config.fft_method}\")\n print(f\"domain: {self.__config.domain}\")\n print(\"\")\n\n print(\"Parameters\")\n print(\"----------\")\n if isinstance(self.__time_steps, int):\n print(f\"number of time steps: {self.__time_steps}\")\n else:\n print(f\"time steps: {self.__time_steps}\")\n print(f\"ensemble size: {self.__config.n_ens_members}\")\n print(f\"parallel threads: {self.__config.num_workers}\")\n print(f\"number of cascade levels: {self.__config.n_cascade_levels}\")\n print(f\"order of the AR(p) model: {self.__config.ar_order}\")\n\n if self.__config.velocity_perturbation_method == \"bps\":\n self.__params.velocity_perturbation_parallel = (\n self.__params.velocity_perturbation_kwargs.get(\n \"p_par\", noise.motion.get_default_params_bps_par()\n )\n )\n self.__params.velocity_perturbation_perpendicular = (\n self.__params.velocity_perturbation_kwargs.get(\n \"p_perp\", noise.motion.get_default_params_bps_perp()\n )\n )\n print(\n f\"velocity perturbations, parallel: {self.__params.velocity_perturbation_parallel[0]},{self.__params.velocity_perturbation_parallel[1]},{self.__params.velocity_perturbation_parallel[2]}\"\n )\n print(\n f\"velocity perturbations, perpendicular: {self.__params.velocity_perturbation_perpendicular[0]},{self.__params.velocity_perturbation_perpendicular[1]},{self.__params.velocity_perturbation_perpendicular[2]}\"\n )\n\n if self.__config.precip_threshold is not None:\n print(f\"precip. intensity threshold: {self.__config.precip_threshold}\")\n\n def __initialize_nowcast_components(self):\n \"\"\"\n Initialize the FFT, bandpass filters, decomposition methods, and extrapolation method.\n \"\"\"\n # Initialize number of ensemble workers\n self.__params.num_ensemble_workers = min(\n self.__config.n_ens_members, self.__config.num_workers\n )\n\n M, N = self.__precip.shape[1:] # Extract the spatial dimensions (height, width)\n\n # Initialize FFT method\n self.__params.fft = utils.get_method(\n self.__config.fft_method, shape=(M, N), n_threads=self.__config.num_workers\n )\n\n # Initialize the band-pass filter for the cascade decomposition\n filter_method = cascade.get_method(self.__config.bandpass_filter_method)\n self.__params.bandpass_filter = filter_method(\n (M, N),\n self.__config.n_cascade_levels,\n **(self.__params.filter_kwargs or {}),\n )\n\n # Get the decomposition method (e.g., FFT)\n (\n self.__params.decomposition_method,\n self.__params.recomposition_method,\n ) = cascade.get_method(self.__config.decomposition_method)\n\n # Get the extrapolation method (e.g., semilagrangian)\n self.__params.extrapolation_method = extrapolation.get_method(\n self.__config.extrapolation_method\n )\n\n # Generate the mesh grid for spatial coordinates\n x_values, y_values = np.meshgrid(np.arange(N), np.arange(M))\n self.__params.xy_coordinates = np.stack([x_values, y_values])\n\n # Determine the domain mask from non-finite values in the precipitation data\n self.__params.domain_mask = np.logical_or.reduce(\n [~np.isfinite(self.__precip[i, :]) for i in range(self.__precip.shape[0])]\n )\n\n print(\"Nowcast components initialized successfully.\")\n\n def __perform_extrapolation(self):\n \"\"\"\n Extrapolate (advect) precipitation fields based on the velocity field to align\n them in time. This prepares the precipitation fields for autoregressive modeling.\n \"\"\"\n # Determine the precipitation threshold mask if conditional is set\n if self.__config.conditional:\n self.__state.mask_threshold = np.logical_and.reduce(\n [\n self.__precip[i, :, :] >= self.__config.precip_threshold\n for i in range(self.__precip.shape[0])\n ]\n )\n else:\n self.__state.mask_threshold = None\n\n extrap_kwargs = self.__state.extrapolation_kwargs.copy()\n extrap_kwargs[\"xy_coords\"] = self.__params.xy_coordinates\n extrap_kwargs[\"allow_nonfinite_values\"] = (\n True if np.any(~np.isfinite(self.__precip)) else False\n )\n\n res = []\n\n def __extrapolate_single_field(precip, i):\n # Extrapolate a single precipitation field using the velocity field\n return self.__params.extrapolation_method(\n precip[i, :, :],\n self.__velocity,\n self.__config.ar_order - i,\n \"min\",\n **extrap_kwargs,\n )[-1]\n\n for i in range(self.__config.ar_order):\n if (\n not DASK_IMPORTED\n ): # If Dask is not available, perform sequential extrapolation\n self.__precip[i, :, :] = __extrapolate_single_field(self.__precip, i)\n else:\n # If Dask is available, accumulate delayed computations for parallel execution\n res.append(dask.delayed(__extrapolate_single_field)(self.__precip, i))\n\n # If Dask is available, perform the parallel computation\n if DASK_IMPORTED and res:\n num_workers_ = min(self.__params.num_ensemble_workers, len(res))\n self.__precip = np.stack(\n list(dask.compute(*res, num_workers=num_workers_))\n + [self.__precip[-1, :, :]]\n )\n\n print(\"Extrapolation complete and precipitation fields aligned.\")\n\n def __apply_noise_and_ar_model(self):\n \"\"\"\n Apply noise and autoregressive (AR) models to precipitation cascades.\n This method applies the AR model to the decomposed precipitation cascades\n and adds noise perturbations if necessary.\n \"\"\"\n # Make a copy of the precipitation data and replace non-finite values\n precip = self.__precip.copy()\n for i in range(self.__precip.shape[0]):\n # Replace non-finite values with the minimum finite value of the precipitation field\n precip[i, ~np.isfinite(precip[i, :])] = np.nanmin(precip[i, :])\n # Store the precipitation data back in the object\n self.__precip = precip\n\n # Initialize the noise generator if the noise_method is provided\n if self.__config.noise_method is not None:\n np.random.seed(\n self.__config.seed\n ) # Set the random seed for reproducibility\n init_noise, generate_noise = noise.get_method(self.__config.noise_method)\n self.__params.noise_generator = generate_noise\n\n self.__params.perturbation_generator = init_noise(\n self.__precip,\n fft_method=self.__params.fft,\n **self.__params.noise_kwargs,\n )\n\n # Handle noise standard deviation adjustments if necessary\n if self.__config.noise_stddev_adj == \"auto\":\n print(\"Computing noise adjustment coefficients... \", end=\"\", flush=True)\n if self.__config.measure_time:\n starttime = time.time()\n\n # Compute noise adjustment coefficients\n self.__params.noise_std_coefficients = (\n noise.utils.compute_noise_stddev_adjs(\n self.__precip[-1, :, :],\n self.__config.precip_threshold,\n np.min(self.__precip),\n self.__params.bandpass_filter,\n self.__params.decomposition_method,\n self.__params.perturbation_generator,\n self.__params.noise_generator,\n 20,\n conditional=self.__config.conditional,\n num_workers=self.__config.num_workers,\n seed=self.__config.seed,\n )\n )\n\n # Measure and print time taken\n if self.__config.measure_time:\n __ = self.__measure_time(\n \"Noise adjustment coefficient computation\", starttime\n )\n else:\n print(\"done.\")\n\n elif self.__config.noise_stddev_adj == \"fixed\":\n # Set fixed noise adjustment coefficients\n func = lambda k: 1.0 / (0.75 + 0.09 * k)\n self.__params.noise_std_coefficients = [\n func(k) for k in range(1, self.__config.n_cascade_levels + 1)\n ]\n\n else:\n # Default to no adjustment\n self.__params.noise_std_coefficients = np.ones(\n self.__config.n_cascade_levels\n )\n\n if self.__config.noise_stddev_adj is not None:\n # Print noise std deviation coefficients if adjustments were made\n print(\n f\"noise std. dev. coeffs: {str(self.__params.noise_std_coefficients)}\"\n )\n\n else:\n # No noise, so set perturbation generator and noise_std_coefficients to None\n self.__params.perturbation_generator = None\n self.__params.noise_std_coefficients = np.ones(\n self.__config.n_cascade_levels\n ) # Keep default as 1.0 to avoid breaking AR model\n\n # Decompose the input precipitation fields\n self.__state.precip_decomposed = []\n for i in range(self.__config.ar_order + 1):\n precip_ = self.__params.decomposition_method(\n self.__precip[i, :, :],\n self.__params.bandpass_filter,\n mask=self.__state.mask_threshold,\n fft_method=self.__params.fft,\n output_domain=self.__config.domain,\n normalize=True,\n compute_stats=True,\n compact_output=True,\n )\n self.__state.precip_decomposed.append(precip_)\n\n # Normalize the cascades and rearrange them into a 4D array\n self.__state.precip_cascades = nowcast_utils.stack_cascades(\n self.__state.precip_decomposed, self.__config.n_cascade_levels\n )\n self.__state.precip_decomposed = self.__state.precip_decomposed[-1]\n self.__state.precip_decomposed = [\n self.__state.precip_decomposed.copy()\n for _ in range(self.__config.n_ens_members)\n ]\n\n # Compute temporal autocorrelation coefficients for each cascade level\n self.__params.autocorrelation_coefficients = np.empty(\n (self.__config.n_cascade_levels, self.__config.ar_order)\n )\n for i in range(self.__config.n_cascade_levels):\n self.__params.autocorrelation_coefficients[i, :] = (\n correlation.temporal_autocorrelation(\n self.__state.precip_cascades[i], mask=self.__state.mask_threshold\n )\n )\n\n nowcast_utils.print_corrcoefs(self.__params.autocorrelation_coefficients)\n\n # Adjust the lag-2 correlation coefficient if AR(2) model is used\n if self.__config.ar_order == 2:\n for i in range(self.__config.n_cascade_levels):\n self.__params.autocorrelation_coefficients[i, 1] = (\n autoregression.adjust_lag2_corrcoef2(\n self.__params.autocorrelation_coefficients[i, 0],\n self.__params.autocorrelation_coefficients[i, 1],\n )\n )\n\n # Estimate the parameters of the AR model using auto-correlation coefficients\n self.__params.ar_model_coefficients = np.empty(\n (self.__config.n_cascade_levels, self.__config.ar_order + 1)\n )\n for i in range(self.__config.n_cascade_levels):\n self.__params.ar_model_coefficients[i, :] = (\n autoregression.estimate_ar_params_yw(\n self.__params.autocorrelation_coefficients[i, :]\n )\n )\n\n nowcast_utils.print_ar_params(self.__params.ar_model_coefficients)\n\n # Discard all except the last ar_order cascades for AR model\n self.__state.precip_cascades = [\n self.__state.precip_cascades[i][-self.__config.ar_order :]\n for i in range(self.__config.n_cascade_levels)\n ]\n\n # Stack the cascades into a list containing all ensemble members\n self.__state.precip_cascades = [\n [\n self.__state.precip_cascades[j].copy()\n for j in range(self.__config.n_cascade_levels)\n ]\n for _ in range(self.__config.n_ens_members)\n ]\n\n # Initialize random generators if noise_method is provided\n if self.__config.noise_method is not None:\n self.__state.random_generator_precip = []\n self.__state.random_generator_motion = []\n seed = self.__config.seed\n for _ in range(self.__config.n_ens_members):\n # Create random state for precipitation noise generator\n rs = np.random.RandomState(seed)\n self.__state.random_generator_precip.append(rs)\n seed = rs.randint(0, high=int(1e9))\n # Create random state for motion perturbations generator\n rs = np.random.RandomState(seed)\n self.__state.random_generator_motion.append(rs)\n seed = rs.randint(0, high=int(1e9))\n else:\n self.__state.random_generator_precip = None\n self.__state.random_generator_motion = None\n print(\"AR model and noise applied to precipitation cascades.\")\n\n def __initialize_velocity_perturbations(self):\n \"\"\"\n Initialize the velocity perturbators for each ensemble member if the velocity\n perturbation method is specified.\n \"\"\"\n if self.__config.velocity_perturbation_method is not None:\n init_vel_noise, generate_vel_noise = noise.get_method(\n self.__config.velocity_perturbation_method\n )\n\n self.__state.velocity_perturbations = []\n for j in range(self.__config.n_ens_members):\n kwargs = {\n \"randstate\": self.__state.random_generator_motion[j],\n \"p_par\": self.__params.velocity_perturbation_kwargs.get(\n \"p_par\", self.__params.velocity_perturbation_parallel\n ),\n \"p_perp\": self.__params.velocity_perturbation_kwargs.get(\n \"p_perp\", self.__params.velocity_perturbation_perpendicular\n ),\n }\n vp = init_vel_noise(\n self.__velocity,\n 1.0 / self.__config.kmperpixel,\n self.__config.timestep,\n **kwargs,\n )\n self.__state.velocity_perturbations.append(\n lambda t, vp=vp: generate_vel_noise(vp, t * self.__config.timestep)\n )\n else:\n self.__state.velocity_perturbations = None\n print(\"Velocity perturbations initialized successfully.\")\n\n def __initialize_precipitation_mask(self):\n \"\"\"\n Initialize the precipitation mask and handle different mask methods (sprog, incremental).\n \"\"\"\n self.__state.precip_forecast = [[] for _ in range(self.__config.n_ens_members)]\n\n if self.__config.probmatching_method == \"mean\":\n self.__params.precipitation_mean = np.mean(\n self.__precip[-1, :, :][\n self.__precip[-1, :, :] >= self.__config.precip_threshold\n ]\n )\n else:\n self.__params.precipitation_mean = None\n\n if self.__config.mask_method is not None:\n self.__state.mask_precip = (\n self.__precip[-1, :, :] >= self.__config.precip_threshold\n )\n\n if self.__config.mask_method == \"sprog\":\n # Compute the wet area ratio and the precipitation mask\n self.__params.wet_area_ratio = np.sum(self.__state.mask_precip) / (\n self.__precip.shape[1] * self.__precip.shape[2]\n )\n self.__state.precip_mask = [\n self.__state.precip_cascades[0][i].copy()\n for i in range(self.__config.n_cascade_levels)\n ]\n self.__state.precip_mask_decomposed = self.__state.precip_decomposed[\n 0\n ].copy()\n\n elif self.__config.mask_method == \"incremental\":\n # Get mask parameters\n self.__params.mask_rim = self.__params.mask_kwargs.get(\"mask_rim\", 10)\n mask_f = self.__params.mask_kwargs.get(\"mask_f\", 1.0)\n # Initialize the structuring element\n self.__params.structuring_element = generate_binary_structure(2, 1)\n # Expand the structuring element based on mask factor and timestep\n n = mask_f * self.__config.timestep / self.__config.kmperpixel\n self.__params.structuring_element = iterate_structure(\n self.__params.structuring_element, int((n - 1) / 2.0)\n )\n # Compute and apply the dilated mask for each ensemble member\n self.__state.mask_precip = nowcast_utils.compute_dilated_mask(\n self.__state.mask_precip,\n self.__params.structuring_element,\n self.__params.mask_rim,\n )\n self.__state.mask_precip = [\n self.__state.mask_precip.copy()\n for _ in range(self.__config.n_ens_members)\n ]\n else:\n self.__state.mask_precip = None\n\n if self.__config.noise_method is None and self.__state.precip_mask is None:\n self.__state.precip_mask = [\n self.__state.precip_cascades[0][i].copy()\n for i in range(self.__config.n_cascade_levels)\n ]\n print(\"Precipitation mask initialized successfully.\")\n\n def __initialize_fft_objects(self):\n \"\"\"\n Initialize FFT objects for each ensemble member.\n \"\"\"\n self.__state.fft_objs = []\n for _ in range(self.__config.n_ens_members):\n fft_obj = utils.get_method(\n self.__config.fft_method, shape=self.__precip.shape[1:]\n )\n self.__state.fft_objs.append(fft_obj)\n print(\"FFT objects initialized successfully.\")\n\n def __return_state_dict(self):\n \"\"\"\n Initialize the state dictionary used during the nowcast iteration.\n \"\"\"\n return {\n \"fft_objs\": self.__state.fft_objs,\n \"mask_prec\": self.__state.mask_precip,\n \"precip_cascades\": self.__state.precip_cascades,\n \"precip_decomp\": self.__state.precip_decomposed,\n \"precip_m\": self.__state.precip_mask,\n \"precip_m_d\": self.__state.precip_mask_decomposed,\n \"randgen_prec\": self.__state.random_generator_precip,\n }\n\n def __return_params_dict(self, precip):\n \"\"\"\n Initialize the params dictionary used during the nowcast iteration.\n \"\"\"\n return {\n \"decomp_method\": self.__params.decomposition_method,\n \"domain\": self.__config.domain,\n \"domain_mask\": self.__params.domain_mask,\n \"filter\": self.__params.bandpass_filter,\n \"fft\": self.__params.fft,\n \"generate_noise\": self.__params.noise_generator,\n \"mask_method\": self.__config.mask_method,\n \"mask_rim\": self.__params.mask_rim,\n \"mu_0\": self.__params.precipitation_mean,\n \"n_cascade_levels\": self.__config.n_cascade_levels,\n \"n_ens_members\": self.__config.n_ens_members,\n \"noise_method\": self.__config.noise_method,\n \"noise_std_coeffs\": self.__params.noise_std_coefficients,\n \"num_ensemble_workers\": self.__params.num_ensemble_workers,\n \"phi\": self.__params.ar_model_coefficients,\n \"pert_gen\": self.__params.perturbation_generator,\n \"probmatching_method\": self.__config.probmatching_method,\n \"precip\": precip,\n \"precip_thr\": self.__config.precip_threshold,\n \"recomp_method\": self.__params.recomposition_method,\n \"struct\": self.__params.structuring_element,\n \"war\": self.__params.wet_area_ratio,\n }\n\n def __update_state(self, state, params):\n \"\"\"\n Update the state during the nowcasting loop. This function handles the AR model iteration,\n noise generation, recomposition, and mask application for each ensemble member.\n \"\"\"\n precip_forecast_out = [None] * params[\"n_ens_members\"]\n\n # Update the deterministic AR(p) model if noise or sprog mask is used\n if params[\"noise_method\"] is None or params[\"mask_method\"] == \"sprog\":\n self.__update_deterministic_ar_model(state, params)\n\n # Worker function for each ensemble member\n def worker(j):\n self.__apply_ar_model_to_cascades(j, state, params)\n precip_forecast_out[j] = self.__recompose_and_apply_mask(j, state, params)\n\n # Use Dask for parallel execution if available\n if (\n DASK_IMPORTED\n and params[\"n_ens_members\"] > 1\n and params[\"num_ensemble_workers\"] > 1\n ):\n res = []\n for j in range(params[\"n_ens_members\"]):\n res.append(dask.delayed(worker)(j))\n dask.compute(*res, num_workers=params[\"num_ensemble_workers\"])\n else:\n for j in range(params[\"n_ens_members\"]):\n worker(j)\n\n return np.stack(precip_forecast_out), state\n\n def __update_deterministic_ar_model(self, state, params):\n \"\"\"\n Update the deterministic AR(p) model for each cascade level if noise is disabled\n or if the sprog mask is used.\n \"\"\"\n for i in range(params[\"n_cascade_levels\"]):\n state[\"precip_m\"][i] = autoregression.iterate_ar_model(\n state[\"precip_m\"][i], params[\"phi\"][i, :]\n )\n\n state[\"precip_m_d\"][\"cascade_levels\"] = [\n state[\"precip_m\"][i][-1] for i in range(params[\"n_cascade_levels\"])\n ]\n\n if params[\"domain\"] == \"spatial\":\n state[\"precip_m_d\"][\"cascade_levels\"] = np.stack(\n state[\"precip_m_d\"][\"cascade_levels\"]\n )\n\n precip_m_ = params[\"recomp_method\"](state[\"precip_m_d\"])\n\n if params[\"domain\"] == \"spectral\":\n precip_m_ = params[\"fft\"].irfft2(precip_m_)\n\n if params[\"mask_method\"] == \"sprog\":\n state[\"mask_prec\"] = compute_percentile_mask(precip_m_, params[\"war\"])\n\n def __apply_ar_model_to_cascades(self, j, state, params):\n \"\"\"\n Apply the AR(p) model to the cascades for each ensemble member, including\n noise generation and normalization.\n \"\"\"\n # Generate noise if enabled\n if params[\"noise_method\"] is not None:\n eps = self.__generate_and_decompose_noise(j, state, params)\n else:\n eps = None\n\n # Iterate the AR(p) model for each cascade level\n for i in range(params[\"n_cascade_levels\"]):\n if eps is not None:\n eps_ = eps[\"cascade_levels\"][i]\n eps_ *= params[\"noise_std_coeffs\"][i]\n else:\n eps_ = None\n\n # Apply the AR(p) model with or without perturbations\n if eps is not None or params[\"vel_pert_method\"] is not None:\n state[\"precip_cascades\"][j][i] = autoregression.iterate_ar_model(\n state[\"precip_cascades\"][j][i], params[\"phi\"][i, :], eps=eps_\n )\n else:\n # use the deterministic AR(p) model computed above if\n # perturbations are disabled\n state[\"precip_cascades\"][j][i] = state[\"precip_m\"][i]\n\n eps = None\n eps_ = None\n\n def __generate_and_decompose_noise(self, j, state, params):\n \"\"\"\n Generate and decompose the noise field into cascades for a given ensemble member.\n \"\"\"\n eps = params[\"generate_noise\"](\n params[\"pert_gen\"],\n randstate=state[\"randgen_prec\"][j],\n fft_method=state[\"fft_objs\"][j],\n domain=params[\"domain\"],\n )\n\n eps = params[\"decomp_method\"](\n eps,\n params[\"filter\"],\n fft_method=state[\"fft_objs\"][j],\n input_domain=params[\"domain\"],\n output_domain=params[\"domain\"],\n compute_stats=True,\n normalize=True,\n compact_output=True,\n )\n\n return eps\n\n def __recompose_and_apply_mask(self, j, state, params):\n \"\"\"\n Recompose the precipitation field from cascades and apply the precipitation mask.\n \"\"\"\n state[\"precip_decomp\"][j][\"cascade_levels\"] = [\n state[\"precip_cascades\"][j][i][-1, :]\n for i in range(params[\"n_cascade_levels\"])\n ]\n\n if params[\"domain\"] == \"spatial\":\n state[\"precip_decomp\"][j][\"cascade_levels\"] = np.stack(\n state[\"precip_decomp\"][j][\"cascade_levels\"]\n )\n\n precip_forecast = params[\"recomp_method\"](state[\"precip_decomp\"][j])\n\n if params[\"domain\"] == \"spectral\":\n precip_forecast = state[\"fft_objs\"][j].irfft2(precip_forecast)\n\n # Apply the precipitation mask\n if params[\"mask_method\"] is not None:\n precip_forecast = self.__apply_precipitation_mask(\n precip_forecast, j, state, params\n )\n\n # Adjust the CDF of the forecast to match the observed precipitation field\n if params[\"probmatching_method\"] == \"cdf\":\n precip_forecast = probmatching.nonparam_match_empirical_cdf(\n precip_forecast, params[\"precip\"]\n )\n # Adjust the mean of the forecast to match the observed mean\n elif params[\"probmatching_method\"] == \"mean\":\n mask = precip_forecast >= params[\"precip_thr\"]\n mu_fct = np.mean(precip_forecast[mask])\n precip_forecast[mask] = precip_forecast[mask] - mu_fct + params[\"mu_0\"]\n\n # Update the mask for incremental method\n if params[\"mask_method\"] == \"incremental\":\n state[\"mask_prec\"][j] = nowcast_utils.compute_dilated_mask(\n precip_forecast >= params[\"precip_thr\"],\n params[\"struct\"],\n params[\"mask_rim\"],\n )\n\n # Apply the domain mask (set masked areas to NaN)\n precip_forecast[params[\"domain_mask\"]] = np.nan\n\n return precip_forecast\n\n def __apply_precipitation_mask(self, precip_forecast, j, state, params):\n \"\"\"\n Apply the precipitation mask to prevent new precipitation from generating\n in areas where it was not observed.\n \"\"\"\n precip_forecast_min = precip_forecast.min()\n\n if params[\"mask_method\"] == \"incremental\":\n precip_forecast = (\n precip_forecast_min\n + (precip_forecast - precip_forecast_min) * state[\"mask_prec\"][j]\n )\n mask_prec_ = precip_forecast > precip_forecast_min\n else:\n mask_prec_ = state[\"mask_prec\"]\n\n # Set to min value outside the mask\n precip_forecast[~mask_prec_] = precip_forecast_min\n\n return precip_forecast\n\n def __measure_time(self, label, start_time):\n \"\"\"\n Measure and print the time taken for a specific part of the process.\n\n Parameters:\n - label: A description of the part of the process being measured.\n - start_time: The timestamp when the process started (from time.time()).\n \"\"\"\n if self.__config.measure_time:\n elapsed_time = time.time() - start_time\n print(f\"{label} took {elapsed_time:.2f} seconds.\")\n return elapsed_time\n return None\n\n def reset_states_and_params(self):\n \"\"\"\n Reset the internal state and parameters of the nowcaster to allow multiple forecasts.\n This method resets the state and params to their initial conditions without reinitializing\n the inputs like precip, velocity, time_steps, or config.\n \"\"\"\n # Re-initialize the state and parameters\n self.__state = StepsNowcasterState()\n self.__params = StepsNowcasterParams()\n\n # Reset time measurement variables\n self.__start_time_init = None\n self.__init_time = None\n self.__mainloop_time = None\n\n\n# Wrapper function to preserve backward compatibility\ndef forecast(\n precip,\n velocity,\n timesteps,\n n_ens_members=24,\n n_cascade_levels=6,\n precip_thr=None,\n norain_thr=0.0,\n kmperpixel=None,\n timestep=None,\n extrap_method=\"semilagrangian\",\n decomp_method=\"fft\",\n bandpass_filter_method=\"gaussian\",\n noise_method=\"nonparametric\",\n noise_stddev_adj=None,\n ar_order=2,\n vel_pert_method=\"bps\",\n conditional=False,\n probmatching_method=\"cdf\",\n mask_method=\"incremental\",\n seed=None,\n num_workers=1,\n fft_method=\"numpy\",\n domain=\"spatial\",\n extrap_kwargs=None,\n filter_kwargs=None,\n noise_kwargs=None,\n vel_pert_kwargs=None,\n mask_kwargs=None,\n measure_time=False,\n callback=None,\n return_output=True,\n):\n \"\"\"\n Generate a nowcast ensemble by using the Short-Term Ensemble Prediction\n System (STEPS) method.\n\n Parameters\n ----------\n precip: array-like\n Array of shape (ar_order+1,m,n) containing the input precipitation fields\n ordered by timestamp from oldest to newest. The time steps between the\n inputs are assumed to be regular.\n velocity: array-like\n Array of shape (2,m,n) containing the x- and y-components of the advection\n field. The velocities are assumed to represent one time step between the\n inputs. All values are required to be finite.\n timesteps: int or list of floats\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed (relative to the input time step). The elements\n of the list are required to be in ascending order.\n n_ens_members: int, optional\n The number of ensemble members to generate.\n n_cascade_levels: int, optional\n The number of cascade levels to use. Defaults to 6, see issue #385\n on GitHub.\n precip_thr: float, optional\n Specifies the threshold value for minimum observable precipitation\n intensity. Required if mask_method is not None or conditional is True.\n norain_thr: float\n Specifies the threshold value for the fraction of rainy (see above) pixels\n in the radar rainfall field below which we consider there to be no rain.\n Depends on the amount of clutter typically present.\n Standard set to 0.0\n kmperpixel: float, optional\n Spatial resolution of the input data (kilometers/pixel). Required if\n vel_pert_method is not None or mask_method is 'incremental'.\n timestep: float, optional\n Time step of the motion vectors (minutes). Required if vel_pert_method is\n not None or mask_method is 'incremental'.\n extrap_method: str, optional\n Name of the extrapolation method to use. See the documentation of\n pysteps.extrapolation.interface.\n decomp_method: {'fft'}, optional\n Name of the cascade decomposition method to use. See the documentation\n of pysteps.cascade.interface.\n bandpass_filter_method: {'gaussian', 'uniform'}, optional\n Name of the bandpass filter method to use with the cascade decomposition.\n See the documentation of pysteps.cascade.interface.\n noise_method: {'parametric','nonparametric','ssft','nested',None}, optional\n Name of the noise generator to use for perturbating the precipitation\n field. See the documentation of pysteps.noise.interface. If set to None,\n no noise is generated.\n noise_stddev_adj: {'auto','fixed',None}, optional\n Optional adjustment for the standard deviations of the noise fields added\n to each cascade level. This is done to compensate incorrect std. dev.\n estimates of casace levels due to presence of no-rain areas. 'auto'=use\n the method implemented in pysteps.noise.utils.compute_noise_stddev_adjs.\n 'fixed'= use the formula given in :cite:`BPS2006` (eq. 6), None=disable\n noise std. dev adjustment.\n ar_order: int, optional\n The order of the autoregressive model to use. Must be >= 1.\n vel_pert_method: {'bps',None}, optional\n Name of the noise generator to use for perturbing the advection field. See\n the documentation of pysteps.noise.interface. If set to None, the advection\n field is not perturbed.\n conditional: bool, optional\n If set to True, compute the statistics of the precipitation field\n conditionally by excluding pixels where the values are below the\n threshold precip_thr.\n mask_method: {'obs','sprog','incremental',None}, optional\n The method to use for masking no precipitation areas in the forecast\n field. The masked pixels are set to the minimum value of the observations.\n 'obs' = apply precip_thr to the most recently observed precipitation\n intensity field, 'sprog' = use the smoothed forecast field from S-PROG,\n where the AR(p) model has been applied, 'incremental' = iteratively\n buffer the mask with a certain rate (currently it is 1 km/min),\n None=no masking.\n probmatching_method: {'cdf','mean',None}, optional\n Method for matching the statistics of the forecast field with those of\n the most recently observed one. 'cdf'=map the forecast CDF to the observed\n one, 'mean'=adjust only the conditional mean value of the forecast field\n in precipitation areas, None=no matching applied. Using 'mean' requires\n that precip_thr and mask_method are not None.\n seed: int, optional\n Optional seed number for the random generators.\n num_workers: int, optional\n The number of workers to use for parallel computation. Applicable if dask\n is enabled or pyFFTW is used for computing the FFT. When num_workers>1, it\n is advisable to disable OpenMP by setting the environment variable\n OMP_NUM_THREADS to 1. This avoids slowdown caused by too many simultaneous\n threads.\n fft_method: str, optional\n A string defining the FFT method to use (see utils.fft.get_method).\n Defaults to 'numpy' for compatibility reasons. If pyFFTW is installed,\n the recommended method is 'pyfftw'.\n domain: {\"spatial\", \"spectral\"}\n If \"spatial\", all computations are done in the spatial domain (the\n classical STEPS model). If \"spectral\", the AR(2) models and stochastic\n perturbations are applied directly in the spectral domain to reduce\n memory footprint and improve performance :cite:`PCH2019b`.\n extrap_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the extrapolation\n method. See the documentation of pysteps.extrapolation.\n filter_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the filter method.\n See the documentation of pysteps.cascade.bandpass_filters.py.\n noise_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the initializer of\n the noise generator. See the documentation of pysteps.noise.fftgenerators.\n vel_pert_kwargs: dict, optional\n Optional dictionary containing keyword arguments 'p_par' and 'p_perp' for\n the initializer of the velocity perturbator. The choice of the optimal\n parameters depends on the domain and the used optical flow method.\n\n Default parameters from :cite:`BPS2006`:\n p_par = [10.88, 0.23, -7.68]\n p_perp = [5.76, 0.31, -2.72]\n\n Parameters fitted to the data (optical flow/domain):\n\n darts/fmi:\n p_par = [13.71259667, 0.15658963, -16.24368207]\n p_perp = [8.26550355, 0.17820458, -9.54107834]\n\n darts/mch:\n p_par = [24.27562298, 0.11297186, -27.30087471]\n p_perp = [-7.80797846e+01, -3.38641048e-02, 7.56715304e+01]\n\n darts/fmi+mch:\n p_par = [16.55447057, 0.14160448, -19.24613059]\n p_perp = [14.75343395, 0.11785398, -16.26151612]\n\n lucaskanade/fmi:\n p_par = [2.20837526, 0.33887032, -2.48995355]\n p_perp = [2.21722634, 0.32359621, -2.57402761]\n\n lucaskanade/mch:\n p_par = [2.56338484, 0.3330941, -2.99714349]\n p_perp = [1.31204508, 0.3578426, -1.02499891]\n\n lucaskanade/fmi+mch:\n p_par = [2.31970635, 0.33734287, -2.64972861]\n p_perp = [1.90769947, 0.33446594, -2.06603662]\n\n vet/fmi:\n p_par = [0.25337388, 0.67542291, 11.04895538]\n p_perp = [0.02432118, 0.99613295, 7.40146505]\n\n vet/mch:\n p_par = [0.5075159, 0.53895212, 7.90331791]\n p_perp = [0.68025501, 0.41761289, 4.73793581]\n\n vet/fmi+mch:\n p_par = [0.29495222, 0.62429207, 8.6804131 ]\n p_perp = [0.23127377, 0.59010281, 5.98180004]\n\n fmi=Finland, mch=Switzerland, fmi+mch=both pooled into the same data set\n\n The above parameters have been fitten by using run_vel_pert_analysis.py\n and fit_vel_pert_params.py located in the scripts directory.\n\n See pysteps.noise.motion for additional documentation.\n mask_kwargs: dict\n Optional dictionary containing mask keyword arguments 'mask_f' and\n 'mask_rim', the factor defining the the mask increment and the rim size,\n respectively.\n The mask increment is defined as mask_f*timestep/kmperpixel.\n measure_time: bool\n If set to True, measure, print and return the computation time.\n callback: function, optional\n Optional function that is called after computation of each time step of\n the nowcast. The function takes one argument: a three-dimensional array\n of shape (n_ens_members,h,w), where h and w are the height and width\n of the input precipitation fields, respectively. This can be used, for\n instance, writing the outputs into files.\n return_output: bool, optional\n Set to False to disable returning the outputs as numpy arrays. This can\n save memory if the intermediate results are written to output files using\n the callback function.\n\n Returns\n -------\n out: ndarray\n If return_output is True, a four-dimensional array of shape\n (n_ens_members,num_timesteps,m,n) containing a time series of forecast\n precipitation fields for each ensemble member. Otherwise, a None value\n is returned. The time series starts from t0+timestep, where timestep is\n taken from the input precipitation fields. If measure_time is True, the\n return value is a three-element tuple containing the nowcast array, the\n initialization time of the nowcast generator and the time used in the\n main loop (seconds).\n\n See also\n --------\n pysteps.extrapolation.interface, pysteps.cascade.interface,\n pysteps.noise.interface, pysteps.noise.utils.compute_noise_stddev_adjs\n\n References\n ----------\n :cite:`Seed2003`, :cite:`BPS2006`, :cite:`SPN2013`, :cite:`PCH2019b`\n \"\"\"\n\n nowcaster_config = StepsNowcasterConfig(\n n_ens_members=n_ens_members,\n n_cascade_levels=n_cascade_levels,\n precip_threshold=precip_thr,\n norain_threshold=norain_thr,\n kmperpixel=kmperpixel,\n timestep=timestep,\n extrapolation_method=extrap_method,\n decomposition_method=decomp_method,\n bandpass_filter_method=bandpass_filter_method,\n noise_method=noise_method,\n noise_stddev_adj=noise_stddev_adj,\n ar_order=ar_order,\n velocity_perturbation_method=vel_pert_method,\n conditional=conditional,\n probmatching_method=probmatching_method,\n mask_method=mask_method,\n seed=seed,\n num_workers=num_workers,\n fft_method=fft_method,\n domain=domain,\n extrapolation_kwargs=extrap_kwargs,\n filter_kwargs=filter_kwargs,\n noise_kwargs=noise_kwargs,\n velocity_perturbation_kwargs=vel_pert_kwargs,\n mask_kwargs=mask_kwargs,\n measure_time=measure_time,\n callback=callback,\n return_output=return_output,\n )\n\n # Create an instance of the new class with all the provided arguments\n nowcaster = StepsNowcaster(\n precip, velocity, timesteps, steps_config=nowcaster_config\n )\n forecast_steps_nowcast = nowcaster.compute_forecast()\n nowcaster.reset_states_and_params()\n # Call the appropriate methods within the class\n return forecast_steps_nowcast\n" }, { "path": "pysteps/nowcasts/utils.py", "content": "\"\"\"\npysteps.nowcasts.utils\n======================\n\nModule with common utilities used by nowcasts methods.\n\n.. autosummary::\n :toctree: ../generated/\n\n binned_timesteps\n compute_dilated_mask\n compute_percentile_mask\n nowcast_main_loop\n print_ar_params\n print_corrcoefs\n stack_cascades\n\"\"\"\n\nimport time\n\nimport numpy as np\nfrom scipy.ndimage import binary_dilation, generate_binary_structure\n\nfrom pysteps import extrapolation\n\ntry:\n import dask\n\n DASK_IMPORTED = True\nexcept ImportError:\n DASK_IMPORTED = False\n\n\ndef binned_timesteps(timesteps):\n \"\"\"\n Compute a binning of the given irregular time steps.\n\n Parameters\n ----------\n timesteps: array_like\n List or one-dimensional array containing the time steps in ascending\n order.\n\n Returns\n -------\n out: list\n List of length int(np.ceil(timesteps[-1]))+1 containing the bins. Each\n element is a list containing the indices of the time steps falling in\n the bin (excluding the right edge).\n \"\"\"\n timesteps = list(timesteps)\n if not sorted(timesteps) == timesteps:\n raise ValueError(\"timesteps is not in ascending order\")\n\n if np.any(np.array(timesteps) < 0):\n raise ValueError(\"negative time steps are not allowed\")\n\n num_bins = int(np.ceil(timesteps[-1]))\n timestep_range = np.arange(num_bins + 1)\n bin_idx = np.digitize(timesteps, timestep_range, right=False)\n\n out = [[] for _ in range(num_bins + 1)]\n for i, bi in enumerate(bin_idx):\n out[bi - 1].append(i)\n\n return out\n\n\ndef compute_dilated_mask(input_mask, kr, r):\n \"\"\"Buffer the input rain mask using the given kernel. Add a grayscale rim\n for smooth rain/no-rain transition by iteratively dilating the mask.\n\n Parameters\n ----------\n input_mask : array_like\n Two-dimensional boolean array containing the input mask.\n kr : array_like\n Structuring element for the dilation.\n r : int\n The number of iterations for the dilation.\n\n Returns\n -------\n out : array_like\n The dilated mask normalized to the range [0,1].\n \"\"\"\n # buffer the input mask\n input_mask = np.ndarray.astype(input_mask.copy(), \"uint8\")\n mask_dilated = binary_dilation(input_mask, kr)\n\n # add grayscale rim\n kr1 = generate_binary_structure(2, 1)\n mask = mask_dilated.astype(float)\n for _ in range(r):\n mask_dilated = binary_dilation(mask_dilated, kr1)\n mask += mask_dilated\n\n # normalize between 0 and 1\n return mask / mask.max()\n\n\ndef compute_percentile_mask(precip, pct):\n \"\"\"Compute a precipitation mask, where True/False values are assigned for\n pixels above/below the given percentile.\n\n .. _ndarray:\\\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\n\n Parameters\n ----------\n precip: array_like\n Two-dimensional array of shape (m,n) containing the input precipitation\n field.\n pct: float\n The percentile value.\n\n Returns\n -------\n out: ndarray_\n Array of shape (m,n), where True/False values are assigned for pixels\n above/below the precipitation intensity corresponding to the given\n percentile.\n \"\"\"\n # obtain the CDF from the input precipitation field\n precip_s = precip.flatten()\n\n # compute the precipitation intensity threshold corresponding to the given\n # percentile\n precip_s.sort(kind=\"quicksort\")\n x = 1.0 * np.arange(1, len(precip_s) + 1)[::-1] / len(precip_s)\n i = np.argmin(np.abs(x - pct))\n # handle ties\n if precip_s[i] == precip_s[i + 1]:\n i = np.where(precip_s == precip_s[i])[0][-1]\n precip_pct_thr = precip_s[i]\n\n # determine the mask using the above threshold value\n return precip >= precip_pct_thr\n\n\ndef zero_precipitation_forecast(\n n_ens_members,\n timesteps,\n precip,\n callback,\n return_output,\n measure_time,\n start_time_init,\n):\n \"\"\"\n Generate a zero-precipitation forecast (filled with the minimum precip value)\n when no precipitation above the threshold is detected. The forecast is\n optionally returned or passed to a callback.\n\n Parameters\n ----------\n n_ens_members: int, optional\n The number of ensemble members to generate.\n timesteps: int or list of floats\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed (relative to the input time step). The elements\n of the list are required to be in ascending order.\n precip: array-like\n Array of shape (ar_order+1,m,n) containing the input precipitation fields\n ordered by timestamp from oldest to newest. The time steps between the\n inputs are assumed to be regular.\n callback: function, optional\n Optional function that is called after computation of each time step of\n the nowcast. The function takes one argument: a three-dimensional array\n of shape (n_ens_members,h,w), where h and w are the height and width\n of the input precipitation fields, respectively. This can be used, for\n instance, writing the outputs into files.\n return_output: bool, optional\n Set to False to disable returning the outputs as numpy arrays. This can\n save memory if the intermediate results are written to output files using\n the callback function.\n measure_time: bool\n If set to True, measure, print and return the computation time.\n start_time_init: float\n The value of the start time counter used to compute total run time.\n\n Returns\n -------\n out: ndarray\n If return_output is True, a four-dimensional array of shape\n (n_ens_members,num_timesteps,m,n) containing a time series of forecast\n precipitation fields for each ensemble member. Otherwise, a None value\n is returned. The time series starts from t0+timestep, where timestep is\n taken from the input precipitation fields. If measure_time is True, the\n return value is a three-element tuple containing the nowcast array, the\n initialization time of the nowcast generator and the time used in the\n main loop (seconds).\n \"\"\"\n print(\"No precipitation above the threshold found in the radar field\")\n print(\"The resulting forecast will contain only zeros\")\n return_single_member = False\n if n_ens_members is None:\n n_ens_members = 1\n return_single_member = True\n # Create the output list\n precip_forecast = [[] for j in range(n_ens_members)]\n\n # Save per time step to ensure the array does not become too large if\n # no return_output is requested and callback is not None.\n timesteps, _, __ = create_timestep_range(timesteps)\n for t, subtimestep_idx in enumerate(timesteps):\n # If the timestep is not the first one, we need to provide the zero forecast\n if t > 0:\n # Create an empty np array with shape [n_ens_members, rows, cols]\n # and fill it with the minimum value from precip (corresponding to\n # zero precipitation)\n N, M = precip.shape[1:]\n precip_forecast_workers = np.full((n_ens_members, N, M), np.nanmin(precip))\n if subtimestep_idx:\n if callback is not None:\n if precip_forecast_workers.shape[1] > 0:\n callback(precip_forecast_workers.squeeze())\n if return_output:\n for j in range(n_ens_members):\n precip_forecast[j].append(precip_forecast_workers[j])\n precip_forecast_workers = None\n\n if measure_time:\n zero_precip_time = time.time() - start_time_init\n\n if return_output:\n precip_forecast_all_members_all_times = np.stack(\n [np.stack(precip_forecast[j]) for j in range(n_ens_members)]\n )\n if return_single_member:\n precip_forecast_all_members_all_times = (\n precip_forecast_all_members_all_times[0]\n )\n\n if measure_time:\n return (\n precip_forecast_all_members_all_times,\n zero_precip_time,\n zero_precip_time,\n )\n else:\n return precip_forecast_all_members_all_times\n else:\n return None\n\n\ndef create_timestep_range(timesteps):\n \"\"\"\n create a range of time steps\n if an integer time step is given, create a simple range iterator\n otherwise, assing the time steps to integer bins so that each bin\n contains a list of time steps belonging to that bin\n \"\"\"\n if isinstance(timesteps, int):\n timesteps = range(timesteps + 1)\n timestep_type = \"int\"\n original_timesteps = None\n else:\n original_timesteps = [0] + list(timesteps)\n timesteps = binned_timesteps(original_timesteps)\n timestep_type = \"list\"\n return timesteps, original_timesteps, timestep_type\n\n\ndef nowcast_main_loop(\n precip,\n velocity,\n state,\n timesteps,\n extrap_method,\n func,\n extrap_kwargs=None,\n velocity_pert_gen=None,\n params=None,\n ensemble=False,\n num_ensemble_members=1,\n callback=None,\n return_output=True,\n num_workers=1,\n measure_time=False,\n):\n \"\"\"Utility method for advection-based nowcast models that are applied in\n the Lagrangian coordinates. In addition, this method allows the case, where\n one or more components of the model (e.g. an autoregressive process) require\n using regular integer time steps but the user-supplied values are irregular\n or non-integer.\n\n Parameters\n ----------\n precip : array_like\n Array of shape (m,n) containing the most recently observed precipitation\n field.\n velocity : array_like\n Array of shape (2,m,n) containing the x- and y-components of the\n advection field.\n state : object\n The initial state of the nowcast model.\n timesteps : int or list of floats\n Number of time steps to forecast or a list of time steps for which the\n forecasts are computed. The elements of the list are required to be in\n ascending order.\n extrap_method : str, optional\n Name of the extrapolation method to use. See the documentation of\n :py:mod:`pysteps.extrapolation.interface`.\n ensemble : bool\n Set to True to produce a nowcast ensemble.\n num_ensemble_members : int\n Number of ensemble members. Applicable if ensemble is set to True.\n func : function\n A function that takes the current state of the nowcast model and its\n parameters and returns a forecast field and the new state. The shape of\n the forecast field is expected to be (m,n) for a deterministic nowcast\n and (n_ens_members,m,n) for an ensemble.\n extrap_kwargs : dict, optional\n Optional dictionary containing keyword arguments for the extrapolation\n method. See the documentation of pysteps.extrapolation.\n velocity_pert_gen : list, optional\n Optional list of functions that generate velocity perturbations. The\n length of the list is expected to be n_ens_members. The functions\n are expected to take lead time (relative to timestep index) as input\n argument and return a perturbation field of shape (2,m,n).\n params : dict, optional\n Optional dictionary containing keyword arguments for func.\n callback : function, optional\n Optional function that is called after computation of each time step of\n the nowcast. The function takes one argument: the nowcast array. This\n can be used, for instance, writing output files.\n return_output : bool, optional\n Set to False to disable returning the output forecast fields and return\n None instead. This can save memory if the intermediate results are\n instead written to files using the callback function.\n num_workers : int, optional\n Number of parallel workers to use. Applicable if a nowcast ensemble is\n generated.\n measure_time : bool, optional\n If set to True, measure, print and return the computation time.\n\n Returns\n -------\n out : list\n List of forecast fields for the given time steps. If measure_time is\n True, return a pair, where the second element is the total computation\n time in the loop.\n \"\"\"\n precip_forecast_out = None\n\n timesteps, original_timesteps, timestep_type = create_timestep_range(timesteps)\n\n state_cur = state\n if not ensemble:\n precip_forecast_prev = precip[np.newaxis, :]\n else:\n precip_forecast_prev = np.stack([precip for _ in range(num_ensemble_members)])\n displacement = None\n t_prev = 0.0\n t_total = 0.0\n\n # initialize the extrapolator\n extrapolator = extrapolation.get_method(extrap_method)\n\n x_values, y_values = np.meshgrid(\n np.arange(precip.shape[1]), np.arange(precip.shape[0])\n )\n\n xy_coords = np.stack([x_values, y_values])\n\n if extrap_kwargs is None:\n extrap_kwargs = dict()\n else:\n extrap_kwargs = extrap_kwargs.copy()\n extrap_kwargs[\"xy_coords\"] = xy_coords\n extrap_kwargs[\"return_displacement\"] = True\n\n if measure_time:\n starttime_total = time.time()\n\n # loop through the integer time steps or bins if non-integer time steps\n # were given\n for t, subtimestep_idx in enumerate(timesteps):\n if timestep_type == \"list\":\n subtimesteps = [original_timesteps[t_] for t_ in subtimestep_idx]\n else:\n subtimesteps = [t]\n\n if (timestep_type == \"list\" and subtimesteps) or (\n timestep_type == \"int\" and t > 0\n ):\n is_nowcast_time_step = True\n else:\n is_nowcast_time_step = False\n\n # print a message if nowcasts are computed for the current integer time\n # step (this is not necessarily the case, since the current bin might\n # not contain any time steps)\n if is_nowcast_time_step:\n print(\n f\"Computing nowcast for time step {t}... \",\n end=\"\",\n flush=True,\n )\n\n if measure_time:\n starttime = time.time()\n\n # call the function to iterate the integer-timestep part of the model\n # for one time step\n precip_forecast_new, state_new = func(state_cur, params)\n\n if not ensemble:\n precip_forecast_new = precip_forecast_new[np.newaxis, :]\n\n # advect the currect forecast field to the subtimesteps in the current\n # timestep bin and append the results to the output list\n # apply temporal interpolation to the forecasts made between the\n # previous and the next integer time steps\n for t_sub in subtimesteps:\n if t_sub > 0:\n t_diff_prev_int = t_sub - int(t_sub)\n if t_diff_prev_int > 0.0:\n precip_forecast_ip = (\n 1.0 - t_diff_prev_int\n ) * precip_forecast_prev + t_diff_prev_int * precip_forecast_new\n else:\n precip_forecast_ip = precip_forecast_prev\n\n t_diff_prev = t_sub - t_prev\n t_total += t_diff_prev\n\n if displacement is None:\n displacement = [None for _ in range(precip_forecast_ip.shape[0])]\n\n if precip_forecast_out is None and return_output:\n precip_forecast_out = [\n [] for _ in range(precip_forecast_ip.shape[0])\n ]\n\n precip_forecast_out_cur = [\n None for _ in range(precip_forecast_ip.shape[0])\n ]\n\n def worker1(i):\n extrap_kwargs_ = extrap_kwargs.copy()\n extrap_kwargs_[\"displacement_prev\"] = displacement[i]\n extrap_kwargs_[\"allow_nonfinite_values\"] = (\n True if np.any(~np.isfinite(precip_forecast_ip[i])) else False\n )\n\n if velocity_pert_gen is not None:\n velocity_ = velocity + velocity_pert_gen[i](t_total)\n else:\n velocity_ = velocity\n\n precip_forecast_ep, displacement[i] = extrapolator(\n precip_forecast_ip[i],\n velocity_,\n [t_diff_prev],\n **extrap_kwargs_,\n )\n\n precip_forecast_out_cur[i] = precip_forecast_ep[0]\n if return_output:\n precip_forecast_out[i].append(precip_forecast_ep[0])\n\n if DASK_IMPORTED and ensemble and num_ensemble_members > 1:\n res = []\n for i in range(precip_forecast_ip.shape[0]):\n res.append(dask.delayed(worker1)(i))\n dask.compute(*res, num_workers=num_workers)\n else:\n for i in range(precip_forecast_ip.shape[0]):\n worker1(i)\n\n if callback is not None:\n precip_forecast_out_cur = np.stack(precip_forecast_out_cur)\n callback(precip_forecast_out_cur)\n\n precip_forecast_out_cur = None\n t_prev = t_sub\n\n # advect the forecast field by one time step if no subtimesteps in the\n # current interval were found\n if not subtimesteps:\n t_diff_prev = t + 1 - t_prev\n t_total += t_diff_prev\n\n if displacement is None:\n displacement = [None for _ in range(precip_forecast_new.shape[0])]\n\n def worker2(i):\n extrap_kwargs_ = extrap_kwargs.copy()\n extrap_kwargs_[\"displacement_prev\"] = displacement[i]\n\n if velocity_pert_gen is not None:\n velocity_ = velocity + velocity_pert_gen[i](t_total)\n else:\n velocity_ = velocity\n\n _, displacement[i] = extrapolator(\n None,\n velocity_,\n [t_diff_prev],\n **extrap_kwargs_,\n )\n\n if DASK_IMPORTED and ensemble and num_ensemble_members > 1:\n res = []\n for i in range(precip_forecast_new.shape[0]):\n res.append(dask.delayed(worker2)(i))\n dask.compute(*res, num_workers=num_workers)\n else:\n for i in range(precip_forecast_new.shape[0]):\n worker2(i)\n\n t_prev = t + 1\n\n precip_forecast_prev = precip_forecast_new\n state_cur = state_new\n\n if is_nowcast_time_step:\n if measure_time:\n print(f\"{time.time() - starttime:.2f} seconds.\")\n else:\n print(\"done.\")\n\n if return_output:\n precip_forecast_out = np.stack(precip_forecast_out)\n if not ensemble:\n precip_forecast_out = precip_forecast_out[0, :]\n\n if measure_time:\n return precip_forecast_out, time.time() - starttime_total\n else:\n return precip_forecast_out\n\n\ndef print_ar_params(phi):\n \"\"\"\n Print the parameters of an AR(p) model.\n\n Parameters\n ----------\n phi: array_like\n Array of shape (n, p) containing the AR(p) parameters for n cascade\n levels.\n \"\"\"\n print(\"****************************************\")\n print(\"* AR(p) parameters for cascade levels: *\")\n print(\"****************************************\")\n\n n = phi.shape[1]\n\n hline_str = \"---------\"\n for _ in range(n):\n hline_str += \"---------------\"\n\n title_str = \"| Level |\"\n for i in range(n - 1):\n title_str += \" Phi-%d |\" % (i + 1)\n title_str += \" Phi-0 |\"\n\n print(hline_str)\n print(title_str)\n print(hline_str)\n\n fmt_str = \"| %-5d |\"\n for _ in range(n):\n fmt_str += \" %-12.6f |\"\n\n for i in range(phi.shape[0]):\n print(fmt_str % ((i + 1,) + tuple(phi[i, :])))\n print(hline_str)\n\n\ndef print_corrcoefs(gamma):\n \"\"\"\n Print the parameters of an AR(p) model.\n\n Parameters\n ----------\n gamma: array_like\n Array of shape (m, n) containing n correlation coefficients for m cascade\n levels.\n \"\"\"\n print(\"************************************************\")\n print(\"* Correlation coefficients for cascade levels: *\")\n print(\"************************************************\")\n\n m = gamma.shape[0]\n n = gamma.shape[1]\n\n hline_str = \"---------\"\n for _ in range(n):\n hline_str += \"----------------\"\n\n title_str = \"| Level |\"\n for i in range(n):\n title_str += \" Lag-%d |\" % (i + 1)\n\n print(hline_str)\n print(title_str)\n print(hline_str)\n\n fmt_str = \"| %-5d |\"\n for _ in range(n):\n fmt_str += \" %-13.6f |\"\n\n for i in range(m):\n print(fmt_str % ((i + 1,) + tuple(gamma[i, :])))\n print(hline_str)\n\n\ndef stack_cascades(precip_decomp, n_levels, convert_to_full_arrays=False):\n \"\"\"\n Stack the given cascades into a larger array.\n\n Parameters\n ----------\n precip_decomp: list\n List of cascades obtained by calling a method implemented in\n pysteps.cascade.decomposition.\n n_levels: int\n The number of cascade levels.\n\n Returns\n -------\n out: tuple\n A list of three-dimensional arrays containing the stacked cascade levels.\n \"\"\"\n out = []\n\n n_inputs = len(precip_decomp)\n\n for i in range(n_levels):\n precip_cur_level = []\n for j in range(n_inputs):\n precip_cur_input = precip_decomp[j][\"cascade_levels\"][i]\n if precip_decomp[j][\"compact_output\"] and convert_to_full_arrays:\n precip_tmp = np.zeros(\n precip_decomp[j][\"weight_masks\"].shape[1:], dtype=complex\n )\n precip_tmp[precip_decomp[j][\"weight_masks\"][i]] = precip_cur_input\n precip_cur_input = precip_tmp\n precip_cur_level.append(precip_cur_input)\n out.append(np.stack(precip_cur_level))\n\n if not np.any(\n [precip_decomp[i][\"compact_output\"] for i in range(len(precip_decomp))]\n ):\n out = np.stack(out)\n\n return out\n" }, { "path": "pysteps/postprocessing/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Methods for post-processing of forecasts.\"\"\"\n\nfrom . import ensemblestats\nfrom .diagnostics import *\nfrom .interface import *\nfrom .ensemblestats import *\n" }, { "path": "pysteps/postprocessing/diagnostics.py", "content": "\"\"\"\npysteps.postprocessing.diagnostics\n====================\n\nMethods for applying diagnostics postprocessing.\n\nThe methods in this module implement the following interface::\n\n diagnostic_xxx(optional arguments)\n\nwhere **xxx** is the name of the diagnostic to be applied.\n\nAvailable Diagnostics Postprocessors\n------------------------\n\n.. autosummary::\n :toctree: ../generated/\n\n\"\"\"\n\n# Add your diagnostic_ function here AND add this method to the _diagnostics_methods\n# dictionary in postprocessing.interface.py\n" }, { "path": "pysteps/postprocessing/ensemblestats.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.postprocessing.ensemblestats\n====================================\n\nMethods for the computation of ensemble statistics.\n\n.. autosummary::\n :toctree: ../generated/\n\n mean\n excprob\n banddepth\n\"\"\"\n\nimport numpy as np\nfrom scipy.special import comb\n\n\ndef mean(X, ignore_nan=False, X_thr=None):\n \"\"\"\n Compute the mean value from a forecast ensemble field.\n\n Parameters\n ----------\n X: array_like\n Array of shape (k, m, n) containing a k-member ensemble of forecast\n fields of shape (m, n).\n ignore_nan: bool\n If True, ignore nan values.\n X_thr: float\n Optional threshold for computing the ensemble mean.\n Values below **X_thr** are ignored.\n\n Returns\n -------\n out: ndarray\n Array of shape (m, n) containing the ensemble mean.\n \"\"\"\n\n X = np.asanyarray(X)\n X_ndim = X.ndim\n\n if X_ndim > 3 or X_ndim <= 1:\n raise Exception(\n \"Number of dimensions of X should be 2 or 3.\" + \"It was: {}\".format(X_ndim)\n )\n elif X.ndim == 2:\n X = X[None, ...]\n\n if ignore_nan or X_thr is not None:\n if X_thr is not None:\n X = X.copy()\n X[X < X_thr] = np.nan\n\n return np.nanmean(X, axis=0)\n else:\n return np.mean(X, axis=0)\n\n\ndef excprob(X, X_thr, ignore_nan=False):\n \"\"\"\n For a given forecast ensemble field, compute exceedance probabilities\n for the given intensity thresholds.\n\n Parameters\n ----------\n X: array_like\n Array of shape (k, m, n, ...) containing an k-member ensemble of\n forecasts with shape (m, n, ...).\n X_thr: float or a sequence of floats\n Intensity threshold(s) for which the exceedance probabilities are\n computed.\n ignore_nan: bool\n If True, ignore nan values.\n\n Returns\n -------\n out: ndarray\n Array of shape (len(X_thr), m, n) containing the exceedance\n probabilities for the given intensity thresholds.\n If len(X_thr)=1, the first dimension is dropped.\n \"\"\"\n # Checks\n X = np.asanyarray(X)\n X_ndim = X.ndim\n\n if X_ndim < 3:\n raise Exception(\n f\"Number of dimensions of X should be 3 or more. It was: {X_ndim}\"\n )\n\n P = []\n\n if np.isscalar(X_thr):\n X_thr = [X_thr]\n scalar_thr = True\n else:\n scalar_thr = False\n\n for x in X_thr:\n X_ = np.zeros(X.shape)\n\n X_[X >= x] = 1.0\n X_[~np.isfinite(X)] = np.nan\n\n if ignore_nan:\n P.append(np.nanmean(X_, axis=0))\n else:\n P.append(np.mean(X_, axis=0))\n\n if not scalar_thr:\n return np.stack(P)\n else:\n return P[0]\n\n\ndef banddepth(X, thr=None, norm=False):\n \"\"\"\n Compute the modified band depth (Lopez-Pintado and Romo, 2009) for a\n k-member ensemble data set.\n\n Implementation of the exact fast algorithm for computing the modified band\n depth as described in Sun et al (2012).\n\n Parameters\n ----------\n X: array_like\n Array of shape (k, m, ...) representing an ensemble of *k* members\n (i.e., samples) with shape (m, ...).\n thr: float\n Optional threshold for excluding pixels that have no samples equal or\n above the **thr** value.\n\n Returns\n -------\n out: array_like\n Array of shape *k* containing the (normalized) band depth values for\n each ensemble member.\n\n References\n ----------\n Lopez-Pintado, Sara, and Juan Romo. 2009. \"On the Concept of Depth for\n Functional Data.\" Journal of the American Statistical Association\n 104 (486): 718–34. https://doi.org/10.1198/jasa.2009.0108.\n\n Sun, Ying, Marc G. Genton, and Douglas W. Nychka. 2012. \"Exact Fast\n Computation of Band Depth for Large Functional Datasets: How Quickly Can\n One Million Curves Be Ranked?\" Stat 1 (1): 68–74.\n https://doi.org/10.1002/sta4.8.\n \"\"\"\n\n # mask invalid pixels\n if thr is None:\n thr = np.nanmin(X)\n mask = np.logical_and(np.all(np.isfinite(X), axis=0), np.any(X >= thr, axis=0))\n\n n = X.shape[0]\n p = np.sum(mask)\n depth = np.zeros(n)\n\n # assign ranks\n b = np.random.random((n, p))\n order = np.lexsort((b, X[:, mask]), axis=0) # random rank for ties\n rank = order.argsort(axis=0) + 1\n\n # compute band depth\n nabove = n - rank\n nbelow = rank - 1\n match = nabove * nbelow\n nchoose2 = comb(n, 2)\n proportion = np.sum(match, axis=1) / p\n depth = (proportion + n - 1) / nchoose2\n\n # normalize depth between 0 and 1\n if norm:\n depth = (depth - depth.min()) / (depth.max() - depth.min())\n\n return depth\n" }, { "path": "pysteps/postprocessing/interface.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.postprocessing.interface\n====================\n\nInterface for the postprocessing module.\n\nSupport postprocessing types:\n - ensmeblestats\n - diagnostics\n\n.. currentmodule:: pysteps.postprocessing.interface\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nimport importlib\nfrom importlib.metadata import entry_points\n\nfrom pysteps.postprocessing import diagnostics, ensemblestats\nfrom pprint import pprint\nimport warnings\n\n_diagnostics_methods = dict()\n\n_ensemblestats_methods = dict(\n mean=ensemblestats.mean,\n excprob=ensemblestats.excprob,\n banddepth=ensemblestats.banddepth,\n)\n\n\ndef add_postprocessor(\n postprocessors_function_name, _postprocessors, module, attributes\n):\n \"\"\"\n Add the postprocessor to the appropriate _methods dictionary and to the module.\n Parameters\n ----------\n\n postprocessors_function_name: str\n for example, e.g. diagnostic_example1\n _postprocessors: function\n the function to be added\n @param module: the module where the function is added, e.g. 'diagnostics'\n @param attributes: the existing functions in the selected module\n \"\"\"\n # the dictionary where the function is added\n methods_dict = (\n _diagnostics_methods if \"diagnostic\" in module else _ensemblestats_methods\n )\n\n # get funtion name without mo\n short_name = postprocessors_function_name.replace(f\"{module}_\", \"\")\n if short_name not in methods_dict:\n methods_dict[short_name] = _postprocessors\n else:\n warnings.warn(\n f\"The {module} identifier '{short_name}' is already available in \"\n f\"'pysteps.postprocessing.interface_{module}_methods'.\\n\"\n f\"Skipping {module}:{'.'.join(attributes)}\",\n RuntimeWarning,\n )\n\n if hasattr(globals()[module], postprocessors_function_name):\n warnings.warn(\n f\"The {module} function '{short_name}' is already an attribute\"\n f\"of 'pysteps.postprocessing.{module}'.\\n\"\n f\"Skipping {module}:{'.'.join(attributes)}\",\n RuntimeWarning,\n )\n else:\n setattr(globals()[module], postprocessors_function_name, _postprocessors)\n\n\ndef discover_postprocessors():\n \"\"\"\n Search for installed postprocessing plugins in the entrypoint 'pysteps.plugins.postprocessors'\n\n The postprocessors found are added to the appropriate `_methods`\n dictionary in 'pysteps.postprocessing.interface' containing the available postprocessors.\n \"\"\"\n\n # Discover the postprocessors available in the plugins\n for plugintype in [\"diagnostic\", \"ensemblestat\"]:\n for entry_point in entry_points(group=f\"pysteps.plugins.{plugintype}\"):\n _postprocessors = entry_point.load()\n\n postprocessors_function_name = _postprocessors.__name__\n\n if plugintype in entry_point.module:\n add_postprocessor(\n postprocessors_function_name,\n _postprocessors,\n f\"{plugintype}s\",\n entry_point.attr,\n )\n\n\ndef print_postprocessors_info(module_name, interface_methods, module_methods):\n \"\"\"\n Helper function to print the postprocessors available in the module and in the interface.\n\n Parameters\n ----------\n module_name: str\n Name of the module, for example 'pysteps.postprocessing.diagnostics'.\n interface_methods: dict\n Dictionary of the postprocessors declared in the interface, for example _diagnostics_methods.\n module_methods: list\n List of the postprocessors available in the module, for example 'diagnostic_example1'.\n\n \"\"\"\n print(f\"\\npostprocessors available in the {module_name} module\")\n pprint(module_methods)\n\n print(\n \"\\npostprocessors available in the pysteps.postprocessing.get_method interface\"\n )\n pprint([(short_name, f.__name__) for short_name, f in interface_methods.items()])\n\n module_methods_set = set(module_methods)\n interface_methods_set = set(interface_methods.keys())\n\n difference = module_methods_set ^ interface_methods_set\n if len(difference) > 0:\n # print(\"\\nIMPORTANT:\")\n _diff = module_methods_set - interface_methods_set\n if len(_diff) > 0:\n print(\n f\"\\nIMPORTANT:\\nThe following postprocessors are available in {module_name} module but not in the pysteps.postprocessing.get_method interface\"\n )\n pprint(_diff)\n _diff = interface_methods_set - module_methods_set\n if len(_diff) > 0:\n print(\n \"\\nWARNING:\\n\"\n f\"The following postprocessors are available in the pysteps.postprocessing.get_method interface but not in the {module_name} module\"\n )\n pprint(_diff)\n\n\ndef postprocessors_info():\n \"\"\"Print all the available postprocessors.\"\"\"\n\n available_postprocessors = set()\n postprocessors_in_the_interface = set()\n # List the plugins that have been added to the postprocessing.[plugintype] module\n for plugintype in [\"diagnostics\", \"ensemblestats\"]:\n # in the dictionary and found by get_methods() function\n interface_methods = (\n _diagnostics_methods\n if plugintype == \"diagnostics\"\n else _ensemblestats_methods\n )\n # in the pysteps.postprocessing module\n module_name = f\"pysteps.postprocessing.{plugintype}\"\n available_module_methods = [\n attr\n for attr in dir(importlib.import_module(module_name))\n if attr.startswith(plugintype[:-1])\n ]\n # add the pre-existing ensemblestats functions (see _ensemblestats_methods above)\n # that do not follow the convention to start with \"ensemblestat_\" as the plugins\n if \"ensemblestats\" in plugintype:\n available_module_methods += [\n em\n for em in _ensemblestats_methods.keys()\n if not em.startswith(\"ensemblestat_\")\n ]\n print_postprocessors_info(\n module_name, interface_methods, available_module_methods\n )\n available_postprocessors = available_postprocessors.union(\n available_module_methods\n )\n postprocessors_in_the_interface = postprocessors_in_the_interface.union(\n interface_methods.keys()\n )\n\n return available_postprocessors, postprocessors_in_the_interface\n\n\ndef get_method(name, method_type):\n \"\"\"\n Return a callable function for the method corresponding to the given\n name.\n\n Parameters\n ----------\n name: str\n Name of the method. The available options are:\\n\n\n diagnostics:\n [nothing pre-installed]\n\n ensemblestats:\n pre-installed: mean, excprob, banddepth\n\n Additional options might exist if plugins are installed.\n\n method_type: {'diagnostics', 'ensemblestats'}\n Type of the method (see tables above).\n\n \"\"\"\n\n if isinstance(method_type, str):\n method_type = method_type.lower()\n else:\n raise TypeError(\n \"Only strings supported for for the method_type\"\n + \" argument\\n\"\n + \"The available types are: 'diagnostics', 'ensemblestats'\"\n ) from None\n\n if isinstance(name, str):\n name = name.lower()\n else:\n raise TypeError(\n \"Only strings supported for the method's names.\\n\"\n + \"\\nAvailable diagnostics names:\"\n + str(list(_diagnostics_methods.keys()))\n + \"\\nAvailable ensemblestats names:\"\n + str(list(_ensemblestats_methods.keys()))\n ) from None\n\n if method_type == \"diagnostics\":\n methods_dict = _diagnostics_methods\n elif method_type == \"ensemblestats\":\n methods_dict = _ensemblestats_methods\n else:\n raise ValueError(\n \"Unknown method type {}\\n\".format(method_type)\n + \"The available types are: 'diagnostics', 'ensemblestats'\"\n ) from None\n\n try:\n return methods_dict[name]\n except KeyError:\n raise ValueError(\n \"Unknown {} method {}\\n\".format(method_type, name)\n + \"The available methods are:\"\n + str(list(methods_dict.keys()))\n ) from None\n" }, { "path": "pysteps/postprocessing/probmatching.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.postprocessing.probmatching\n===================================\n\nMethods for matching the probability distribution of two data sets.\n\n.. autosummary::\n :toctree: ../generated/\n\n compute_empirical_cdf\n nonparam_match_empirical_cdf\n pmm_init\n pmm_compute\n shift_scale\n resample_distributions\n\"\"\"\n\nimport numpy as np\nfrom scipy import interpolate as sip\nfrom scipy import optimize as sop\n\n\ndef compute_empirical_cdf(bin_edges, hist):\n \"\"\"\n Compute an empirical cumulative distribution function from the given\n histogram.\n\n Parameters\n ----------\n bin_edges: array_like\n Coordinates of left edges of the histogram bins.\n hist: array_like\n Histogram counts for each bin.\n\n Returns\n -------\n out: ndarray\n CDF values corresponding to the bin edges.\n\n \"\"\"\n cdf = []\n xs = 0.0\n\n for x, h in zip(zip(bin_edges[:-1], bin_edges[1:]), hist):\n cdf.append(xs)\n xs += (x[1] - x[0]) * h\n\n cdf.append(xs)\n cdf = np.array(cdf) / xs\n\n return cdf\n\n\ndef nonparam_match_empirical_cdf(initial_array, target_array, ignore_indices=None):\n \"\"\"\n Matches the empirical CDF of the initial array with the empirical CDF\n of a target array. Initial ranks are conserved, but empirical distribution\n matches the target one. Zero-pixels (i.e. pixels having the minimum value)\n in the initial array are conserved.\n\n Parameters\n ----------\n initial_array: array_like\n The initial array whose CDF is to be matched with the target.\n target_array: array_like\n The target array\n ignore_indices: array_like, optional\n Indices of pixels in the initial_array which are to be ignored (not\n rescaled) or an array of booleans with True at the pixel locations to\n be ignored in initial_array and False elsewhere.\n\n\n Returns\n -------\n output_array: ndarray\n The matched array of the same shape as the initial array.\n \"\"\"\n\n if np.all(np.isnan(initial_array)):\n raise ValueError(\"Initial array contains only nans.\")\n if initial_array.size != target_array.size:\n raise ValueError(\n \"dimension mismatch between initial_array and target_array: \"\n f\"initial_array.shape={initial_array.shape}, target_array.shape={target_array.shape}\"\n )\n\n initial_array_copy = np.array(initial_array, dtype=float)\n target_array = np.array(target_array, dtype=float)\n\n # Determine zero in initial array and set nans to zero\n zvalue = np.nanmin(initial_array_copy)\n if ignore_indices is not None:\n initial_array_copy[ignore_indices] = zvalue\n # Check if there are still nans left after setting the values at ignore_indices to zero.\n if np.any(~np.isfinite(initial_array_copy)):\n raise ValueError(\n \"Initial array contains non-finite values outside ignore_indices mask.\"\n )\n\n idxzeros = initial_array_copy == zvalue\n\n # Determine zero of target_array and set nans to zero.\n zvalue_trg = np.nanmin(target_array)\n target_array = np.where(np.isnan(target_array), zvalue_trg, target_array)\n\n # adjust the fraction of rain in target distribution if the number of\n # nonzeros is greater than in the initial array (the lowest values will be set to zero)\n if np.sum(target_array > zvalue_trg) > np.sum(initial_array_copy > zvalue):\n war = np.sum(initial_array_copy > zvalue) / initial_array_copy.size\n p = np.percentile(target_array, 100 * (1 - war))\n target_array[target_array < p] = zvalue_trg\n\n # flatten the arrays without copying them\n arrayshape = initial_array_copy.shape\n target_array = target_array.reshape(-1)\n initial_array_copy = initial_array_copy.reshape(-1)\n\n # rank target values\n order = target_array.argsort()\n ranked = target_array[order]\n\n # rank initial values order\n orderin = initial_array_copy.argsort()\n ranks = np.empty(len(initial_array_copy), int)\n ranks[orderin] = np.arange(len(initial_array_copy))\n\n # get ranked values from target and rearrange with the initial order\n output_array = ranked[ranks]\n\n # reshape to the original array dimensions\n output_array = output_array.reshape(arrayshape)\n\n # read original zeros\n output_array[idxzeros] = zvalue_trg\n\n # Put back the original values outside the nan-mask of the target array.\n if ignore_indices is not None:\n output_array[ignore_indices] = initial_array[ignore_indices]\n return output_array\n\n\n# TODO: A more detailed explanation of the PMM method + references.\ndef pmm_init(bin_edges_1, cdf_1, bin_edges_2, cdf_2):\n \"\"\"\n Initialize a probability matching method (PMM) object from binned\n cumulative distribution functions (CDF).\n\n Parameters\n ----------\n bin_edges_1: array_like\n Coordinates of the left bin edges of the source cdf.\n cdf_1: array_like\n Values of the source CDF at the bin edges.\n bin_edges_2: array_like\n Coordinates of the left bin edges of the target cdf.\n cdf_2: array_like\n Values of the target CDF at the bin edges.\n \"\"\"\n pmm = {}\n\n pmm[\"bin_edges_1\"] = bin_edges_1.copy()\n pmm[\"cdf_1\"] = cdf_1.copy()\n pmm[\"bin_edges_2\"] = bin_edges_2.copy()\n pmm[\"cdf_2\"] = cdf_2.copy()\n pmm[\"cdf_interpolator\"] = sip.interp1d(bin_edges_1, cdf_1, kind=\"linear\")\n\n return pmm\n\n\ndef pmm_compute(pmm, x):\n \"\"\"\n For a given PMM object and x-coordinate, compute the probability matched\n value (i.e. the x-coordinate for which the target CDF has the same value as\n the source CDF).\n\n Parameters\n ----------\n pmm: dict\n A PMM object returned by pmm_init.\n x: float\n The coordinate for which to compute the probability matched value.\n \"\"\"\n mask = np.logical_and(x >= pmm[\"bin_edges_1\"][0], x <= pmm[\"bin_edges_1\"][-1])\n p = pmm[\"cdf_interpolator\"](x[mask])\n\n result = np.ones(len(mask)) * np.nan\n result[mask] = _invfunc(p, pmm[\"bin_edges_2\"], pmm[\"cdf_2\"])\n\n return result\n\n\ndef shift_scale(R, f, rain_fraction_trg, second_moment_trg, **kwargs):\n \"\"\"\n Find shift and scale that is needed to return the required second_moment\n and rain area. The optimization is performed with the Nelder-Mead algorithm\n available in scipy.\n It assumes a forward transformation ln_rain = ln(rain)-ln(min_rain) if\n rain > min_rain, else 0.\n\n Parameters\n ----------\n R: array_like\n The initial array to be shift and scaled.\n f: function\n The inverse transformation that is applied after the shift and scale.\n rain_fraction_trg: float\n The required rain fraction to be matched by shifting.\n second_moment_trg: float\n The required second moment to be matched by scaling.\n The second_moment is defined as second_moment = var + mean^2.\n\n Other Parameters\n ----------------\n scale: float\n Optional initial value of the scale parameter for the Nelder-Mead\n optimisation.\n Typically, this would be the scale parameter estimated the previous\n time step.\n Default: 1.\n max_iterations: int\n Maximum allowed number of iterations and function evaluations.\n More details: https://docs.scipy.org/doc/scipy/reference/optimize.minimize-neldermead.html\n Deafult: 100.\n tol: float\n Tolerance for termination.\n More details: https://docs.scipy.org/doc/scipy/reference/optimize.minimize-neldermead.html\n Default: 0.05*second_moment_trg, i.e. terminate the search if the error\n is less than 5% since the second moment is a bit unstable.\n\n Returns\n -------\n shift: float\n The shift value that produces the required rain fraction.\n scale: float\n The scale value that produces the required second_moment.\n R: array_like\n The shifted, scaled and back-transformed array.\n \"\"\"\n\n shape = R.shape\n R = R.flatten()\n\n # defaults\n scale = kwargs.get(\"scale\", 1.0)\n max_iterations = kwargs.get(\"max_iterations\", 100)\n tol = kwargs.get(\"tol\", 0.05 * second_moment_trg)\n\n # calculate the shift parameter based on the required rain fraction\n shift = np.percentile(R, 100 * (1 - rain_fraction_trg))\n idx_wet = R > shift\n\n # define objective function\n def _get_error(scale):\n R_ = np.zeros_like(R)\n R_[idx_wet] = f((R[idx_wet] - shift) * scale)\n R_[~idx_wet] = 0\n second_moment = np.nanstd(R_) ** 2 + np.nanmean(R_) ** 2\n return np.abs(second_moment - second_moment_trg)\n\n # Nelder-Mead optimisation\n nm_scale = sop.minimize(\n _get_error,\n scale,\n method=\"Nelder-Mead\",\n tol=tol,\n options={\"disp\": False, \"maxiter\": max_iterations},\n )\n scale = nm_scale[\"x\"][0]\n\n R[idx_wet] = f((R[idx_wet] - shift) * scale)\n R[~idx_wet] = 0\n\n return shift, scale, R.reshape(shape)\n\n\ndef resample_distributions(\n first_array, second_array, probability_first_array, randgen=np.random\n):\n \"\"\"\n Merges two distributions (e.g., from the extrapolation nowcast and NWP in the blending module)\n to effectively combine two distributions for probability matching without losing extremes.\n Entries for which one array has a nan will not be included from the other array either.\n\n Parameters\n ----------\n first_array: array_like\n One of the two arrays from which the distribution should be sampled (e.g., the extrapolation\n cascade). It must be of the same shape as `second_array`. Input must not contain NaNs.\n second_array: array_like\n One of the two arrays from which the distribution should be sampled (e.g., the NWP (model)\n cascade). It must be of the same shape as `first_array`. Input must not contain NaNs.\n probability_first_array: float\n The weight that `first_array` should get (a value between 0 and 1). This determines the\n likelihood of selecting elements from `first_array` over `second_array`.\n randgen: numpy.random or numpy.RandomState\n The random number generator to be used for the binomial distribution. You can pass a seeded\n random state here for reproducibility. Default is numpy.random.\n\n Returns\n -------\n csort: array_like\n The combined output distribution. This is an array of the same shape as the input arrays,\n where each element is chosen from either `first_array` or `second_array` based on the specified\n probability, and then sorted in descending order.\n\n Raises\n ------\n ValueError\n If `first_array` and `second_array` do not have the same shape.\n \"\"\"\n\n # Valide inputs\n if first_array.shape != second_array.shape:\n raise ValueError(\"first_array and second_array must have the same shape\")\n probability_first_array = np.clip(probability_first_array, 0.0, 1.0)\n\n # Propagate the NaN values of the arrays to each other if there are any; convert to float to make sure this works.\n nanmask = np.isnan(first_array) | np.isnan(second_array)\n if np.any(nanmask):\n first_array = first_array.astype(float)\n first_array[nanmask] = np.nan\n second_array = second_array.astype(float)\n second_array[nanmask] = np.nan\n\n # Flatten and sort the arrays\n asort = np.sort(first_array, axis=None)[::-1]\n bsort = np.sort(second_array, axis=None)[::-1]\n n = asort.shape[0]\n\n # Resample the distributions\n idxsamples = randgen.binomial(1, probability_first_array, n).astype(bool)\n csort = np.where(idxsamples, asort, bsort)\n csort = np.sort(csort)[::-1]\n\n # Return the resampled array in descending order (starting with the nan values)\n return csort\n\n\ndef _invfunc(y, fx, fy):\n if len(y) == 0:\n return np.array([])\n\n b = np.digitize(y, fy)\n mask = np.logical_and(b > 0, b < len(fy))\n c = (y[mask] - fy[b[mask] - 1]) / (fy[b[mask]] - fy[b[mask] - 1])\n\n result = np.ones(len(y)) * np.nan\n result[mask] = c * fx[b[mask]] + (1.0 - c) * fx[b[mask] - 1]\n\n return result\n" }, { "path": "pysteps/pystepsrc", "content": "// pysteps configuration\n{\n // \"silent_import\" : whether to suppress the initial pysteps message\n \"silent_import\": false,\n \"outputs\": {\n // path_outputs : path where to save results (figures, forecasts, etc)\n \"path_outputs\": \"./\"\n },\n \"plot\": {\n // \"motion_plot\" : \"streamplot\" or \"quiver\"\n \"motion_plot\": \"quiver\",\n // \"colorscale\" : \"BOM-RF3\", \"pysteps\" or \"STEPS-BE\"\n \"colorscale\": \"pysteps\"\n },\n \"data_sources\": {\n \"bom\": {\n \"root_path\": \"./radar/bom\",\n \"path_fmt\": \"prcp-cscn/2/%Y/%m/%d\",\n \"fn_pattern\": \"2_%Y%m%d_%H%M00.prcp-cscn\",\n \"fn_ext\": \"nc\",\n \"importer\": \"bom_rf3\",\n \"timestep\": 6,\n \"importer_kwargs\": {\n \"gzipped\": true\n }\n },\n \"dwd\": {\n \"root_path\": \"./radar/dwd/RY\",\n \"path_fmt\": \"%Y/%m/%d\",\n \"fn_pattern\": \"%Y%m%d_%H%M_RY\",\n \"fn_ext\": \"h5\",\n \"importer\": \"dwd_hdf5\",\n \"timestep\": 5,\n \"importer_kwargs\": {\n \"qty\": \"RATE\"\n }\n },\n \"fmi\": {\n \"root_path\": \"./radar/fmi/pgm\",\n \"path_fmt\": \"%Y%m%d\",\n \"fn_pattern\": \"%Y%m%d%H%M_fmi.radar.composite.lowest_FIN_SUOMI1\",\n \"fn_ext\": \"pgm.gz\",\n \"importer\": \"fmi_pgm\",\n \"timestep\": 5,\n \"importer_kwargs\": {\n \"gzipped\": true\n }\n },\n \"fmi_geotiff\": {\n \"root_path\": \"./radar/fmi/geotiff\",\n \"path_fmt\": \"%Y%m%d\",\n \"fn_pattern\": \"%Y%m%d%H%M_FINUTM.tif\",\n \"fn_ext\": \"tif\",\n \"importer\": \"geotiff\",\n \"timestep\": 5,\n \"importer_kwargs\": {}\n },\n \"mch\": {\n \"root_path\": \"./radar/mch\",\n \"path_fmt\": \"%Y%m%d\",\n \"fn_pattern\": \"AQC%y%j%H%M?_00005.801\",\n \"fn_ext\": \"gif\",\n \"importer\": \"mch_gif\",\n \"timestep\": 5,\n \"importer_kwargs\": {\n \"product\": \"AQC\",\n \"unit\": \"mm\",\n \"accutime\": 5\n }\n },\n \"mrms\": {\n \"root_path\": \"./mrms\",\n \"path_fmt\": \"%Y/%m/%d\",\n \"fn_pattern\": \"PrecipRate_00.00_%Y%m%d-%H%M%S\",\n \"fn_ext\": \"grib2\",\n \"importer\": \"mrms_grib\",\n \"timestep\": 2,\n \"importer_kwargs\": {}\n },\n \"opera\": {\n \"root_path\": \"./radar/OPERA\",\n \"path_fmt\": \"%Y%m%d\",\n \"fn_pattern\": \"T_PAAH21_C_EUOC_%Y%m%d%H%M%S\",\n \"fn_ext\": \"hdf\",\n \"importer\": \"opera_hdf5\",\n \"timestep\": 15,\n \"importer_kwargs\": {}\n },\n \"knmi\": {\n \"root_path\": \"./radar/KNMI\",\n \"path_fmt\": \"%Y/%m\",\n \"fn_pattern\": \"RAD_NL25_RAP_5min_%Y%m%d%H%M\",\n \"fn_ext\": \"h5\",\n \"importer\": \"knmi_hdf5\",\n \"timestep\": 5,\n \"importer_kwargs\": {\n \"accutime\": 5,\n \"qty\": \"ACRR\",\n \"pixelsize\": 1000.0\n\t\t\t}\n },\n\t\t\"rmi\": {\n \"root_path\": \"./radar/rmi/radqpe\",\n \"path_fmt\": \"%Y%m%d\",\n \"fn_pattern\": \"%Y%m%d%H%M00.rad.best.comp.rate.qpe\",\n \"fn_ext\": \"hdf\",\n \"importer\": \"odim_hdf5\",\n \"timestep\": 5,\n \"importer_kwargs\": {\n\t\t\t\t\"accutime\": 5.0\n }\n },\n \"saf\": {\n \"root_path\": \"./saf\",\n \"path_fmt\": \"%Y%m%d/CRR\",\n \"fn_pattern\": \"S_NWC_CRR_MSG4_Europe-VISIR_%Y%m%dT%H%M00Z\",\n \"fn_ext\": \"nc\",\n \"importer\": \"saf_crri\",\n \"timestep\": 15,\n \"importer_kwargs\": {\n \"gzipped\": true\n }\n },\n \"bom_nwp\": {\n \"root_path\": \"./nwp/bom\",\n \"path_fmt\": \"%Y/%m/%d\",\n \"fn_pattern\": \"%Y%m%d_%H00_regrid_short\",\n \"fn_ext\": \"nc\",\n \"importer\": \"bom_nwp\",\n \"timestep\": 10,\n \"importer_kwargs\": {\n \"gzipped\": true\n }\n },\n \"dwd_nwp\": {\n \"root_path\": \"./nwp/dwd\",\n \"path_fmt\": \"%Y/%m/%d\",\n \"fn_ext\": \"grib2\",\n \"fn_pattern\": \"%Y%m%d_%H%M_PR_GSP_060_120\",\n \"importer\": \"dwd_nwp\",\n \"timestep\": 5,\n \"importer_kwargs\": {\n \"varname\": \"lsprate\",\n \"grid_file_path\": \"./aux/grid_files/dwd/icon/R19B07/icon_grid_0047_R19B07_L.nc\"\n }\n \n },\n \"knmi_nwp\": {\n \"root_path\": \"./nwp/knmi\",\n \"path_fmt\": \"%Y/%m/%d\",\n \"fn_pattern\": \"%Y%m%d_%H00_Pforecast_Harmonie\",\n \"fn_ext\": \"nc\",\n \"importer\": \"knmi_nwp\",\n \"timestep\": 60,\n \"importer_kwargs\": {\n \"gzipped\": true\n }\n },\n \"rmi_nwp\": {\n \"root_path\": \"./nwp/rmi\",\n \"path_fmt\": \"%Y/%m/%d\",\n \"fn_pattern\": \"ao13_%Y%m%d%H_native_5min\",\n \"fn_ext\": \"nc\",\n \"importer\": \"rmi_nwp\",\n \"timestep\": 5,\n \"importer_kwargs\": {\n \"gzipped\": true\n }\n }\n }\n}\n" }, { "path": "pysteps/pystepsrc_schema.json", "content": "{\n \"title\": \"pystepsrc params\",\n \"description\": \"Pysteps default parameters\",\n \"required\": [\n \"outputs\",\n \"plot\",\n \"data_sources\"\n ],\n \"type\": \"object\",\n \"properties\": {\n \"outputs\": {\n \"type\": \"object\",\n \"required\": [\n \"path_outputs\"\n ],\n \"properties\": {\n \"path_outputs\": {\n \"type\": \"string\"\n }\n }\n },\n \"plot\": {\n \"type\": \"object\",\n \"required\": [\n \"motion_plot\",\n \"colorscale\"\n ],\n \"properties\": {\n \"motion_plot\": {\n \"type\": \"string\"\n },\n \"colorscale\": {\n \"type\": \"string\"\n }\n }\n },\n \"data_sources\": {\n \"type\": \"object\",\n \"patternProperties\": {\n \"\": {\n \"type\": \"object\",\n \"required\": [\n \"root_path\",\n \"path_fmt\",\n \"fn_pattern\",\n \"fn_ext\",\n \"importer\",\n \"timestep\",\n \"importer_kwargs\"\n ],\n \"properties\": {\n \"root_path\": {\n \"type\": \"string\"\n },\n \"path_fmt\": {\n \"type\": \"string\"\n },\n \"fn_pattern\": {\n \"type\": \"string\"\n },\n \"fn_ext\": {\n \"type\": \"string\"\n },\n \"importer\": {\n \"type\": \"string\"\n },\n \"timestep\": {\n \"type\": \"number\"\n },\n \"importer_kwargs\": {\n \"type\": \"object\"\n }\n }\n }\n }\n }\n }\n}\n" }, { "path": "pysteps/scripts/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nStandalone utility scripts for pysteps (e.g. parameter estimation from the\ngiven data).\n\"\"\"\n" }, { "path": "pysteps/scripts/fit_vel_pert_params.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Fit STEPS motion perturbation parameters to the output of run_vel_pert_analysis.py\nand optionally plots the results. For a description of the method, see\n:cite:`BPS2006`.\"\"\"\n\nimport argparse\nimport pickle\nfrom matplotlib import pyplot\nimport numpy as np\nfrom scipy.optimize import curve_fit\n\ndescription = (\n \"Fit STEPS motion perturbation parameters to the results produced\"\n \" by run_vel_pert_analysis.py and optionally plot the results.\"\n)\n\nargparser = argparse.ArgumentParser(description=description)\nargparser.add_argument(\"inputfile\", type=str, help=\"name of the input file\")\nargparser.add_argument(\n \"--plot\",\n nargs=\"?\",\n type=str,\n metavar=\"filename\",\n help=\"plot the results and save the figure to \",\n)\nargs = argparser.parse_args()\n\nwith open(args.inputfile, \"rb\") as f:\n results = pickle.load(f)\n\nf = lambda t, a, b, c: a * pow(t, b) + c\n\nleadtimes = sorted(results.keys())\n\nstd_par = []\nstd_perp = []\n\nfor lt in leadtimes:\n dp_par_sum = results[lt][\"dp_par_sum\"]\n dp_par_sq_sum = results[lt][\"dp_par_sq_sum\"]\n dp_par_n = results[lt][\"n_samples\"]\n mu = dp_par_sum / dp_par_n\n\n std_par.append(\n np.sqrt((dp_par_sq_sum - 2 * mu * dp_par_sum + dp_par_n * mu**2) / dp_par_n)\n )\n\n dp_perp_sum = results[lt][\"dp_perp_sum\"]\n dp_perp_sq_sum = results[lt][\"dp_perp_sq_sum\"]\n dp_perp_n = results[lt][\"n_samples\"]\n mu = dp_perp_sum / dp_perp_n\n\n std_perp.append(\n np.sqrt((dp_perp_sq_sum - 2 * mu * dp_perp_sum + dp_perp_n * mu**2) / dp_perp_n)\n )\n\ntry:\n p_par = curve_fit(f, leadtimes, std_par)[0]\n p_perp = curve_fit(f, leadtimes, std_perp)[0]\n fit_succeeded = True\n print(\"p_par = %s\" % str(p_par))\n print(\"p_perp = %s\" % str(p_perp))\nexcept RuntimeError:\n fit_succeeded = False\n print(\"Parameter fitting failed.\")\n\nif args.plot is not None:\n pyplot.figure()\n\n pyplot.scatter(leadtimes, std_par, c=\"r\")\n t = np.linspace(0.5 * leadtimes[0], 1.025 * leadtimes[-1], 200)\n pyplot.scatter(leadtimes, std_perp, c=\"g\")\n if fit_succeeded:\n (l1,) = pyplot.plot(t, f(t, *p_par), \"r-\")\n (l2,) = pyplot.plot(t, f(t, *p_perp), \"g-\")\n\n p_str_1 = lambda p: \"%.2f\\\\cdot t^{%.2f}+%.2f\" % (p[0], p[1], p[2])\n p_str_2 = lambda p: \"%.2f\\\\cdot t^{%.2f}%.2f\" % (p[0], p[1], p[2])\n if fit_succeeded:\n lbl = lambda p: p_str_1(p) if p[2] > 0.0 else p_str_2(p)\n pyplot.legend(\n [l1, l2],\n [\n \"Parallel: $f(t)=%s$\" % lbl(p_par),\n \"Perpendicular: $f(t)=%s$\" % lbl(p_perp),\n ],\n fontsize=12,\n )\n pyplot.xlim(0.5 * leadtimes[0], 1.025 * leadtimes[-1])\n pyplot.xlabel(\"Lead time (minutes)\", fontsize=12)\n pyplot.ylabel(\"Standard deviation of differences (km/h)\", fontsize=12)\n pyplot.grid(True)\n\n pyplot.savefig(args.plot, bbox_inches=\"tight\")\n" }, { "path": "pysteps/scripts/run_vel_pert_analysis.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Analyze uncertainty of motion field with increasing lead time. The analyses\nare done by comparing initial motion fields to those estimated in the future.\nFor a description of the method, see :cite:`BPS2006`.\"\"\"\n\nimport argparse\nfrom datetime import datetime, timedelta\nimport pickle\nimport numpy as np\nfrom scipy import linalg as la\nfrom pysteps import io, motion\nfrom pysteps import rcparams\nfrom pysteps.utils import transformation\n\n# TODO: Don't hard-code these.\nnum_prev_files = 9\nuse_precip_mask = False\nR_min = 0.1\n\nargparser = argparse.ArgumentParser(\n description=\"Estimate motion perturbation parameters for STEPS.\"\n)\nargparser.add_argument(\"startdate\", type=str, help=\"start date (YYYYmmDDHHMM)\")\nargparser.add_argument(\"enddate\", type=str, help=\"end date (YYYYmmDDHHMM)\")\nargparser.add_argument(\"datasource\", type=str, help=\"data source to use\")\nargparser.add_argument(\n \"oflow\", type=str, help=\"optical flow method to use (darts, lucaskanade or vet)\"\n)\nargparser.add_argument(\n \"maxleadtime\", type=int, help=\"maximum lead time for the analyses (minutes)\"\n)\nargparser.add_argument(\"outfile\", type=str, help=\"output file name\")\nargparser.add_argument(\n \"--accum\",\n nargs=\"?\",\n type=str,\n metavar=\"filename\",\n help=\"accumulate statistics to previously computed file \",\n)\nargs = argparser.parse_args()\n\ndatasource = rcparams[\"data_sources\"][args.datasource]\n\nstartdate = datetime.strptime(args.startdate, \"%Y%m%d%H%M\")\nenddate = datetime.strptime(args.enddate, \"%Y%m%d%H%M\")\n\nimporter = io.get_method(datasource[\"importer\"], \"importer\")\n\nmotionfields = {}\n\noflow = motion.get_method(args.oflow)\n\n# compute motion fields\n# ---------------------\n\n# TODO: This keeps all motion fields in memory during the analysis period, which\n# can take a lot of memory.\n\ncurdate = startdate\nwhile curdate <= enddate:\n try:\n fns = io.archive.find_by_date(\n curdate,\n datasource[\"root_path\"],\n datasource[\"path_fmt\"],\n datasource[\"fn_pattern\"],\n datasource[\"fn_ext\"],\n datasource[\"timestep\"],\n num_prev_files=9,\n )\n except IOError:\n curdate += timedelta(minutes=datasource[\"timestep\"])\n continue\n\n if any([fn[0] is None for fn in fns]):\n curdate += timedelta(minutes=datasource[\"timestep\"])\n continue\n\n R, _, metadata = io.readers.read_timeseries(\n fns, importer, **datasource[\"importer_kwargs\"]\n )\n\n # TODO: Here we assume that metadata[\"xpixelsize\"] = metadata[\"ypixelsize\"]\n vsf = 60.0 / datasource[\"timestep\"] * metadata[\"xpixelsize\"] / 1000.0\n\n missing_data = False\n for i in range(R.shape[0]):\n if not np.any(np.isfinite(R[i, :, :])):\n missing_data = True\n break\n\n if missing_data:\n curdate += timedelta(minutes=datasource[\"timestep\"])\n continue\n\n R[~np.isfinite(R)] = metadata[\"zerovalue\"]\n if use_precip_mask:\n MASK = np.any(R < R_min, axis=0)\n R = transformation.dB_transform(R)[0]\n\n if args.oflow == \"vet\":\n R_ = R[-2:, :, :]\n else:\n R_ = R\n\n # TODO: Allow the user to supply parameters for the optical flow.\n V = oflow(R_) * vsf\n # discard the motion field if the mean velocity is abnormally large\n if np.nanmean(np.linalg.norm(V, axis=0)) > 0.5 * R.shape[1]:\n curdate += timedelta(minutes=datasource[\"timestep\"])\n continue\n\n if use_precip_mask:\n V[0, :, :][MASK] = np.nan\n V[1, :, :][MASK] = np.nan\n motionfields[curdate] = V.astype(np.float32)\n\n curdate += timedelta(minutes=datasource[\"timestep\"])\n\n# compare initial and future motion fields\n# ----------------------------------------\n\ndates = sorted(motionfields.keys())\nif args.accum is None:\n results = {}\nelse:\n with open(args.accum, \"rb\") as f:\n results = pickle.load(f)\n\nfor i, date1 in enumerate(dates):\n V1 = motionfields[date1].astype(float)\n if not use_precip_mask:\n N = la.norm(V1, axis=0)\n else:\n N = np.ones(V1.shape[1:]) * np.nan\n MASK = np.isfinite(V1[0, :, :])\n N[MASK] = la.norm(V1[:, MASK], axis=0)\n V1_par = V1 / N\n V1_perp = np.stack([-V1_par[1, :, :], V1_par[0, :, :]])\n\n if date1 + timedelta(minutes=args.maxleadtime) > enddate:\n continue\n\n for date2 in dates[i + 1 :]:\n lt = (date2 - date1).total_seconds() / 60\n if lt > args.maxleadtime:\n continue\n\n V2 = motionfields[date2].astype(float)\n\n DV = V2 - V1\n\n DP_par = DV[0, :, :] * V1_par[0, :, :] + DV[1, :, :] * V1_par[1, :, :]\n DP_perp = DV[0, :, :] * V1_perp[0, :, :] + DV[1, :, :] * V1_perp[1, :, :]\n\n if not lt in results.keys():\n results[lt] = {}\n results[lt][\"dp_par_sum\"] = 0.0\n results[lt][\"dp_par_sq_sum\"] = 0.0\n results[lt][\"dp_perp_sum\"] = 0.0\n results[lt][\"dp_perp_sq_sum\"] = 0.0\n results[lt][\"n_samples\"] = 0\n\n if use_precip_mask:\n MASK = np.logical_and(np.isfinite(V1[0, :, :]), np.isfinite(V2[0, :, :]))\n DP_par = DP_par[MASK]\n DP_perp = DP_perp[MASK]\n n_samples = np.sum(MASK)\n else:\n n_samples = DP_par.size\n\n results[lt][\"dp_par_sum\"] += np.sum(DP_par)\n results[lt][\"dp_par_sq_sum\"] += np.sum(DP_par**2)\n results[lt][\"dp_perp_sum\"] += np.sum(DP_perp)\n results[lt][\"dp_perp_sq_sum\"] += np.sum(DP_perp**2)\n results[lt][\"n_samples\"] += n_samples\n\nwith open(\"%s\" % args.outfile, \"wb\") as f:\n pickle.dump(results, f)\n" }, { "path": "pysteps/tests/__init__.py", "content": "" }, { "path": "pysteps/tests/helpers.py", "content": "\"\"\"\nTesting helper functions\n=======================\n\nCollection of helper functions for the testing suite.\n\"\"\"\n\nfrom datetime import datetime\n\nimport numpy as np\nimport pytest\n\nimport pysteps as stp\nfrom pysteps import io, rcparams\nfrom pysteps.utils import aggregate_fields_space\n\n_reference_dates = dict()\n_reference_dates[\"bom\"] = datetime(2018, 6, 16, 10, 0)\n_reference_dates[\"fmi\"] = datetime(2016, 9, 28, 16, 0)\n_reference_dates[\"knmi\"] = datetime(2010, 8, 26, 0, 0)\n_reference_dates[\"mch\"] = datetime(2015, 5, 15, 16, 30)\n_reference_dates[\"dwd\"] = datetime(2025, 6, 4, 17, 0)\n_reference_dates[\"opera\"] = datetime(2018, 8, 24, 18, 0)\n_reference_dates[\"saf\"] = datetime(2018, 6, 1, 7, 0)\n_reference_dates[\"mrms\"] = datetime(2019, 6, 10, 0, 0)\n\n\ndef get_precipitation_fields(\n num_prev_files=0,\n num_next_files=0,\n return_raw=False,\n metadata=False,\n upscale=None,\n source=\"mch\",\n log_transform=True,\n clip=None,\n **importer_kwargs,\n):\n \"\"\"\n Get a precipitation field from the archive to be used as reference.\n\n Source: bom\n Reference time: 2018/06/16 10000 UTC\n\n Source: fmi\n Reference time: 2016/09/28 1600 UTC\n\n Source: knmi\n Reference time: 2010/08/26 0000 UTC\n\n Source: mch\n Reference time: 2015/05/15 1630 UTC\n\n Source: dwd\n Reference time: 2025/06/04 1700 UTC\n\n Source: opera\n Reference time: 2018/08/24 1800 UTC\n\n Source: saf\n Reference time: 2018/06/01 0700 UTC\n\n Source: mrms\n Reference time: 2019/06/10 0000 UTC\n\n Parameters\n ----------\n\n num_prev_files: int, optional\n Number of previous times (files) to return with respect to the\n reference time.\n\n num_next_files: int, optional\n Number of future times (files) to return with respect to the\n reference time.\n\n return_raw: bool, optional\n Do not preprocess the precipitation fields. False by default.\n The pre-processing steps are: 1) Convert to mm/h,\n 2) Mask invalid values, 3) Log-transform the data [dBR].\n\n metadata: bool, optional\n If True, also return file metadata.\n\n upscale: float or None, optional\n Upscale fields in space during the pre-processing steps.\n If it is None, the precipitation field is not modified.\n If it is a float, represents the length of the space window that is\n used to upscale the fields.\n\n source: {\"bom\", \"fmi\" , \"knmi\", \"mch\", \"opera\", \"saf\", \"mrms\"}, optional\n Name of the data source to be used.\n\n log_transform: bool\n Whether to transform the output to dB.\n\n clip: scalars (left, right, bottom, top), optional\n The extent of the bounding box in data coordinates to be used to clip\n the data.\n\n Other Parameters\n ----------------\n\n importer_kwargs : dict\n Additional keyword arguments passed to the importer.\n\n Returns\n -------\n reference_field : array\n metadata : dict\n \"\"\"\n\n if source == \"bom\":\n pytest.importorskip(\"netCDF4\")\n\n if source == \"fmi\":\n pytest.importorskip(\"pyproj\")\n\n if source == \"knmi\":\n pytest.importorskip(\"h5py\")\n\n if source == \"mch\":\n pytest.importorskip(\"PIL\")\n\n if source == \"dwd\":\n pytest.importorskip(\"h5py\")\n\n if source == \"opera\":\n pytest.importorskip(\"h5py\")\n\n if source == \"saf\":\n pytest.importorskip(\"netCDF4\")\n\n if source == \"mrms\":\n pytest.importorskip(\"pygrib\")\n\n try:\n date = _reference_dates[source]\n except KeyError:\n raise ValueError(\n f\"Unknown source name '{source}'\\n\"\n \"The available data sources are: \"\n f\"{str(list(_reference_dates.keys()))}\"\n )\n\n data_source = rcparams.data_sources[source]\n root_path = data_source[\"root_path\"]\n path_fmt = data_source[\"path_fmt\"]\n fn_pattern = data_source[\"fn_pattern\"]\n fn_ext = data_source[\"fn_ext\"]\n importer_name = data_source[\"importer\"]\n _importer_kwargs = data_source[\"importer_kwargs\"].copy()\n _importer_kwargs.update(**importer_kwargs)\n timestep = data_source[\"timestep\"]\n\n # Find the input files from the archive\n fns = io.archive.find_by_date(\n date,\n root_path,\n path_fmt,\n fn_pattern,\n fn_ext,\n timestep=timestep,\n num_prev_files=num_prev_files,\n num_next_files=num_next_files,\n )\n\n # Read the radar composites\n importer = io.get_method(importer_name, \"importer\")\n\n reference_field, __, ref_metadata = io.read_timeseries(\n fns, importer, **_importer_kwargs\n )\n\n if not return_raw:\n if (num_prev_files == 0) and (num_next_files == 0):\n # Remove time dimension\n reference_field = np.squeeze(reference_field)\n\n # Convert to mm/h\n reference_field, ref_metadata = stp.utils.to_rainrate(\n reference_field, ref_metadata\n )\n\n # Clip domain\n reference_field, ref_metadata = stp.utils.clip_domain(\n reference_field, ref_metadata, clip\n )\n\n # Upscale data\n reference_field, ref_metadata = aggregate_fields_space(\n reference_field, ref_metadata, upscale\n )\n\n # Mask invalid values\n reference_field = np.ma.masked_invalid(reference_field)\n\n if log_transform:\n # Log-transform the data [dBR]\n reference_field, ref_metadata = stp.utils.dB_transform(\n reference_field, ref_metadata, threshold=0.1, zerovalue=-15.0\n )\n\n # Set missing values with the fill value\n np.ma.set_fill_value(reference_field, ref_metadata[\"zerovalue\"])\n reference_field.data[reference_field.mask] = ref_metadata[\"zerovalue\"]\n\n if metadata:\n return reference_field, ref_metadata\n\n return reference_field\n\n\ndef smart_assert(actual_value, expected, tolerance=None):\n \"\"\"\n Assert by equality for non-numeric values, or by approximation otherwise.\n\n If the precision keyword is None, assert by equality.\n When the precision is not None, assert that two numeric values\n (or two sets of numbers) are equal to each other within the tolerance.\n \"\"\"\n\n if tolerance is None:\n assert actual_value == expected\n else:\n # Compare numbers up to a certain precision\n assert actual_value == pytest.approx(\n expected, rel=tolerance, abs=tolerance, nan_ok=True\n )\n\n\ndef get_invalid_mask(input_array, fillna=np.nan):\n \"\"\"\n Return a bool array indicating the invalid values in ``input_array``.\n\n If the input array is a MaskedArray, its mask will be returned.\n Otherwise, it returns an array with the ``input_array == fillna``\n element-wise comparison.\n \"\"\"\n if isinstance(input_array, np.ma.MaskedArray):\n invalid_mask = np.ma.getmaskarray(input_array)\n else:\n if fillna is np.nan:\n invalid_mask = ~np.isfinite(input_array)\n else:\n invalid_mask = input_array == fillna\n\n return invalid_mask\n" }, { "path": "pysteps/tests/test_archive.py", "content": "# -*- coding: utf-8 -*-\n\nimport pytest\nfrom datetime import datetime\n\nfrom pysteps.io.archive import _generate_path\n\ntest_argvalues = [\n (\"20190130_1200\", \"%Y/foo/%m\", \"./2019/foo/01\"),\n (\"20190225_1200\", \"%Y/foo/%m\", \"./2019/foo/02\"),\n (\"20190122_2222\", \"%Y/foo/%m\", \"./2019/foo/01\"),\n (\"20190130_1200\", \"%Y/foo/%m\", \"./2019/foo/01\"),\n (\"20190130_1205\", \"%Y%m%d/foo/bar/%H%M\", \"./20190130/foo/bar/1205\"),\n (\"20190130_1205\", \"foo/bar/%H%M\", \"./foo/bar/1205\"),\n]\n\n\n@pytest.mark.parametrize(\"timestamp, path_fmt, expected_path\", test_argvalues)\ndef test_generate_path(timestamp, path_fmt, expected_path):\n date = datetime.strptime(timestamp, \"%Y%m%d_%H%M\")\n assert _generate_path(date, \"./\", path_fmt) == expected_path\n" }, { "path": "pysteps/tests/test_blending_clim.py", "content": "# -*- coding: utf-8 -*-\n\n\nfrom datetime import datetime, timedelta\nfrom os.path import join, exists\nimport pickle\nimport random\n\nimport numpy as np\nfrom numpy.testing import assert_array_equal\nimport pytest\n\nfrom pysteps.blending.clim import save_skill, calc_clim_skill\n\nrandom.seed(12356)\nn_cascade_levels = 7\nmodel_names = [\"alaro13\", \"arome13\"]\ndefault_start_skill = [0.8, 0.5]\n\n\n# Helper functions\ndef generate_fixed_skill(n_cascade_levels, n_models=1):\n \"\"\"\n Generate skill starting at default_start_skill which decay exponentially with scale.\n \"\"\"\n start_skill = np.resize(default_start_skill, n_models)\n powers = np.arange(1, n_cascade_levels + 1)\n return pow(start_skill[:, np.newaxis], powers)\n\n\n# Test arguments\nclim_arg_names = (\"startdatestr\", \"enddatestr\", \"n_models\", \"expected_skill_today\")\n\ntest_enddates = [\"20210701235500\", \"20210702000000\", \"20200930235500\"]\n\nclim_arg_values = [\n (\n \"20210701230000\",\n \"20210701235500\",\n 1,\n {\n \"mean_skill\": generate_fixed_skill(n_cascade_levels),\n \"n\": 12,\n \"last_validtime\": datetime.strptime(test_enddates[0], \"%Y%m%d%H%M%S\"),\n },\n ),\n (\n \"20210701235500\",\n \"20210702000000\",\n 1,\n {\n \"mean_skill\": generate_fixed_skill(n_cascade_levels),\n \"n\": 1,\n \"last_validtime\": datetime.strptime(test_enddates[1], \"%Y%m%d%H%M%S\"),\n },\n ),\n (\n \"20200801000000\",\n \"20200930235500\",\n 1,\n {\n \"mean_skill\": generate_fixed_skill(n_cascade_levels),\n \"n\": 288,\n \"last_validtime\": datetime.strptime(test_enddates[2], \"%Y%m%d%H%M%S\"),\n },\n ),\n (\n \"20210701230000\",\n \"20210701235500\",\n 2,\n {\n \"mean_skill\": generate_fixed_skill(n_cascade_levels, 2),\n \"n\": 12,\n \"last_validtime\": datetime.strptime(test_enddates[0], \"%Y%m%d%H%M%S\"),\n },\n ),\n (\n \"20210701230000\",\n \"20210702000000\",\n 2,\n {\n \"mean_skill\": generate_fixed_skill(n_cascade_levels, 2),\n \"n\": 1,\n \"last_validtime\": datetime.strptime(test_enddates[1], \"%Y%m%d%H%M%S\"),\n },\n ),\n (\n \"20200801000000\",\n \"20200930235500\",\n 2,\n {\n \"mean_skill\": generate_fixed_skill(n_cascade_levels, 2),\n \"n\": 288,\n \"last_validtime\": datetime.strptime(test_enddates[2], \"%Y%m%d%H%M%S\"),\n },\n ),\n]\n\n\n@pytest.mark.parametrize(clim_arg_names, clim_arg_values)\ndef test_save_skill(startdatestr, enddatestr, n_models, expected_skill_today, tmpdir):\n \"\"\"Test if the skill are saved correctly and the daily average is computed\"\"\"\n\n # get validtime\n currentdate = datetime.strptime(startdatestr, \"%Y%m%d%H%M%S\")\n enddate = datetime.strptime(enddatestr, \"%Y%m%d%H%M%S\")\n timestep = timedelta(minutes=5)\n\n outdir_path = tmpdir\n\n while currentdate <= enddate:\n current_skill = generate_fixed_skill(n_cascade_levels, n_models)\n print(\"Saving skill: \", current_skill, currentdate, outdir_path)\n save_skill(\n current_skill, currentdate, outdir_path, n_models=n_models, window_length=2\n )\n currentdate += timestep\n\n skill_today_file = join(outdir_path, \"NWP_skill_today.pkl\")\n assert exists(skill_today_file)\n with open(skill_today_file, \"rb\") as f:\n skill_today = pickle.load(f)\n\n # Check type\n assert isinstance(skill_today, dict)\n assert \"mean_skill\" in skill_today\n assert \"n\" in skill_today\n assert \"last_validtime\" in skill_today\n assert_array_equal(skill_today[\"mean_skill\"], expected_skill_today[\"mean_skill\"])\n assert skill_today[\"n\"] == expected_skill_today[\"n\"]\n assert skill_today[\"last_validtime\"] == expected_skill_today[\"last_validtime\"]\n\n # Finally, check if the clim skill calculation returns an array of values\n clim_skill = calc_clim_skill(\n outdir_path=tmpdir,\n n_cascade_levels=n_cascade_levels,\n n_models=n_models,\n window_length=2,\n )\n\n assert clim_skill.shape[0] == n_models\n assert clim_skill.shape[1] == n_cascade_levels\n\n\nif __name__ == \"__main__\":\n save_skill(\n generate_fixed_skill(n_cascade_levels, 1),\n datetime.strptime(\"20200801000000\", \"%Y%m%d%H%M%S\"),\n \"./tmp/\",\n )\n" }, { "path": "pysteps/tests/test_blending_linear_blending.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom pysteps.blending.linear_blending import forecast, _get_ranked_salience, _get_ws\nfrom numpy.testing import assert_array_almost_equal\nfrom pysteps.utils import transformation\n\n# Test function arguments\nlinear_arg_values = [\n (5, 30, 60, 20, 45, \"eulerian\", None, 1, False, True, False),\n (5, 30, 60, 20, 45, \"eulerian\", None, 2, False, False, False),\n (5, 30, 60, 20, 45, \"eulerian\", None, 0, False, False, False),\n (4, 23, 33, 9, 28, \"eulerian\", None, 1, False, False, False),\n (3, 18, 36, 13, 27, \"eulerian\", None, 1, False, False, False),\n (7, 30, 68, 11, 49, \"eulerian\", None, 1, False, False, False),\n (7, 30, 68, 11, 49, \"eulerian\", None, 1, False, False, True),\n (10, 100, 160, 25, 130, \"eulerian\", None, 1, False, False, False),\n (6, 60, 180, 22, 120, \"eulerian\", None, 1, False, False, False),\n (5, 100, 200, 40, 150, \"eulerian\", None, 1, False, False, False),\n (\n 5,\n 30,\n 60,\n 20,\n 45,\n \"extrapolation\",\n np.zeros((2, 200, 200)),\n 1,\n False,\n False,\n False,\n ),\n (\n 4,\n 23,\n 33,\n 9,\n 28,\n \"extrapolation\",\n np.zeros((2, 200, 200)),\n 1,\n False,\n False,\n False,\n ),\n (\n 3,\n 18,\n 36,\n 13,\n 27,\n \"extrapolation\",\n np.zeros((2, 200, 200)),\n 1,\n False,\n False,\n False,\n ),\n (\n 7,\n 30,\n 68,\n 11,\n 49,\n \"extrapolation\",\n np.zeros((2, 200, 200)),\n 1,\n False,\n False,\n False,\n ),\n (\n 10,\n 100,\n 160,\n 25,\n 130,\n \"extrapolation\",\n np.zeros((2, 200, 200)),\n 1,\n False,\n False,\n False,\n ),\n (\n 6,\n 60,\n 180,\n 22,\n 120,\n \"extrapolation\",\n np.zeros((2, 200, 200)),\n 1,\n False,\n False,\n False,\n ),\n (\n 5,\n 100,\n 200,\n 40,\n 150,\n \"extrapolation\",\n np.zeros((2, 200, 200)),\n 1,\n False,\n False,\n False,\n ),\n (\n 5,\n 100,\n 200,\n 40,\n 150,\n \"extrapolation\",\n np.zeros((2, 200, 200)),\n 1,\n False,\n False,\n True,\n ),\n (5, 30, 60, 20, 45, \"eulerian\", None, 1, True, True, False),\n (5, 30, 60, 20, 45, \"eulerian\", None, 2, True, False, False),\n (5, 30, 60, 20, 45, \"eulerian\", None, 0, True, False, False),\n (\n 5,\n 30,\n 60,\n 20,\n 45,\n \"extrapolation\",\n np.zeros((2, 200, 200)),\n 1,\n True,\n False,\n False,\n ),\n (4, 23, 33, 9, 28, \"extrapolation\", np.zeros((2, 200, 200)), 1, True, False, False),\n (\n 3,\n 18,\n 36,\n 13,\n 27,\n \"extrapolation\",\n np.zeros((2, 200, 200)),\n 1,\n True,\n False,\n False,\n ),\n]\n\n\n@pytest.mark.parametrize(\n \"timestep, start_blending, end_blending, n_timesteps, controltime, nowcast_method, V, n_models, salient_blending, squeeze_nwp_array, fill_nwp\",\n linear_arg_values,\n)\ndef test_linear_blending(\n timestep,\n start_blending,\n end_blending,\n n_timesteps,\n controltime,\n nowcast_method,\n V,\n n_models,\n salient_blending,\n squeeze_nwp_array,\n fill_nwp,\n):\n \"\"\"Tests if the linear blending function is correct. For the nowcast data a precipitation field\n which is constant over time is taken. One half of the field has no rain and the other half\n has a set value. For the NWP data a similar field is taken, the only difference\n being that now the other half of the field is zero. The blended field should have a\n constant value over the entire field at the timestep right in the middle between the start\n of the blending and the end of the blending. This assertion is checked to see if the\n linear blending function works well.\"\"\"\n\n # The argument controltime gives the timestep at which the field is assumed to be\n # entirely constant\n\n # Assert that the control time step is in the range of the forecasted time steps\n assert controltime <= (\n n_timesteps * timestep\n ), \"Control time needs to be within reach of forecasts, controltime = {} and n_timesteps = {}\".format(\n controltime, n_timesteps\n )\n\n # Assert that the start time of the blending comes before the end time of the blending\n assert (\n start_blending < end_blending\n ), \"Start time of blending needs to be smaller than end time of blending\"\n\n # Assert that the control time is a multiple of the time step\n assert (\n not controltime % timestep\n ), \"Control time needs to be a multiple of the time step\"\n\n # Initialise dummy NWP data\n if n_models == 0:\n r_nwp = None\n else:\n r_nwp = np.zeros((n_models, n_timesteps, 200, 200))\n\n for i in range(100):\n r_nwp[:, :, i, :] = 11.0\n\n if squeeze_nwp_array:\n r_nwp = np.squeeze(r_nwp)\n\n # Define nowcast input data (alternate between 2D and 3D arrays for testing)\n if timestep % 2 == 0:\n r_input = np.zeros((4, 200, 200))\n for i in range(100, 200):\n r_input[:, i, :] = 11.0\n else:\n r_input = np.zeros((200, 200))\n for i in range(100, 200):\n r_input[i, :] = 11.0\n\n # Transform from mm/h to dB\n r_input, _ = transformation.dB_transform(\n r_input, None, threshold=0.1, zerovalue=-15.0\n )\n\n # Calculate the blended field\n r_blended = forecast(\n r_input,\n dict({\"unit\": \"mm/h\", \"transform\": \"dB\"}),\n V,\n n_timesteps,\n timestep,\n nowcast_method,\n r_nwp,\n dict({\"unit\": \"mm/h\", \"transform\": None}),\n start_blending=start_blending,\n end_blending=end_blending,\n fill_nwp=fill_nwp,\n saliency=salient_blending,\n )\n\n # Assert that the blended field has the expected dimension\n if n_models > 1:\n assert r_blended.shape == (\n n_models,\n n_timesteps,\n 200,\n 200,\n ), \"The shape of the blended array does not have the expected value. The shape is {}\".format(\n r_blended.shape\n )\n else:\n assert r_blended.shape == (\n n_timesteps,\n 200,\n 200,\n ), \"The shape of the blended array does not have the expected value. The shape is {}\".format(\n r_blended.shape\n )\n\n # Assert that the blended field at the control time step is equal to\n # a constant field with the expected value.\n if salient_blending == False:\n if n_models > 1:\n assert_array_almost_equal(\n r_blended[0, controltime // timestep - 1],\n np.ones((200, 200)) * 5.5,\n err_msg=\"The blended array does not have the expected value\",\n )\n elif n_models > 0:\n assert_array_almost_equal(\n r_blended[controltime // timestep - 1],\n np.ones((200, 200)) * 5.5,\n err_msg=\"The blended array does not have the expected value\",\n )\n\n\nranked_salience_values = [\n (np.ones((200, 200)), np.ones((200, 200)), 0.9),\n (np.zeros((200, 200)), np.random.rand(200, 200), 0.7),\n (np.random.rand(200, 200), np.random.rand(200, 200), 0.5),\n]\n\n\n@pytest.mark.parametrize(\n \"nowcast, nwp, weight_nowcast\",\n ranked_salience_values,\n)\ndef test_salient_weight(\n nowcast,\n nwp,\n weight_nowcast,\n):\n ranked_salience = _get_ranked_salience(nowcast, nwp)\n ws = _get_ws(weight_nowcast, ranked_salience)\n\n assert np.min(ws) >= 0, \"Negative value for the ranked saliency output\"\n assert np.max(ws) <= 1, \"Too large value for the ranked saliency output\"\n\n assert ws.shape == (\n 200,\n 200,\n ), \"The shape of the ranked salience array does not have the expected value. The shape is {}\".format(\n ws.shape\n )\n" }, { "path": "pysteps/tests/test_blending_pca_ens_kalman_filter.py", "content": "# -*- coding: utf-8 -*-\n\nimport datetime\n\nimport numpy as np\nimport pytest\n\nfrom pysteps import blending, motion, utils\n\n# fmt: off\npca_enkf_arg_values = [\n # Standard setting\n (20,30,0,-60,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,False),\n # Smooth radar mask\n (20,30,0,-60,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,20,False),\n # Coarser NWP temporal resolution\n (20,30,0,-60,False,False,5,15,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,False),\n # Coarser Obs temporal resolution\n (20,30,0,-60,False,False,10,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,False),\n # Larger shift of the NWP init\n (20,30,0,-30,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,False),\n # Zero rain case in observation\n (20,30,0,-60,True,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,False),\n # Zero rain case in NWP\n (20,30,0,-60,False,True,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,False),\n # Zero rain in both\n (20,30,0,-60,True,True,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,False),\n # Accumulated sampling probability\n (20,30,0,-60,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",True,False,0,False),\n # Use full NWP weight\n (20,30,0,-60,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,True,0,False),\n # Both\n (20,30,0,-60,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",True,True,0,False),\n # Explained variance as sampling probability source\n (20,30,0,-60,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"explained_var\",False,False,0,False),\n # No combination\n (20,30,0,-60,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",False,None,1.0,\"ensemble\",False,False,0,False),\n # Standard deviation adjustment\n (20,30,0,-60,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,\"auto\",1.0,\"ensemble\",False,False,0,False),\n # Other number of ensemble members\n (10,30,0,-60,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,False),\n # Other forecast length\n (20,35,0,-60,False,False,5,5,0.05,0.01,\"ssft\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,False),\n # Other noise method\n (20,30,0,-60,False,False,5,5,0.05,0.01,\"nonparametric\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,False),\n # Verbose output\n (20,30,0,-60,False,False,5,5,0.05,0.01,\"nonparametric\",\"masked_enkf\",True,None,1.0,\"ensemble\",False,False,0,True),]\n# fmt: on\n\npca_enkf_arg_names = (\n \"n_ens_members\",\n \"forecast_length\",\n \"forecast_shift_radar\",\n \"forecast_shift_nwp\",\n \"zero_radar\",\n \"zero_nwp\",\n \"temporal_res_radar\",\n \"temporal_res_nwp\",\n \"thr_prec\",\n \"norain_thr\",\n \"noise_method\",\n \"enkf_method\",\n \"enable_combination\",\n \"noise_stddev_adj\",\n \"inflation_factor_bg\",\n \"sampling_prob_source\",\n \"use_accum_sampling_prob\",\n \"ensure_full_nwp_weight\",\n \"smooth_radar_mask_range\",\n \"verbose_output\",\n)\n\n\n@pytest.mark.parametrize(pca_enkf_arg_names, pca_enkf_arg_values)\ndef test_pca_enkf_combination(\n n_ens_members,\n forecast_length,\n forecast_shift_radar,\n forecast_shift_nwp,\n zero_radar,\n zero_nwp,\n temporal_res_radar,\n temporal_res_nwp,\n thr_prec,\n norain_thr,\n noise_method,\n enkf_method,\n enable_combination,\n noise_stddev_adj,\n inflation_factor_bg,\n sampling_prob_source,\n use_accum_sampling_prob,\n ensure_full_nwp_weight,\n smooth_radar_mask_range,\n verbose_output,\n):\n pytest.importorskip(\"sklearn\")\n\n # Set forecast init\n forecast_init = datetime.datetime(2025, 6, 4, 17, 0)\n\n # Initialize dummy radar data\n radar_precip = np.zeros((2, 200, 200))\n if not zero_radar:\n for i in range(radar_precip.shape[0]):\n a = 5 * i\n radar_precip[i, 5 + a : 100 - a, 30 + a : 180 - a] = 0.1\n radar_precip[i, 10 + a : 105 - a, 35 + a : 178 - a] = 0.5\n radar_precip[i, 15 + a : 110 - a, 40 + a : 176 - a] = 0.5\n radar_precip[i, 20 + a : 115 - a, 45 + a : 174 - a] = 5.0\n radar_precip[i, 25 + a : 120 - a, 50 + a : 172 - a] = 5.0\n radar_precip[i, 30 + a : 125 - a, 55 + a : 170 - a] = 4.5\n radar_precip[i, 35 + a : 130 - a, 60 + a : 168 - a] = 4.5\n radar_precip[i, 40 + a : 135 - a, 65 + a : 166 - a] = 4.0\n radar_precip[i, 45 + a : 140 - a, 70 + a : 164 - a] = 1.0\n radar_precip[i, 50 + a : 145 - a, 75 + a : 162 - a] = 0.5\n radar_precip[i, 55 + a : 150 - a, 80 + a : 160 - a] = 0.5\n radar_precip[i, 60 + a : 155 - a, 85 + a : 158 - a] = 0.1\n\n radar_precip_timestamps = np.array(\n sorted(\n [\n forecast_init\n + datetime.timedelta(minutes=forecast_shift_radar)\n - datetime.timedelta(minutes=i * temporal_res_radar)\n for i in range(radar_precip.shape[0])\n ]\n )\n )\n\n # Initialize dummy NWP data\n nwp_precip = np.zeros((n_ens_members, 20, 200, 200))\n if not zero_nwp:\n for n_model in range(n_ens_members):\n for i in range(nwp_precip.shape[1]):\n a = 2 * n_model\n b = 2 * i\n nwp_precip[n_model, i, 20 + b : 160 - b, 30 + a : 180 - a] = 0.1\n nwp_precip[n_model, i, 22 + b : 162 - b, 35 + a : 178 - a] = 0.1\n nwp_precip[n_model, i, 24 + b : 164 - b, 40 + a : 176 - a] = 1.0\n nwp_precip[n_model, i, 26 + b : 166 - b, 45 + a : 174 - a] = 5.0\n nwp_precip[n_model, i, 28 + b : 168 - b, 50 + a : 172 - a] = 5.0\n nwp_precip[n_model, i, 30 + b : 170 - b, 35 + a : 170 - a] = 4.5\n nwp_precip[n_model, i, 32 + b : 172 - b, 40 + a : 168 - a] = 4.5\n nwp_precip[n_model, i, 34 + b : 174 - b, 45 + a : 166 - a] = 4.0\n nwp_precip[n_model, i, 36 + b : 176 - b, 50 + a : 164 - a] = 2.0\n nwp_precip[n_model, i, 38 + b : 178 - b, 55 + a : 162 - a] = 1.0\n nwp_precip[n_model, i, 40 + b : 180 - b, 60 + a : 160 - a] = 0.5\n nwp_precip[n_model, i, 42 + b : 182 - b, 65 + a : 158 - a] = 0.1\n\n nwp_precip_timestamps = np.array(\n sorted(\n [\n forecast_init\n + datetime.timedelta(minutes=forecast_shift_nwp)\n + datetime.timedelta(minutes=i * temporal_res_nwp)\n for i in range(nwp_precip.shape[1])\n ]\n )\n )\n\n # Metadata of dummy data is necessary for data conversion\n metadata = dict()\n metadata[\"unit\"] = \"mm\"\n metadata[\"transformation\"] = \"dB\"\n metadata[\"accutime\"] = 5.0\n metadata[\"transform\"] = None\n metadata[\"zerovalue\"] = 0.0\n metadata[\"threshold\"] = thr_prec\n metadata[\"zr_a\"] = 200.0\n metadata[\"zr_b\"] = 1.6\n\n # Converting the input data\n # Thresholding\n radar_precip[radar_precip < metadata[\"threshold\"]] = 0.0\n nwp_precip[nwp_precip < metadata[\"threshold\"]] = 0.0\n\n # Convert the data\n converter = utils.get_method(\"mm/h\")\n radar_precip, _ = converter(radar_precip, metadata)\n nwp_precip, metadata = converter(nwp_precip, metadata)\n\n # Transform the data\n transformer = utils.get_method(metadata[\"transformation\"])\n radar_precip, _ = transformer(radar_precip, metadata)\n nwp_precip, metadata = transformer(nwp_precip, metadata)\n\n # Set NaN equal to zero\n radar_precip[~np.isfinite(radar_precip)] = metadata[\"zerovalue\"]\n nwp_precip[~np.isfinite(nwp_precip)] = metadata[\"zerovalue\"]\n\n assert (\n np.any(~np.isfinite(radar_precip)) == False\n ), \"There are still infinite values in the input radar data\"\n assert (\n np.any(~np.isfinite(nwp_precip)) == False\n ), \"There are still infinite values in the NWP data\"\n\n # Initialize radar velocity\n oflow_method = motion.get_method(\"LK\")\n radar_velocity = oflow_method(radar_precip)\n\n # Set the combination kwargs\n combination_kwargs = dict(\n n_tapering=0,\n non_precip_mask=True,\n n_ens_prec=1,\n lien_criterion=True,\n n_lien=10,\n prob_matching=\"iterative\",\n inflation_factor_bg=inflation_factor_bg,\n inflation_factor_obs=1.0,\n offset_bg=0.0,\n offset_obs=0.0,\n nwp_hres_eff=14.0,\n sampling_prob_source=sampling_prob_source,\n use_accum_sampling_prob=use_accum_sampling_prob,\n ensure_full_nwp_weight=ensure_full_nwp_weight,\n )\n\n # Call the reduced-spaced ensemble Kalman filter approach.\n combined_forecast = blending.pca_ens_kalman_filter.forecast(\n obs_precip=radar_precip,\n obs_timestamps=radar_precip_timestamps,\n nwp_precip=nwp_precip,\n nwp_timestamps=nwp_precip_timestamps,\n velocity=radar_velocity,\n forecast_horizon=forecast_length,\n issuetime=forecast_init,\n n_ens_members=n_ens_members,\n precip_mask_dilation=1,\n smooth_radar_mask_range=smooth_radar_mask_range,\n n_cascade_levels=6,\n precip_thr=metadata[\"threshold\"],\n norain_thr=norain_thr,\n extrap_method=\"semilagrangian\",\n decomp_method=\"fft\",\n bandpass_filter_method=\"gaussian\",\n noise_method=noise_method,\n enkf_method=enkf_method,\n enable_combination=enable_combination,\n noise_stddev_adj=noise_stddev_adj,\n ar_order=1,\n callback=None,\n return_output=True,\n seed=None,\n num_workers=1,\n fft_method=\"numpy\",\n domain=\"spatial\",\n extrap_kwargs=None,\n filter_kwargs=None,\n noise_kwargs=None,\n combination_kwargs=combination_kwargs,\n measure_time=False,\n verbose_output=verbose_output,\n )\n\n if verbose_output:\n assert len(combined_forecast) == 2, \"Wrong amount of output data\"\n combined_forecast = combined_forecast[0]\n\n assert combined_forecast.ndim == 4, \"Wrong amount of dimensions in forecast output\"\n assert (\n combined_forecast.shape[0] == n_ens_members\n ), \"Wrong amount of output ensemble members in forecast output\"\n assert (\n combined_forecast.shape[1] == forecast_length // temporal_res_radar + 1\n ), \"Wrong amount of output time steps in forecast output\"\n\n # Transform the data back into mm/h\n combined_forecast, _ = converter(combined_forecast, metadata)\n\n assert (\n combined_forecast.ndim == 4\n ), \"Wrong amount of dimensions in converted forecast output\"\n assert (\n combined_forecast.shape[0] == n_ens_members\n ), \"Wrong amount of output ensemble members in converted forecast output\"\n assert (\n combined_forecast.shape[1] == forecast_length // temporal_res_radar + 1\n ), \"Wrong amount of output time steps in converted forecast output\"\n\n return\n" }, { "path": "pysteps/tests/test_blending_skill_scores.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.blending.skill_scores import (\n spatial_correlation,\n lt_dependent_cor_nwp,\n lt_dependent_cor_extrapolation,\n clim_regr_values,\n)\n\n# Set the climatological correlations values\nclim_cor_values_8lev = np.array(\n [0.848, 0.537, 0.237, 0.065, 0.020, 0.0044, 0.0052, 0.0040]\n)\nclim_cor_values_6lev = np.array([0.848, 0.537, 0.237, 0.065, 0.020, 0.0044])\nclim_cor_values_9lev = np.array(\n [0.848, 0.537, 0.237, 0.065, 0.020, 0.0044, 0.0052, 0.0040, 1e-4]\n)\n\n# Set the regression values\nregr_pars_8lev = np.array(\n [\n [130.0, 165.0, 120.0, 55.0, 50.0, 15.0, 15.0, 10.0],\n [155.0, 220.0, 200.0, 75.0, 10e4, 10e4, 10e4, 10e4],\n ]\n)\nregr_pars_6lev = np.array(\n [\n [130.0, 165.0, 120.0, 55.0, 50.0, 15.0],\n [155.0, 220.0, 200.0, 75.0, 10e4, 10e4],\n ]\n)\nregr_pars_9lev = np.array(\n [\n [130.0, 165.0, 120.0, 55.0, 50.0, 15.0, 15.0, 10.0, 10.0],\n [155.0, 220.0, 200.0, 75.0, 10e4, 10e4, 10e4, 10e4, 10e4],\n ]\n)\n\n# Set the dummy observation and model values\ndummy_2d_array = np.array([[1.0, 2.0], [3.0, 4.0]])\nobs_8lev = np.repeat(dummy_2d_array[None, :, :], 8, axis=0)\nobs_6lev = np.repeat(dummy_2d_array[None, :, :], 6, axis=0)\nobs_9lev = np.repeat(dummy_2d_array[None, :, :], 9, axis=0)\nmod_8lev = np.repeat(dummy_2d_array[None, :, :], 8, axis=0)\nmod_6lev = np.repeat(dummy_2d_array[None, :, :], 6, axis=0)\nmod_9lev = np.repeat(dummy_2d_array[None, :, :], 9, axis=0)\n\n# Gives some dummy values to PHI\ndummy_phi = np.array([0.472650, 0.523825, 0.103454])\nPHI_8lev = np.repeat(dummy_phi[None, :], 8, axis=0)\nPHI_6lev = np.repeat(dummy_phi[None, :], 6, axis=0)\nPHI_9lev = np.repeat(dummy_phi[None, :], 9, axis=0)\n\n# Test function arguments\nskill_scores_arg_names = (\n \"obs\",\n \"mod\",\n \"lt\",\n \"PHI\",\n \"cor_prev\",\n \"clim_cor_values\",\n \"regr_pars\",\n \"n_cascade_levels\",\n \"expected_cor_t0\",\n \"expected_cor_nwp_lt\",\n \"expected_cor_nowcast_lt\",\n \"n_model\",\n \"number_of_models\",\n)\n\n# Test function values\nskill_scores_arg_values = [\n (\n obs_8lev,\n mod_8lev,\n 60,\n PHI_8lev,\n None,\n clim_cor_values_8lev,\n regr_pars_8lev,\n 8,\n np.repeat(1.0, 8),\n np.array(\n [\n 0.97455941,\n 0.9356775,\n 0.81972779,\n 0.55202975,\n 0.31534738,\n 0.02264599,\n 0.02343133,\n 0.00647032,\n ]\n ),\n np.array(\n [\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n ]\n ),\n 0,\n None,\n ),\n (\n obs_6lev,\n mod_6lev,\n 60,\n PHI_6lev,\n None,\n clim_cor_values_6lev,\n regr_pars_6lev,\n 6,\n np.repeat(1.0, 6),\n np.array(\n [0.97455941, 0.9356775, 0.81972779, 0.55202975, 0.31534738, 0.02264599]\n ),\n np.array([0.996475, 0.996475, 0.996475, 0.996475, 0.996475, 0.996475]),\n 0,\n 1,\n ),\n (\n obs_9lev,\n mod_9lev,\n 60,\n PHI_9lev,\n None,\n clim_cor_values_9lev,\n regr_pars_9lev,\n 9,\n np.repeat(1.0, 9),\n np.array(\n [\n 0.97455941,\n 0.9356775,\n 0.81972779,\n 0.55202975,\n 0.31534738,\n 0.02264599,\n 0.02343133,\n 0.00647032,\n 0.00347776,\n ]\n ),\n np.array(\n [\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n ]\n ),\n 0,\n 1,\n ),\n (\n obs_8lev,\n mod_8lev,\n 0,\n PHI_8lev,\n None,\n clim_cor_values_8lev,\n regr_pars_8lev,\n 8,\n np.repeat(1.0, 8),\n np.repeat(1.0, 8),\n np.array(\n [\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n ]\n ),\n 0,\n 1,\n ),\n (\n obs_8lev,\n mod_8lev,\n 0,\n PHI_8lev,\n None,\n clim_cor_values_8lev,\n regr_pars_8lev,\n 8,\n np.repeat(1.0, 8),\n np.repeat(1.0, 8),\n np.array(\n [\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n 0.996475,\n ]\n ),\n 1,\n 2,\n ),\n]\n\n\n# The test\n@pytest.mark.parametrize(skill_scores_arg_names, skill_scores_arg_values)\n\n# The test function to be used\ndef test_blending_skill_scores(\n obs,\n mod,\n lt,\n PHI,\n cor_prev,\n clim_cor_values,\n regr_pars,\n n_cascade_levels,\n expected_cor_t0,\n expected_cor_nwp_lt,\n expected_cor_nowcast_lt,\n n_model,\n number_of_models,\n):\n \"\"\"Tests if the skill_score functions behave correctly. A dummy gridded\n model and observation field should be given for n_cascade_levels, which\n leads to a given spatial correlation per cascade level. Then, the function\n tests if the correlation regresses towards the climatological values given\n lead time lt for the NWP fields or given the PHI-values for the\n extrapolation field.\n\n \"\"\"\n if number_of_models != None:\n skill_kwargs = {\"n_models\": number_of_models}\n else:\n skill_kwargs = None\n\n domain_mask = np.full(obs[0, :, :].shape, False, dtype=bool)\n\n # Calculate the spatial correlation of the given model field\n correlations_t0 = np.array(spatial_correlation(obs, mod, domain_mask))\n\n # Check if the field has the same number of cascade levels as the model\n # field and as the given n_cascade_levels\n assert (\n correlations_t0.shape[0] == mod.shape[0]\n ), \"Number of cascade levels should be the same as in the model field\"\n assert (\n correlations_t0.shape[0] == n_cascade_levels\n ), \"Number of cascade levels should be the same as n_cascade_levels\"\n\n # Check if the returned values are as expected\n assert_array_almost_equal(\n correlations_t0,\n expected_cor_t0,\n decimal=3,\n err_msg=\"Returned spatial correlation is not the same as the expected value\",\n )\n\n # Test if the NWP correlation regresses towards the correct value given\n # a lead time in minutes\n # First, check if the climatological values are returned correctly\n correlations_clim, regr_clim = clim_regr_values(\n n_cascade_levels=n_cascade_levels,\n outdir_path=\"./tmp/\",\n n_model=n_model,\n skill_kwargs=skill_kwargs,\n )\n assert (\n correlations_clim.shape[0] == n_cascade_levels\n ), \"Number of cascade levels should be the same as n_cascade_levels\"\n assert_array_almost_equal(\n correlations_clim,\n clim_cor_values,\n decimal=3,\n err_msg=\"Not the correct climatological correlations were returned\",\n )\n assert_array_almost_equal(\n regr_clim,\n regr_pars,\n decimal=3,\n err_msg=\"Not the correct regression parameters were returned\",\n )\n\n # Then, check the regression of the correlation values\n correlations_nwp_lt = lt_dependent_cor_nwp(\n lt=lt, correlations=correlations_t0, outdir_path=\"./tmp/\"\n )\n assert (\n correlations_nwp_lt.shape[0] == mod.shape[0]\n ), \"Number of cascade levels should be the same as in the model field\"\n assert (\n correlations_nwp_lt.shape[0] == n_cascade_levels\n ), \"Number of cascade levels should be the same as n_cascade_levels\"\n assert_array_almost_equal(\n correlations_nwp_lt,\n expected_cor_nwp_lt,\n decimal=3,\n err_msg=\"Correlations of NWP not equal to the expected correlations\",\n )\n\n # Finally, make sure nowcast correlation regresses towards the correct\n # value given some PHI-values.\n correlations_nowcast_lt, __ = lt_dependent_cor_extrapolation(\n PHI, correlations_t0, cor_prev\n )\n\n print(correlations_nowcast_lt)\n assert (\n correlations_nowcast_lt.shape[0] == mod.shape[0]\n ), \"Number of cascade levels should be the same as in the model field\"\n assert (\n correlations_nowcast_lt.shape[0] == n_cascade_levels\n ), \"Number of cascade levels should be the same as n_cascade_levels\"\n assert_array_almost_equal(\n correlations_nowcast_lt,\n expected_cor_nowcast_lt,\n decimal=3,\n err_msg=\"Correlations of nowcast not equal to the expected correlations\",\n )\n" }, { "path": "pysteps/tests/test_blending_steps.py", "content": "# -*- coding: utf-8 -*-\n\nimport datetime\n\nimport numpy as np\nimport pytest\n\nimport pysteps\nfrom pysteps import blending, cascade\n\n# fmt:off\nsteps_arg_values = [\n (1, 3, 4, 8, 'steps', None, None, False, \"spn\", True, 4, False, False, 0, False, None, None, None),\n (1, 3, 4, 8,'steps', \"obs\", None, False, \"spn\", True, 4, False, False, 0, False, None, None, None),\n (1, 3, 4, 8,'steps', \"incremental\", None, False, \"spn\", True, 4, False, False, 0, False, None, None, None),\n (1, 3, 4, 8,'steps', None, \"mean\", False, \"spn\", True, 4, False, False, 0, False, None, None, None),\n (1, 3, 4, 8,'steps', None, \"mean\", False, \"spn\", True, 4, False, False, 0, True, None, None, None),\n (1, 3, 4, 8,'steps', None, \"cdf\", False, \"spn\", True, 4, False, False, 0, False, None, None, None),\n (1, [1, 2, 3], 4, 8,'steps', None, \"cdf\", False, \"spn\", True, 4, False, False, 0, False, None, None, None),\n (1, 3, 4, 8,'steps', \"incremental\", \"cdf\", False, \"spn\", True, 4, False, False, 0, False, None, None, None),\n (1, 3, 4, 6,'steps', \"incremental\", \"cdf\", False, \"bps\", True, 4, False, False, 0, False, None, None, None),\n (1, 3, 4, 6,'steps', \"incremental\", \"cdf\", False, \"bps\", False, 4, False, False, 0, False, None, None, None),\n (1, 3, 4, 6,'steps', \"incremental\", \"cdf\", False, \"bps\", False, 4, False, False, 0, True, None, None, None),\n (1, 3, 4, 9,'steps', \"incremental\", \"cdf\", False, \"spn\", True, 4, False, False, 0, False, None, None, None),\n (2, 3, 10, 8,'steps', \"incremental\", \"cdf\", False, \"spn\", True, 10, False, False, 0, False, None, None, None),\n (5, 3, 5, 8,'steps', \"incremental\", \"cdf\", False, \"spn\", True, 5, False, False, 0, False, None, None, None),\n (1, 10, 1, 8,'steps', \"incremental\", \"cdf\", False, \"spn\", True, 1, False, False, 0, False, None, None, None),\n (2, 3, 2, 8,'steps', \"incremental\", \"cdf\", True, \"spn\", True, 2, False, False, 0, False, None, None, None),\n (1, 3, 6, 8,'steps', None, None, False, \"spn\", True, 6, False, False, 0, False, None, None, None),\n (1, 3, 6, 8,'steps', None, None, False, \"spn\", True, 6, False, False, 0, False, \"bps\", None, None),\n # Test the case where the radar image contains no rain.\n (1, 3, 6, 8,'steps', None, None, False, \"spn\", True, 6, True, False, 0, False, None, None, None),\n (5, 3, 5, 6,'steps', \"incremental\", \"cdf\", False, \"spn\", False, 5, True, False, 0, False, None, None, None),\n (5, 3, 5, 6,'steps', \"incremental\", \"cdf\", False, \"spn\", False, 5, True, False, 0, True, None, None, None),\n # Test the case where the NWP fields contain no rain.\n (1, 3, 6, 8,'steps', None, None, False, \"spn\", True, 6, False, True, 0, False, None, None, None),\n (5, 3, 5, 6,'steps', \"incremental\", \"cdf\", False, \"spn\", False, 5, False, True, 0, True, None, None, None),\n # Test the case where both the radar image and the NWP fields contain no rain.\n (1, 3, 6, 8,'steps', None, None, False, \"spn\", True, 6, True, True, 0, False, None, None, None),\n (5, 3, 5, 6,'steps', \"incremental\", \"cdf\", False, \"spn\", False, 5, True, True, 0, False, None, None, None),\n (5, 3, 5, 6,'steps', \"obs\", \"mean\", True, \"spn\", True, 5, True, True, 0, False, None, None, None),\n # Test cases where we apply timestep_start_full_nwp_weight\n (1, 10, 2, 6,'steps', \"incremental\", \"cdf\", False, \"bps\", False, 2, False, False, 0, True, None, None, 5),\n (1, 10, 2, 6,'steps', \"incremental\", \"cdf\", False, \"spn\", False, 2, False, False, 0, False, None, None, 5),\n # Test for smooth radar mask\n (1, 3, 6, 8,'steps', None, None, False, \"spn\", True, 6, False, False, 80, False, None, None, None),\n (5, 3, 5, 6,'steps', \"incremental\", \"cdf\", False, \"spn\", False, 5, False, False, 80, False, None, None, None),\n (5, 3, 5, 6,'steps', \"obs\", \"mean\", False, \"spn\", False, 5, False, False, 80, False, None, None, None),\n (1, 3, 6, 8,'steps', None, None, False, \"spn\", True, 6, False, True, 80, False, None, None, None),\n (5, 3, 5, 6,'steps', \"incremental\", \"cdf\", False, \"spn\", False, 5, True, False, 80, True, None, None, None),\n (5, 3, 5, 6,'steps', \"obs\", \"mean\", False, \"spn\", False, 5, True, True, 80, False, None, None, None),\n (5, [1, 2, 3], 5, 6,'steps', \"obs\", \"mean\", False, \"spn\", False, 5, True, True, 80, False, None, None, None),\n (5, [1, 3], 5, 6,'steps', \"obs\", \"mean\", False, \"spn\", False, 5, True, True, 80, False, None, None, None),\n # Test the usage of a max_mask_rim in the mask_kwargs\n (1, 3, 6, 8,'steps', None, None, False, \"bps\", True, 6, False, False, 80, False, None, 40, None),\n (5, 3, 5, 6,'steps', \"obs\", \"mean\", False, \"bps\", False, 5, False, False, 80, False, None, 40, None),\n (5, 3, 5, 6,'steps', \"incremental\", \"cdf\", False, \"bps\", False, 5, False, False, 80, False, None, 25, None),\n (5, 3, 5, 6,'steps', \"incremental\", \"cdf\", False, \"bps\", False, 5, False, False, 80, False, None, 40, None),\n (5, 3, 5, 6,'steps', \"incremental\", \"cdf\", False, \"bps\", False, 5, False, False, 80, False, None, 60, None),\n #Test the externally provided nowcast\n (1, 10, 1, 8,'external_nowcast_det', None, None, False, \"spn\", True, 1, False, False, 0, False, None, None, None),\n (1, 10, 1, 8,'external_nowcast_det', \"incremental\", None, False, \"bps\", True, 1, False, False, 0, False, None, None, None),\n (1, 10, 1, 8,'external_nowcast_det', \"incremental\", None, False, \"spn\", True, 1, False, False, 80, False, None, None, None),\n (1, 10, 1, 8,'external_nowcast_det', \"incremental\", None, False, \"bps\", True, 1, True, False, 0, False, None, None, None),\n (1, 10, 1, 8,'external_nowcast_det', \"incremental\", None, False, \"spn\", True, 1, False, True, 0, False, None, None, None),\n (1, 10, 1, 8,'external_nowcast_det', \"incremental\", None, False, \"bps\", True, 1, True, True, 0, False, None, None, None),\n (1, 10, 1, 8,'external_nowcast_det', \"incremental\", \"cdf\", False, \"spn\", True, 1, False, False, 0, True, None, None, None),\n (1, 10, 1, 8,'external_nowcast_det', \"incremental\", \"obs\", False, \"bps\", True, 1, False, False, 0, False, None, None, None),\n (1, 10, 1, 8,'external_nowcast_det', \"incremental\", None, False, \"bps\", True, 1, False, False, 0, False, None, None, 5),\n (5, 10, 5, 8,'external_nowcast_ens', \"incremental\", None, False, \"spn\", True, 5, False, False, 0, False, None, None, None),\n (5, 10, 5, 8,'external_nowcast_ens', \"incremental\", None, False, \"spn\", True, 5, False, False, 0, False, None, None, None),\n (1, 10, 5, 8,'external_nowcast_ens', \"incremental\", None, False, \"spn\", True, 5, False, False, 0, False, None, None, None),\n (1, 10, 1, 8,'external_nowcast_ens', \"incremental\", \"cdf\", False, \"bps\", True, 5, False, False, 0, False, None, None, None),\n (5, 10, 1, 8,'external_nowcast_ens', \"incremental\", \"obs\", False, \"spn\", True, 5, False, False, 0, False, None, None, None),\n (1, 10, 5, 8,'external_nowcast_ens', \"incremental\", \"cdf\", False, \"bps\", True, 5, False, False, 0, False, None, None, 5)\n]\n\n# fmt:on\n\nsteps_arg_names = (\n \"n_models\",\n \"timesteps\",\n \"n_ens_members\",\n \"n_cascade_levels\",\n \"nowcasting_method\",\n \"mask_method\",\n \"probmatching_method\",\n \"blend_nwp_members\",\n \"weights_method\",\n \"decomposed_nwp\",\n \"expected_n_ens_members\",\n \"zero_radar\",\n \"zero_nwp\",\n \"smooth_radar_mask_range\",\n \"resample_distribution\",\n \"vel_pert_method\",\n \"max_mask_rim\",\n \"timestep_start_full_nwp_weight\",\n)\n\n\n@pytest.mark.parametrize(steps_arg_names, steps_arg_values)\ndef test_steps_blending(\n n_models,\n timesteps,\n n_ens_members,\n n_cascade_levels,\n nowcasting_method,\n mask_method,\n probmatching_method,\n blend_nwp_members,\n weights_method,\n decomposed_nwp,\n expected_n_ens_members,\n zero_radar,\n zero_nwp,\n smooth_radar_mask_range,\n resample_distribution,\n vel_pert_method,\n max_mask_rim,\n timestep_start_full_nwp_weight,\n):\n pytest.importorskip(\"cv2\")\n\n ###\n # The input data\n ###\n # Initialise dummy NWP data\n if not isinstance(timesteps, int):\n n_timesteps = len(timesteps)\n last_timestep = timesteps[-1]\n else:\n n_timesteps = timesteps\n last_timestep = timesteps\n\n nwp_precip = np.zeros((n_models, last_timestep + 1, 200, 200))\n\n if not zero_nwp:\n for n_model in range(n_models):\n for i in range(nwp_precip.shape[1]):\n nwp_precip[n_model, i, 30:185, 30 + 1 * (i + 1) * n_model] = 0.1\n nwp_precip[n_model, i, 30:185, 31 + 1 * (i + 1) * n_model] = 0.1\n nwp_precip[n_model, i, 30:185, 32 + 1 * (i + 1) * n_model] = 1.0\n nwp_precip[n_model, i, 30:185, 33 + 1 * (i + 1) * n_model] = 5.0\n nwp_precip[n_model, i, 30:185, 34 + 1 * (i + 1) * n_model] = 5.0\n nwp_precip[n_model, i, 30:185, 35 + 1 * (i + 1) * n_model] = 4.5\n nwp_precip[n_model, i, 30:185, 36 + 1 * (i + 1) * n_model] = 4.5\n nwp_precip[n_model, i, 30:185, 37 + 1 * (i + 1) * n_model] = 4.0\n nwp_precip[n_model, i, 30:185, 38 + 1 * (i + 1) * n_model] = 2.0\n nwp_precip[n_model, i, 30:185, 39 + 1 * (i + 1) * n_model] = 1.0\n nwp_precip[n_model, i, 30:185, 40 + 1 * (i + 1) * n_model] = 0.5\n nwp_precip[n_model, i, 30:185, 41 + 1 * (i + 1) * n_model] = 0.1\n\n # Define dummy nowcast input data\n radar_precip = np.zeros((3, 200, 200))\n\n if not zero_radar:\n for i in range(2):\n radar_precip[i, 5:150, 30 + 1 * i] = 0.1\n radar_precip[i, 5:150, 31 + 1 * i] = 0.5\n radar_precip[i, 5:150, 32 + 1 * i] = 0.5\n radar_precip[i, 5:150, 33 + 1 * i] = 5.0\n radar_precip[i, 5:150, 34 + 1 * i] = 5.0\n radar_precip[i, 5:150, 35 + 1 * i] = 4.5\n radar_precip[i, 5:150, 36 + 1 * i] = 4.5\n radar_precip[i, 5:150, 37 + 1 * i] = 4.0\n radar_precip[i, 5:150, 38 + 1 * i] = 1.0\n radar_precip[i, 5:150, 39 + 1 * i] = 0.5\n radar_precip[i, 5:150, 40 + 1 * i] = 0.5\n radar_precip[i, 5:150, 41 + 1 * i] = 0.1\n radar_precip[2, 30:155, 30 + 1 * 2] = 0.1\n radar_precip[2, 30:155, 31 + 1 * 2] = 0.1\n radar_precip[2, 30:155, 32 + 1 * 2] = 1.0\n radar_precip[2, 30:155, 33 + 1 * 2] = 5.0\n radar_precip[2, 30:155, 34 + 1 * 2] = 5.0\n radar_precip[2, 30:155, 35 + 1 * 2] = 4.5\n radar_precip[2, 30:155, 36 + 1 * 2] = 4.5\n radar_precip[2, 30:155, 37 + 1 * 2] = 4.0\n radar_precip[2, 30:155, 38 + 1 * 2] = 2.0\n radar_precip[2, 30:155, 39 + 1 * 2] = 1.0\n radar_precip[2, 30:155, 40 + 1 * 3] = 0.5\n radar_precip[2, 30:155, 41 + 1 * 3] = 0.1\n\n precip_nowcast = np.zeros((n_ens_members, last_timestep + 1, 200, 200))\n\n if nowcasting_method == \"external_nowcast_ens\":\n nowcasting_method = \"external_nowcast\"\n for n_ens_member in range(n_ens_members):\n for i in range(precip_nowcast.shape[1]):\n precip_nowcast[\n n_ens_member, i, 30:165, 30 + 1 * (i + 1) * n_ens_member\n ] = 0.1\n precip_nowcast[\n n_ens_member, i, 30:165, 31 + 1 * (i + 1) * n_ens_member\n ] = 0.5\n precip_nowcast[\n n_ens_member, i, 30:165, 32 + 1 * (i + 1) * n_ens_member\n ] = 0.5\n precip_nowcast[\n n_ens_member, i, 30:165, 33 + 1 * (i + 1) * n_ens_member\n ] = 5.0\n precip_nowcast[\n n_ens_member, i, 30:165, 34 + 1 * (i + 1) * n_ens_member\n ] = 5.0\n precip_nowcast[\n n_ens_member, i, 30:165, 35 + 1 * (i + 1) * n_ens_member\n ] = 4.5\n precip_nowcast[\n n_ens_member, i, 30:165, 36 + 1 * (i + 1) * n_ens_member\n ] = 4.5\n precip_nowcast[\n n_ens_member, i, 30:165, 37 + 1 * (i + 1) * n_ens_member\n ] = 4.0\n precip_nowcast[\n n_ens_member, i, 30:165, 38 + 1 * (i + 1) * n_ens_member\n ] = 1.0\n precip_nowcast[\n n_ens_member, i, 30:165, 39 + 1 * (i + 1) * n_ens_member\n ] = 0.5\n precip_nowcast[\n n_ens_member, i, 30:165, 40 + 1 * (i + 1) * n_ens_member\n ] = 0.5\n precip_nowcast[\n n_ens_member, i, 30:165, 41 + 1 * (i + 1) * n_ens_member\n ] = 0.1\n if n_ens_members < expected_n_ens_members:\n n_ens_members = expected_n_ens_members\n\n elif nowcasting_method == \"external_nowcast_det\":\n nowcasting_method = \"external_nowcast\"\n for i in range(precip_nowcast.shape[1]):\n precip_nowcast[0, i, 30:165, 30 + 1 * i] = 0.1\n precip_nowcast[0, i, 30:165, 31 + 1 * i] = 0.5\n precip_nowcast[0, i, 30:165, 32 + 1 * i] = 0.5\n precip_nowcast[0, i, 30:165, 33 + 1 * i] = 5.0\n precip_nowcast[0, i, 30:165, 34 + 1 * i] = 5.0\n precip_nowcast[0, i, 30:165, 35 + 1 * i] = 4.5\n precip_nowcast[0, i, 30:165, 36 + 1 * i] = 4.5\n precip_nowcast[0, i, 30:165, 37 + 1 * i] = 4.0\n precip_nowcast[0, i, 30:165, 38 + 1 * i] = 1.0\n precip_nowcast[0, i, 30:165, 39 + 1 * i] = 0.5\n precip_nowcast[0, i, 30:165, 40 + 1 * i] = 0.5\n precip_nowcast[0, i, 30:165, 41 + 1 * i] = 0.1\n\n metadata = dict()\n metadata[\"unit\"] = \"mm\"\n metadata[\"transformation\"] = \"dB\"\n metadata[\"accutime\"] = 5.0\n metadata[\"transform\"] = \"dB\"\n metadata[\"zerovalue\"] = 0.0\n metadata[\"threshold\"] = 0.01\n metadata[\"zr_a\"] = 200.0\n metadata[\"zr_b\"] = 1.6\n\n # Also set the outdir_path, clim_kwargs and mask_kwargs\n outdir_path_skill = \"./tmp/\"\n if n_models == 1:\n clim_kwargs = None\n else:\n clim_kwargs = dict({\"n_models\": n_models, \"window_length\": 30})\n\n if max_mask_rim is not None:\n mask_kwargs = dict({\"mask_rim\": 10, \"max_mask_rim\": max_mask_rim})\n else:\n mask_kwargs = None\n\n ###\n # First threshold the data and convert it to dBR\n ###\n # threshold the data\n radar_precip[radar_precip < metadata[\"threshold\"]] = 0.0\n nwp_precip[nwp_precip < metadata[\"threshold\"]] = 0.0\n\n # convert the data\n converter = pysteps.utils.get_method(\"mm/h\")\n radar_precip, _ = converter(radar_precip, metadata)\n nwp_precip, metadata = converter(nwp_precip, metadata)\n\n # transform the data\n transformer = pysteps.utils.get_method(metadata[\"transformation\"])\n radar_precip, _ = transformer(radar_precip, metadata)\n nwp_precip, metadata = transformer(nwp_precip, metadata)\n\n # set NaN equal to zero\n radar_precip[~np.isfinite(radar_precip)] = metadata[\"zerovalue\"]\n nwp_precip[~np.isfinite(nwp_precip)] = metadata[\"zerovalue\"]\n\n assert (\n np.any(~np.isfinite(radar_precip)) == False\n ), \"There are still infinite values in the input radar data\"\n assert (\n np.any(~np.isfinite(nwp_precip)) == False\n ), \"There are still infinite values in the NWP data\"\n\n ###\n # Decompose the R_NWP data\n ###\n\n # Initial decomposition settings\n decomp_method, _ = cascade.get_method(\"fft\")\n bandpass_filter_method = \"gaussian\"\n precip_shape = radar_precip.shape[1:]\n filter_method = cascade.get_method(bandpass_filter_method)\n bp_filter = filter_method(precip_shape, n_cascade_levels)\n\n # If we only use one model:\n if nwp_precip.ndim == 3:\n nwp_precip = nwp_precip[None, :]\n\n if decomposed_nwp:\n nwp_precip_decomp = []\n # Loop through the n_models\n for i in range(nwp_precip.shape[0]):\n R_d_models_ = []\n # Loop through the time steps\n for j in range(nwp_precip.shape[1]):\n R_ = decomp_method(\n field=nwp_precip[i, j, :, :],\n bp_filter=bp_filter,\n normalize=True,\n compute_stats=True,\n compact_output=True,\n )\n R_d_models_.append(R_)\n nwp_precip_decomp.append(R_d_models_)\n\n nwp_precip_decomp = np.array(nwp_precip_decomp)\n\n assert nwp_precip_decomp.ndim == 2, \"Wrong number of dimensions in R_d_models\"\n\n else:\n nwp_precip_decomp = nwp_precip.copy()\n\n assert nwp_precip_decomp.ndim == 4, \"Wrong number of dimensions in R_d_models\"\n\n ###\n # Determine the velocity fields\n ###\n oflow_method = pysteps.motion.get_method(\"lucaskanade\")\n radar_velocity = oflow_method(radar_precip)\n nwp_velocity = []\n # Loop through the models\n for n_model in range(nwp_precip.shape[0]):\n # Loop through the timesteps. We need two images to construct a motion\n # field, so we can start from timestep 1. Timestep 0 will be the same\n # as timestep 0.\n _V_NWP_ = []\n for t in range(1, nwp_precip.shape[1]):\n V_NWP_ = oflow_method(nwp_precip[n_model, t - 1 : t + 1, :])\n _V_NWP_.append(V_NWP_)\n V_NWP_ = None\n _V_NWP_ = np.insert(_V_NWP_, 0, _V_NWP_[0], axis=0)\n nwp_velocity.append(_V_NWP_)\n\n nwp_velocity = np.stack(nwp_velocity)\n\n assert nwp_velocity.ndim == 5, \"nwp_velocity must be a five-dimensional array\"\n\n ###\n # The blending\n ###\n precip_forecast = blending.steps.forecast(\n precip=radar_precip,\n precip_models=nwp_precip_decomp,\n velocity=radar_velocity,\n velocity_models=nwp_velocity,\n timesteps=timesteps,\n timestep=5.0,\n issuetime=datetime.datetime.strptime(\"202112012355\", \"%Y%m%d%H%M\"),\n n_ens_members=n_ens_members,\n n_cascade_levels=n_cascade_levels,\n blend_nwp_members=blend_nwp_members,\n precip_thr=metadata[\"threshold\"],\n kmperpixel=1.0,\n extrap_method=\"semilagrangian\",\n decomp_method=\"fft\",\n bandpass_filter_method=\"gaussian\",\n noise_method=\"nonparametric\",\n noise_stddev_adj=\"auto\",\n ar_order=2,\n vel_pert_method=vel_pert_method,\n weights_method=weights_method,\n timestep_start_full_nwp_weight=timestep_start_full_nwp_weight,\n conditional=False,\n probmatching_method=probmatching_method,\n mask_method=mask_method,\n resample_distribution=resample_distribution,\n smooth_radar_mask_range=smooth_radar_mask_range,\n callback=None,\n return_output=True,\n seed=None,\n num_workers=1,\n fft_method=\"numpy\",\n domain=\"spatial\",\n outdir_path_skill=outdir_path_skill,\n extrap_kwargs=None,\n filter_kwargs=None,\n noise_kwargs=None,\n vel_pert_kwargs=None,\n clim_kwargs=clim_kwargs,\n mask_kwargs=mask_kwargs,\n measure_time=False,\n )\n\n assert precip_forecast.ndim == 4, \"Wrong amount of dimensions in forecast output\"\n\n assert (\n precip_forecast.shape[0] == expected_n_ens_members\n ), \"Wrong amount of output ensemble members in forecast output\"\n\n assert (\n precip_forecast.shape[1] == n_timesteps\n ), \"Wrong amount of output time steps in forecast output\"\n\n # Transform the data back into mm/h\n precip_forecast, _ = converter(precip_forecast, metadata)\n\n assert (\n precip_forecast.ndim == 4\n ), \"Wrong amount of dimensions in converted forecast output\"\n\n assert (\n precip_forecast.shape[0] == expected_n_ens_members\n ), \"Wrong amount of output ensemble members in converted forecast output\"\n\n assert (\n precip_forecast.shape[1] == n_timesteps\n ), \"Wrong amount of output time steps in converted forecast output\"\n" }, { "path": "pysteps/tests/test_blending_utils.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nimport pysteps\nfrom pysteps.blending.utils import (\n blend_cascades,\n blend_optical_flows,\n compute_smooth_dilated_mask,\n compute_store_nwp_motion,\n decompose_NWP,\n load_NWP,\n recompose_cascade,\n stack_cascades,\n)\nfrom pysteps.utils.check_norain import check_norain\n\npytest.importorskip(\"netCDF4\")\n\nprecip_nwp = np.zeros((24, 564, 564))\n\nfor t in range(precip_nwp.shape[0]):\n precip_nwp[t, 30 + t : 185 + t, 30 + 2 * t] = 0.1\n precip_nwp[t, 30 + t : 185 + t, 31 + 2 * t] = 0.1\n precip_nwp[t, 30 + t : 185 + t, 32 + 2 * t] = 1.0\n precip_nwp[t, 30 + t : 185 + t, 33 + 2 * t] = 5.0\n precip_nwp[t, 30 + t : 185 + t, 34 + 2 * t] = 5.0\n precip_nwp[t, 30 + t : 185 + t, 35 + 2 * t] = 4.5\n precip_nwp[t, 30 + t : 185 + t, 36 + 2 * t] = 4.5\n precip_nwp[t, 30 + t : 185 + t, 37 + 2 * t] = 4.0\n precip_nwp[t, 30 + t : 185 + t, 38 + 2 * t] = 2.0\n precip_nwp[t, 30 + t : 185 + t, 39 + 2 * t] = 1.0\n precip_nwp[t, 30 + t : 185 + t, 40 + 2 * t] = 0.5\n precip_nwp[t, 30 + t : 185 + t, 41 + 2 * t] = 0.1\n\nnwp_proj = (\n \"+proj=lcc +lon_0=4.55 +lat_1=50.8 +lat_2=50.8 \"\n \"+a=6371229 +es=0 +lat_0=50.8 +x_0=365950 +y_0=-365950.000000001\"\n)\n\nnwp_metadata = dict(\n projection=nwp_proj,\n institution=\"Royal Meteorological Institute of Belgium\",\n transform=None,\n zerovalue=0.0,\n threshold=0,\n unit=\"mm\",\n accutime=5,\n xpixelsize=1300.0,\n ypixelsize=1300.0,\n yorigin=\"upper\",\n cartesian_unit=\"m\",\n x1=0.0,\n x2=731900.0,\n y1=-731900.0,\n y2=0.0,\n)\n\n# Get the analysis time and valid time\ntimes_nwp = np.array(\n [\n \"2021-07-04T16:05:00.000000000\",\n \"2021-07-04T16:10:00.000000000\",\n \"2021-07-04T16:15:00.000000000\",\n \"2021-07-04T16:20:00.000000000\",\n \"2021-07-04T16:25:00.000000000\",\n \"2021-07-04T16:30:00.000000000\",\n \"2021-07-04T16:35:00.000000000\",\n \"2021-07-04T16:40:00.000000000\",\n \"2021-07-04T16:45:00.000000000\",\n \"2021-07-04T16:50:00.000000000\",\n \"2021-07-04T16:55:00.000000000\",\n \"2021-07-04T17:00:00.000000000\",\n \"2021-07-04T17:05:00.000000000\",\n \"2021-07-04T17:10:00.000000000\",\n \"2021-07-04T17:15:00.000000000\",\n \"2021-07-04T17:20:00.000000000\",\n \"2021-07-04T17:25:00.000000000\",\n \"2021-07-04T17:30:00.000000000\",\n \"2021-07-04T17:35:00.000000000\",\n \"2021-07-04T17:40:00.000000000\",\n \"2021-07-04T17:45:00.000000000\",\n \"2021-07-04T17:50:00.000000000\",\n \"2021-07-04T17:55:00.000000000\",\n \"2021-07-04T18:00:00.000000000\",\n ],\n dtype=\"datetime64[ns]\",\n)\n\n\n# Prepare input NWP files\n# Convert to rain rates [mm/h]\nconverter = pysteps.utils.get_method(\"mm/h\")\nprecip_nwp, nwp_metadata = converter(precip_nwp, nwp_metadata)\n\n# Threshold the data\nnwp_metadata[\"threshold\"] = 0.1\nprecip_nwp[precip_nwp < nwp_metadata[\"threshold\"]] = 0.0\n\n# Transform the data\ntransformer = pysteps.utils.get_method(\"dB\")\nprecip_nwp, nwp_metadata = transformer(\n precip_nwp, nwp_metadata, threshold=nwp_metadata[\"threshold\"]\n)\n\n# Set two issue times for testing\nissue_time_first = times_nwp[0]\nissue_time_second = times_nwp[3]\n\n# Set the blending weights (we'll blend with a 50-50 weight)\nweights = np.full((2, 8), fill_value=0.5)\n\n# Set the testing arguments\n# Test function arguments\nutils_arg_names = (\n \"precip_nwp\",\n \"nwp_model\",\n \"issue_times\",\n \"timestep\",\n \"n_timesteps\",\n \"valid_times\",\n \"shape\",\n \"weights\",\n)\n\n# Test function values\nutils_arg_values = [\n (\n precip_nwp,\n \"test\",\n [issue_time_first, issue_time_second],\n 5.0,\n 3,\n times_nwp,\n precip_nwp.shape[1:],\n weights,\n )\n]\n\nsmoothing_arg_names = (\n \"precip_nwp\",\n \"max_padding_size_in_px\",\n \"gaussian_kernel_size\",\n \"inverted\",\n \"non_linear_growth_kernel_sizes\",\n)\n\nsmoothing_arg_values = [\n (precip_nwp, 80, 9, False, False),\n (precip_nwp, 10, 9, False, False),\n (precip_nwp, 80, 5, False, False),\n (precip_nwp, 80, 9, True, False),\n (precip_nwp, 80, 9, False, True),\n]\n\n\n###\n# The test\n###\n@pytest.mark.parametrize(utils_arg_names, utils_arg_values)\n# The test function to be used\ndef test_blending_utils(\n precip_nwp,\n nwp_model,\n issue_times,\n timestep,\n n_timesteps,\n valid_times,\n shape,\n weights,\n):\n \"\"\"Tests if all blending utils functions behave correctly.\"\"\"\n\n # First, make the output path if it does not exist yet\n tmpdir = \"./tmp/\"\n os.makedirs(tmpdir, exist_ok=True)\n\n # Get the optical flow method\n oflow_method = pysteps.motion.get_method(\"lucaskanade\")\n\n ###\n # Compute and store the motion\n ###\n compute_store_nwp_motion(\n precip_nwp=precip_nwp,\n oflow_method=oflow_method,\n analysis_time=valid_times[0],\n nwp_model=nwp_model,\n output_path=tmpdir,\n )\n\n # Check if file exists\n date_string = np.datetime_as_string(valid_times[0], \"s\")\n motion_file = os.path.join(\n tmpdir,\n \"motion_\"\n + nwp_model\n + \"_\"\n + date_string[:4]\n + date_string[5:7]\n + date_string[8:10]\n + date_string[11:13]\n + date_string[14:16]\n + date_string[17:19]\n + \".npy\",\n )\n assert os.path.exists(motion_file)\n\n ###\n # Decompose and store NWP forecast\n ###\n decompose_NWP(\n R_NWP=precip_nwp,\n NWP_model=nwp_model,\n analysis_time=valid_times[0],\n timestep=timestep,\n valid_times=valid_times,\n num_cascade_levels=8,\n num_workers=1,\n output_path=tmpdir,\n decomp_method=\"fft\",\n fft_method=\"numpy\",\n domain=\"spatial\",\n normalize=True,\n compute_stats=True,\n compact_output=False,\n )\n\n # Check if file exists\n decomp_file = os.path.join(\n tmpdir,\n \"cascade_\"\n + nwp_model\n + \"_\"\n + date_string[:4]\n + date_string[5:7]\n + date_string[8:10]\n + date_string[11:13]\n + date_string[14:16]\n + date_string[17:19]\n + \".nc\",\n )\n assert os.path.exists(decomp_file)\n\n ###\n # Now check if files load correctly for two different issue times\n ###\n precip_decomposed_nwp_first, v_nwp_first = load_NWP(\n input_nc_path_decomp=os.path.join(decomp_file),\n input_path_velocities=os.path.join(motion_file),\n start_time=issue_times[0],\n n_timesteps=n_timesteps,\n )\n\n precip_decomposed_nwp_second, v_nwp_second = load_NWP(\n input_nc_path_decomp=os.path.join(decomp_file),\n input_path_velocities=os.path.join(motion_file),\n start_time=issue_times[1],\n n_timesteps=n_timesteps,\n )\n\n # Check if the output type and shapes are correct\n assert isinstance(precip_decomposed_nwp_first, list)\n assert isinstance(precip_decomposed_nwp_second, list)\n assert isinstance(precip_decomposed_nwp_first[0], dict)\n assert isinstance(precip_decomposed_nwp_second[0], dict)\n\n assert \"domain\" in precip_decomposed_nwp_first[0]\n assert \"normalized\" in precip_decomposed_nwp_first[0]\n assert \"compact_output\" in precip_decomposed_nwp_first[0]\n assert \"valid_times\" in precip_decomposed_nwp_first[0]\n assert \"cascade_levels\" in precip_decomposed_nwp_first[0]\n assert \"means\" in precip_decomposed_nwp_first[0]\n assert \"stds\" in precip_decomposed_nwp_first[0]\n\n assert precip_decomposed_nwp_first[0][\"cascade_levels\"].shape == (\n 8,\n shape[0],\n shape[1],\n )\n assert precip_decomposed_nwp_first[0][\"domain\"] == \"spatial\"\n assert precip_decomposed_nwp_first[0][\"normalized\"] == True\n assert precip_decomposed_nwp_first[0][\"compact_output\"] == False\n assert len(precip_decomposed_nwp_first) == n_timesteps + 1\n assert len(precip_decomposed_nwp_second) == n_timesteps + 1\n assert precip_decomposed_nwp_first[0][\"means\"].shape[0] == 8\n assert precip_decomposed_nwp_first[0][\"stds\"].shape[0] == 8\n\n assert np.array(v_nwp_first).shape == (n_timesteps + 1, 2, shape[0], shape[1])\n assert np.array(v_nwp_second).shape == (n_timesteps + 1, 2, shape[0], shape[1])\n\n # Check if the right times are loaded\n assert (\n precip_decomposed_nwp_first[0][\"valid_times\"][0] == valid_times[0]\n ), \"Not the right valid times were loaded for the first forecast\"\n assert (\n precip_decomposed_nwp_second[0][\"valid_times\"][0] == valid_times[3]\n ), \"Not the right valid times were loaded for the second forecast\"\n\n # Check, for a sample, if the stored motion fields are as expected\n assert_array_almost_equal(\n v_nwp_first[1],\n oflow_method(precip_nwp[0:2, :, :]),\n decimal=3,\n err_msg=\"Stored motion field of first forecast not equal to expected motion field\",\n )\n assert_array_almost_equal(\n v_nwp_second[1],\n oflow_method(precip_nwp[3:5, :, :]),\n decimal=3,\n err_msg=\"Stored motion field of second forecast not equal to expected motion field\",\n )\n\n ###\n # Stack the cascades\n ###\n precip_decomposed_first_stack, mu_first_stack, sigma_first_stack = stack_cascades(\n R_d=precip_decomposed_nwp_first, donorm=False\n )\n\n print(precip_decomposed_nwp_first)\n print(precip_decomposed_first_stack)\n print(mu_first_stack)\n\n (\n precip_decomposed_second_stack,\n mu_second_stack,\n sigma_second_stack,\n ) = stack_cascades(R_d=precip_decomposed_nwp_second, donorm=False)\n\n # Check if the array shapes are still correct\n assert precip_decomposed_first_stack.shape == (\n n_timesteps + 1,\n 8,\n shape[0],\n shape[1],\n )\n assert mu_first_stack.shape == (n_timesteps + 1, 8)\n assert sigma_first_stack.shape == (n_timesteps + 1, 8)\n\n ###\n # Blend the cascades\n ###\n precip_decomposed_blended = blend_cascades(\n cascades_norm=np.stack(\n (precip_decomposed_first_stack[0], precip_decomposed_second_stack[0])\n ),\n weights=weights,\n )\n\n assert precip_decomposed_blended.shape == precip_decomposed_first_stack[0].shape\n\n ###\n # Blend the optical flow fields\n ###\n v_nwp_blended = blend_optical_flows(\n flows=np.stack((v_nwp_first[1], v_nwp_second[1])), weights=weights[:, 1]\n )\n\n assert v_nwp_blended.shape == v_nwp_first[1].shape\n assert_array_almost_equal(\n v_nwp_blended,\n (oflow_method(precip_nwp[0:2, :, :]) + oflow_method(precip_nwp[3:5, :, :])) / 2,\n decimal=3,\n err_msg=\"Blended motion field does not equal average of the two motion fields\",\n )\n\n ###\n # Recompose the fields (the non-blended fields are used for this here)\n ###\n precip_recomposed_first = recompose_cascade(\n combined_cascade=precip_decomposed_first_stack[0],\n combined_mean=mu_first_stack[0],\n combined_sigma=sigma_first_stack[0],\n )\n precip_recomposed_second = recompose_cascade(\n combined_cascade=precip_decomposed_second_stack[0],\n combined_mean=mu_second_stack[0],\n combined_sigma=sigma_second_stack[0],\n )\n\n assert_array_almost_equal(\n precip_recomposed_first,\n precip_nwp[0, :, :],\n decimal=3,\n err_msg=\"Recomposed field of first forecast does not equal original field\",\n )\n assert_array_almost_equal(\n precip_recomposed_second,\n precip_nwp[3, :, :],\n decimal=3,\n err_msg=\"Recomposed field of second forecast does not equal original field\",\n )\n\n precip_arr = precip_nwp\n # rainy fraction is 0.005847\n assert not check_norain(precip_arr, win_fun=None)\n assert not check_norain(\n precip_arr, precip_thr=nwp_metadata[\"threshold\"], win_fun=None\n )\n assert not check_norain(\n precip_arr, precip_thr=nwp_metadata[\"threshold\"], norain_thr=0.005, win_fun=None\n )\n assert not check_norain(precip_arr, norain_thr=0.005, win_fun=None)\n # so with norain_thr beyond this number it should report that there's no rain\n assert check_norain(precip_arr, norain_thr=0.006, win_fun=None)\n assert check_norain(\n precip_arr, precip_thr=nwp_metadata[\"threshold\"], norain_thr=0.006, win_fun=None\n )\n\n # also if we set the precipitation threshold sufficiently high, it should report there's no rain\n # rainy fraction > 4mm/h is 0.004385\n assert not check_norain(precip_arr, precip_thr=4.0, norain_thr=0.004, win_fun=None)\n assert check_norain(precip_arr, precip_thr=4.0, norain_thr=0.005, win_fun=None)\n\n # no rain above 100mm/h so it should give norain\n assert check_norain(precip_arr, precip_thr=100, win_fun=None)\n\n # should always give norain if the threshold is set to 100%\n assert check_norain(precip_arr, norain_thr=1.0, win_fun=None)\n\n\n# Finally, also test the compute_smooth_dilated mask functionality\n@pytest.mark.parametrize(smoothing_arg_names, smoothing_arg_values)\ndef test_blending_smoothing_utils(\n precip_nwp,\n max_padding_size_in_px,\n gaussian_kernel_size,\n inverted,\n non_linear_growth_kernel_sizes,\n):\n # First add some nans to indicate a mask\n precip_nwp[:, 0:100, 0:100] = np.nan\n nan_indices = np.isnan(precip_nwp[0])\n new_mask = compute_smooth_dilated_mask(\n nan_indices,\n max_padding_size_in_px=max_padding_size_in_px,\n gaussian_kernel_size=gaussian_kernel_size,\n inverted=inverted,\n non_linear_growth_kernel_sizes=non_linear_growth_kernel_sizes,\n )\n assert new_mask.shape == nan_indices.shape\n\n if max_padding_size_in_px > 0 and inverted == False:\n assert np.sum((new_mask > 0) & (new_mask < 1)) > 0\n" }, { "path": "pysteps/tests/test_cascade.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nimport pysteps\nfrom pysteps import nowcasts\nfrom pysteps.cascade.bandpass_filters import filter_gaussian\nfrom pysteps.cascade.bandpass_filters import filter_uniform\nfrom pysteps.cascade.decomposition import decomposition_fft, recompose_fft\nfrom pysteps.tests.helpers import smart_assert\n\n\ndef test_decompose_recompose():\n \"\"\"Tests cascade decomposition.\"\"\"\n\n pytest.importorskip(\"netCDF4\")\n\n root_path = pysteps.rcparams.data_sources[\"bom\"][\"root_path\"]\n rel_path = os.path.join(\"prcp-cscn\", \"2\", \"2018\", \"06\", \"16\")\n filename = os.path.join(root_path, rel_path, \"2_20180616_120000.prcp-cscn.nc\")\n precip, _, metadata = pysteps.io.import_bom_rf3(filename)\n\n # Convert to rain rate from mm\n precip, metadata = pysteps.utils.to_rainrate(precip, metadata)\n\n # Log-transform the data\n precip, metadata = pysteps.utils.dB_transform(\n precip, metadata, threshold=0.1, zerovalue=-15.0\n )\n\n # Set Nans as the fill value\n precip[~np.isfinite(precip)] = metadata[\"zerovalue\"]\n\n # Set number of cascade levels\n num_cascade_levels = 9\n\n # Construct the Gaussian bandpass filters\n _filter = filter_gaussian(precip.shape, num_cascade_levels)\n\n # Decompose precip\n decomp = decomposition_fft(precip, _filter)\n\n # Recomposed precip from decomp\n recomposed = recompose_fft(decomp)\n # Assert\n assert_array_almost_equal(recomposed.squeeze(), precip)\n\n\ntest_metadata_filter = [\n (\"central_freqs\", None, None),\n (\"central_wavenumbers\", None, None),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_metadata_filter)\ndef test_filter_uniform(variable, expected, tolerance):\n _filter = filter_uniform((8, 8), 1)\n smart_assert(_filter[variable], expected, tolerance)\n\n\ndef test_filter_uniform_weights_1d():\n _filter = filter_uniform((8, 8), 1)\n assert_array_almost_equal(_filter[\"weights_1d\"], np.ones((1, 5)))\n\n\ndef test_filter_uniform_weights_2d():\n _filter = filter_uniform((8, 8), 1)\n assert_array_almost_equal(_filter[\"weights_2d\"], np.ones((1, 8, 5)))\n" }, { "path": "pysteps/tests/test_datasets.py", "content": "# -*- coding: utf-8 -*-\nimport os\nfrom tempfile import TemporaryDirectory\n\nimport pytest\nfrom _pytest.outcomes import Skipped\n\nimport pysteps\nfrom pysteps.datasets import (\n download_pysteps_data,\n create_default_pystepsrc,\n load_dataset,\n)\nfrom pysteps.exceptions import DirectoryNotEmpty\n\n_datasets_opt_deps = dict(\n fmi=[\"pyproj\"],\n mch=[\"PIL\"],\n bom=[\"netCDF4\"],\n knmi=[\"h5py\"],\n opera=[\"h5py\"],\n mrms=[\"pygrib\"],\n)\n\n\n@pytest.mark.parametrize(\"case_name\", _datasets_opt_deps.keys())\ndef test_load_dataset(case_name):\n \"\"\"Test the load dataset function.\"\"\"\n\n with pytest.raises(ValueError):\n load_dataset(frames=100)\n\n for mod_name in _datasets_opt_deps[case_name]:\n pytest.importorskip(mod_name)\n\n try:\n load_dataset(case=case_name, frames=1)\n except Skipped:\n pass\n\n\ndef _test_download_data():\n \"\"\"Test the example data installers.\"\"\"\n temp_dir = TemporaryDirectory()\n\n try:\n download_pysteps_data(temp_dir.name, force=True)\n with pytest.raises(DirectoryNotEmpty):\n download_pysteps_data(temp_dir.name, force=False)\n\n params_file = create_default_pystepsrc(temp_dir.name, config_dir=temp_dir.name)\n\n pysteps.load_config_file(params_file)\n\n finally:\n temp_dir.cleanup()\n pysteps.load_config_file()\n\n\ndef _default_path():\n \"\"\"\n Default pystepsrc path.\"\"\"\n home_dir = os.path.expanduser(\"~\")\n if os.name == \"nt\":\n subdir = \"pysteps\"\n else:\n subdir = \".pysteps\"\n return os.path.join(home_dir, subdir, \"pystepsrc\")\n\n\ntest_params_paths = [\n (None, \"pystepsrc\", _default_path()),\n (\"/root/path\", \"pystepsrc\", \"/root/path/pystepsrc\"),\n (\"/root/path\", \"pystepsrc2\", \"/root/path/pystepsrc2\"),\n (\"relative/path\", \"pystepsrc2\", \"relative/path/pystepsrc2\"),\n (\"relative/path\", \"pystepsrc\", \"relative/path/pystepsrc\"),\n]\n\n\n@pytest.mark.parametrize(\"config_dir, file_name, expected_path\", test_params_paths)\ndef test_params_file_creation_path(config_dir, file_name, expected_path):\n \"\"\"Test that the default pysteps parameters file is created in the right place.\"\"\"\n\n # For windows compatibility\n if config_dir is not None:\n config_dir = os.path.normpath(config_dir)\n expected_path = os.path.normpath(expected_path)\n\n pysteps_data_dir = \"dummy/path/to/data\"\n params_file_path = create_default_pystepsrc(\n pysteps_data_dir, config_dir=config_dir, file_name=file_name, dryrun=True\n )\n\n assert expected_path == params_file_path\n" }, { "path": "pysteps/tests/test_decorators.py", "content": "# -*- coding: utf-8 -*-\nimport time\n\nfrom pysteps.decorators import memoize\n\n\ndef test_memoize():\n @memoize(maxsize=1)\n def _slow_function(x, **kwargs):\n time.sleep(1)\n return x\n\n for i in range(2):\n out = _slow_function(i, hkey=i)\n assert out == i\n\n # cached result\n t0 = time.monotonic()\n out = _slow_function(1, hkey=1)\n assert time.monotonic() - t0 < 1\n assert out == 1\n\n # maxsize exceeded\n t0 = time.monotonic()\n out = _slow_function(0, hkey=0)\n assert time.monotonic() - t0 >= 1\n assert out == 0\n\n # no hash\n t0 = time.monotonic()\n out = _slow_function(1)\n assert time.monotonic() - t0 >= 1\n assert out == 1\n" }, { "path": "pysteps/tests/test_downscaling_rainfarm.py", "content": "# -*- coding: utf-8 -*-\n\nimport pytest\nimport numpy as np\nfrom pysteps import downscaling\nfrom pysteps.tests.helpers import get_precipitation_fields\nfrom pysteps.utils import aggregate_fields_space, square_domain, aggregate_fields\n\n\n@pytest.fixture(scope=\"module\")\ndef data():\n precip, metadata = get_precipitation_fields(\n num_prev_files=0, num_next_files=0, return_raw=False, metadata=True\n )\n precip = precip.filled()\n precip, metadata = square_domain(precip, metadata, \"crop\")\n return precip, metadata\n\n\nrainfarm_arg_names = (\n \"alpha\",\n \"ds_factor\",\n \"threshold\",\n \"return_alpha\",\n \"spectral_fusion\",\n \"kernel_type\",\n)\nrainfarm_arg_values = [\n (1.0, 1, 0, False, False, None),\n (1, 2, 0, False, False, \"gaussian\"),\n (1, 4, 0, False, False, \"tophat\"),\n (1, 4, 0, False, True, \"uniform\"),\n]\n\n\n@pytest.mark.parametrize(rainfarm_arg_names, rainfarm_arg_values)\ndef test_rainfarm_shape(\n data,\n alpha,\n ds_factor,\n threshold,\n return_alpha,\n spectral_fusion,\n kernel_type,\n):\n \"\"\"Test that the output of rainfarm is consistent with the downscaling factor.\"\"\"\n precip, metadata = data\n window = metadata[\"xpixelsize\"] * ds_factor\n precip_lr, __ = aggregate_fields_space(precip, metadata, window)\n\n rainfarm = downscaling.get_method(\"rainfarm\")\n precip_hr = rainfarm(\n precip_lr,\n alpha=alpha,\n ds_factor=ds_factor,\n threshold=threshold,\n return_alpha=return_alpha,\n spectral_fusion=spectral_fusion,\n kernel_type=kernel_type,\n )\n\n assert precip_hr.ndim == precip.ndim\n assert precip_hr.shape[0] == precip.shape[0]\n assert precip_hr.shape[1] == precip.shape[1]\n\n\nrainfarm_arg_values = [\n (1.0, 1, 0, False, False, None),\n (1, 2, 0, False, False, None),\n (1, 4, 0, False, False, None),\n (1, 4, 0, False, True, None),\n]\n\n\n@pytest.mark.parametrize(rainfarm_arg_names, rainfarm_arg_values)\ndef test_rainfarm_aggregate(\n data,\n alpha,\n ds_factor,\n threshold,\n return_alpha,\n spectral_fusion,\n kernel_type,\n):\n \"\"\"Test that the output of rainfarm is equal to original when aggregated.\"\"\"\n precip, metadata = data\n window = metadata[\"xpixelsize\"] * ds_factor\n precip_lr, __ = aggregate_fields_space(precip, metadata, window)\n\n rainfarm = downscaling.get_method(\"rainfarm\")\n precip_hr = rainfarm(\n precip_lr,\n alpha=alpha,\n ds_factor=ds_factor,\n threshold=threshold,\n return_alpha=return_alpha,\n spectral_fusion=spectral_fusion,\n kernel_type=kernel_type,\n )\n precip_low = aggregate_fields(precip_hr, ds_factor, axis=(0, 1))\n precip_lr[precip_lr < threshold] = 0.0\n\n np.testing.assert_array_almost_equal(precip_lr, precip_low)\n\n\nrainfarm_arg_values = [(1.0, 2, 0, True, False, None), (None, 2, 0, True, True, None)]\n\n\n@pytest.mark.parametrize(rainfarm_arg_names, rainfarm_arg_values)\ndef test_rainfarm_alpha(\n data,\n alpha,\n ds_factor,\n threshold,\n return_alpha,\n spectral_fusion,\n kernel_type,\n):\n \"\"\"Test that rainfarm computes and returns alpha.\"\"\"\n precip, metadata = data\n window = metadata[\"xpixelsize\"] * ds_factor\n precip_lr, __ = aggregate_fields_space(precip, metadata, window)\n\n rainfarm = downscaling.get_method(\"rainfarm\")\n precip_hr = rainfarm(\n precip_lr,\n alpha=alpha,\n ds_factor=ds_factor,\n threshold=threshold,\n return_alpha=return_alpha,\n spectral_fusion=spectral_fusion,\n kernel_type=kernel_type,\n )\n\n assert len(precip_hr) == 2\n if alpha is None:\n assert not precip_hr[1] == alpha\n else:\n assert precip_hr[1] == alpha\n" }, { "path": "pysteps/tests/test_ensscores.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.tests.helpers import get_precipitation_fields\nfrom pysteps.verification import ensscores\n\nprecip = get_precipitation_fields(num_next_files=10, return_raw=True)\nnp.random.seed(42)\n\n# rankhist\ntest_data = [\n (precip[:10], precip[-1], None, True, 11),\n (precip[:10], precip[-1], None, False, 11),\n]\n\n\n@pytest.mark.parametrize(\"X_f, X_o, X_min, normalize, expected\", test_data)\ndef test_rankhist_size(X_f, X_o, X_min, normalize, expected):\n \"\"\"Test the rankhist.\"\"\"\n assert_array_almost_equal(\n ensscores.rankhist(X_f, X_o, X_min, normalize).size, expected\n )\n\n\n# ensemble_skill\ntest_data = [\n (\n precip[:10],\n precip[-1],\n \"RMSE\",\n {\"axis\": None, \"conditioning\": \"single\"},\n 0.26054151,\n ),\n (precip[:10], precip[-1], \"CSI\", {\"thr\": 1.0, \"axis\": None}, 0.22017924),\n (precip[:10], precip[-1], \"FSS\", {\"thr\": 1.0, \"scale\": 10}, 0.63239752),\n]\n\n\n@pytest.mark.parametrize(\"X_f, X_o, metric, kwargs, expected\", test_data)\ndef test_ensemble_skill(X_f, X_o, metric, kwargs, expected):\n \"\"\"Test the ensemble_skill.\"\"\"\n assert_array_almost_equal(\n ensscores.ensemble_skill(X_f, X_o, metric, **kwargs), expected\n )\n\n\n# ensemble_spread\ntest_data = [\n (precip, \"RMSE\", {\"axis\": None, \"conditioning\": \"single\"}, 0.22635757),\n (precip, \"CSI\", {\"thr\": 1.0, \"axis\": None}, 0.25218158),\n (precip, \"FSS\", {\"thr\": 1.0, \"scale\": 10}, 0.70235667),\n]\n\n\n@pytest.mark.parametrize(\"X_f, metric, kwargs, expected\", test_data)\ndef test_ensemble_spread(X_f, metric, kwargs, expected):\n \"\"\"Test the ensemble_spread.\"\"\"\n assert_array_almost_equal(\n ensscores.ensemble_spread(X_f, metric, **kwargs), expected\n )\n" }, { "path": "pysteps/tests/test_exporters.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\nimport tempfile\nfrom datetime import datetime\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.io import import_netcdf_pysteps\nfrom pysteps.io.exporters import _get_geotiff_filename\nfrom pysteps.io.exporters import close_forecast_files\nfrom pysteps.io.exporters import export_forecast_dataset\nfrom pysteps.io.exporters import initialize_forecast_exporter_netcdf\nfrom pysteps.io.exporters import _convert_proj4_to_grid_mapping\nfrom pysteps.tests.helpers import get_precipitation_fields, get_invalid_mask\n\n# Test arguments\nexporter_arg_names = (\n \"n_ens_members\",\n \"incremental\",\n \"datatype\",\n \"fill_value\",\n \"scale_factor\",\n \"offset\",\n \"n_timesteps\",\n)\n\nexporter_arg_values = [\n (1, None, np.float32, None, None, None, 3),\n (1, \"timestep\", np.float32, 65535, None, None, 3),\n (2, None, np.float32, 65535, None, None, 3),\n (2, None, np.float32, 65535, None, None, [1, 2, 4]),\n (2, \"timestep\", np.float32, None, None, None, 3),\n (2, \"timestep\", np.float32, None, None, None, [1, 2, 4]),\n (2, \"member\", np.float64, None, 0.01, 1.0, 3),\n]\n\n\ndef test_get_geotiff_filename():\n \"\"\"Test the geotif name generator.\"\"\"\n\n start_date = datetime.strptime(\"201909082022\", \"%Y%m%d%H%M\")\n\n n_timesteps = 50\n timestep = 5\n\n for timestep_index in range(n_timesteps):\n file_name = _get_geotiff_filename(\n \"test/path\", start_date, n_timesteps, timestep, timestep_index\n )\n expected = (\n f\"test/path_201909082022_\" f\"{(timestep_index + 1) * timestep:03d}.tif\"\n )\n assert expected == file_name\n\n\n@pytest.mark.parametrize(exporter_arg_names, exporter_arg_values)\ndef test_io_export_netcdf_one_member_one_time_step(\n n_ens_members, incremental, datatype, fill_value, scale_factor, offset, n_timesteps\n):\n \"\"\"\n Test the export netcdf.\n Also, test that the exported file can be read by the importer.\n \"\"\"\n\n pytest.importorskip(\"pyproj\")\n\n precip, metadata = get_precipitation_fields(\n num_prev_files=2, return_raw=True, metadata=True, source=\"fmi\"\n )\n\n invalid_mask = get_invalid_mask(precip)\n\n with tempfile.TemporaryDirectory() as outpath:\n # save it back to disk\n outfnprefix = \"test_netcdf_out\"\n file_path = os.path.join(outpath, outfnprefix + \".nc\")\n startdate = metadata[\"timestamps\"][0]\n timestep = metadata[\"accutime\"]\n shape = precip.shape[1:]\n\n exporter = initialize_forecast_exporter_netcdf(\n outpath,\n outfnprefix,\n startdate,\n timestep,\n n_timesteps,\n shape,\n metadata,\n n_ens_members=n_ens_members,\n datatype=datatype,\n incremental=incremental,\n fill_value=fill_value,\n scale_factor=scale_factor,\n offset=offset,\n )\n\n if n_ens_members > 1:\n precip = np.repeat(precip[np.newaxis, :, :, :], n_ens_members, axis=0)\n\n if incremental == None:\n export_forecast_dataset(precip, exporter)\n if incremental == \"timestep\":\n if isinstance(n_timesteps, list):\n timesteps = len(n_timesteps)\n else:\n timesteps = n_timesteps\n for t in range(timesteps):\n if n_ens_members > 1:\n export_forecast_dataset(precip[:, t, :, :], exporter)\n else:\n export_forecast_dataset(precip[t, :, :], exporter)\n if incremental == \"member\":\n for ens_mem in range(n_ens_members):\n export_forecast_dataset(precip[ens_mem, :, :, :], exporter)\n\n close_forecast_files(exporter)\n\n # assert if netcdf file was saved and file size is not zero\n assert os.path.exists(file_path) and os.path.getsize(file_path) > 0\n\n # Test that the file can be read by the nowcast_importer\n output_file_path = os.path.join(outpath, f\"{outfnprefix}.nc\")\n\n precip_new, _ = import_netcdf_pysteps(output_file_path)\n\n assert_array_almost_equal(precip.squeeze(), precip_new.data)\n assert precip_new.dtype == \"single\"\n\n precip_new, _ = import_netcdf_pysteps(output_file_path, dtype=\"double\")\n assert_array_almost_equal(precip.squeeze(), precip_new.data)\n assert precip_new.dtype == \"double\"\n\n precip_new, _ = import_netcdf_pysteps(output_file_path, fillna=-1000)\n new_invalid_mask = precip_new == -1000\n assert (new_invalid_mask == invalid_mask).all()\n\n\n@pytest.mark.parametrize(\n [\"proj4str\", \"expected_value\"],\n [\n (\n \"+proj=lcc +lat_1=49.83333333333334 +lat_2=51.16666666666666 +lat_0=50.797815 +lon_0=4.359215833333333 +x_0=649328 +y_0=665262 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs \",\n (\n \"lcc\",\n \"lambert_conformal_conic\",\n {\n \"false_easting\": 649328.0,\n \"false_northing\": 665262.0,\n \"longitude_of_central_meridian\": 4.359215833333333,\n \"latitude_of_projection_origin\": 50.797815,\n \"standard_parallel\": (49.83333333333334, 51.16666666666666),\n \"reference_ellipsoid_name\": \"GRS80\",\n \"towgs84\": \"0,0,0,0,0,0,0\",\n },\n ),\n ),\n (\n \"+proj=aea +lat_0=-37.852 +lon_0=144.752 +lat_1=-18.0 +lat_2=-36.0 +a=6378.137 +b=6356.752 +x_0=0 +y_0=0\",\n (\n \"proj\",\n \"albers_conical_equal_area\",\n {\n \"false_easting\": 0.0,\n \"false_northing\": 0.0,\n \"longitude_of_central_meridian\": 144.752,\n \"latitude_of_projection_origin\": -37.852,\n \"standard_parallel\": (-18.0, -36.0),\n },\n ),\n ),\n (\n \"+proj=stere +lat_0=90 +lon_0=0.0 +lat_ts=60.0 +a=6378.137 +b=6356.752 +x_0=0 +y_0=0\",\n (\n \"polar_stereographic\",\n \"polar_stereographic\",\n {\n \"straight_vertical_longitude_from_pole\": 0.0,\n \"latitude_of_projection_origin\": 90.0,\n \"standard_parallel\": 60.0,\n \"false_easting\": 0.0,\n \"false_northing\": 0.0,\n },\n ),\n ),\n ],\n)\ndef test_convert_proj4_to_grid_mapping(proj4str, expected_value):\n \"\"\"\n test the grid mapping in function _convert_proj4_to_grid_mapping()\n \"\"\"\n output = _convert_proj4_to_grid_mapping(proj4str)\n\n assert output == expected_value\n" }, { "path": "pysteps/tests/test_extrapolation_semilagrangian.py", "content": "# -*- coding: utf-8 -*-\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.extrapolation.semilagrangian import extrapolate\n\n\ndef test_semilagrangian():\n \"\"\"Test semilagrangian extrapolation with number of timesteps.\"\"\"\n # inputs\n precip = np.zeros((8, 8))\n precip[0, 0] = 1\n v = np.ones((8, 8))\n velocity = np.stack([v, v])\n num_timesteps = 1\n # expected\n expected = np.zeros((1, 8, 8))\n expected[:, :, 0] = np.nan\n expected[:, 0, :] = np.nan\n expected[:, 1, 1] = 1\n # result\n result = extrapolate(precip, velocity, num_timesteps)\n assert_array_almost_equal(result, expected)\n\n\ndef test_wrong_input_dimensions():\n p_1d = np.ones(8)\n p_2d = np.ones((8, 8))\n p_3d = np.ones((8, 8, 2))\n v_2d = np.ones((8, 8))\n v_3d = np.stack([v_2d, v_2d])\n\n num_timesteps = 1\n\n invalid_inputs = [\n (p_1d, v_3d),\n (p_2d, v_2d),\n (p_3d, v_2d),\n (p_3d, v_3d),\n ]\n for precip, velocity in invalid_inputs:\n with pytest.raises(ValueError):\n extrapolate(precip, velocity, num_timesteps)\n\n\ndef test_ascending_time_step():\n precip = np.ones((8, 8))\n v = np.ones((8, 8))\n velocity = np.stack([v, v])\n\n not_ascending_timesteps = [1, 2, 3, 5, 4, 6, 7]\n with pytest.raises(ValueError):\n extrapolate(precip, velocity, not_ascending_timesteps)\n\n\ndef test_semilagrangian_timesteps():\n \"\"\"Test semilagrangian extrapolation with list of timesteps.\"\"\"\n # inputs\n precip = np.zeros((8, 8))\n precip[0, 0] = 1\n v = np.ones((8, 8)) * 10\n velocity = np.stack([v, v])\n timesteps = [0.1]\n # expected\n expected = np.zeros((1, 8, 8))\n expected[:, :, 0] = np.nan\n expected[:, 0, :] = np.nan\n expected[:, 1, 1] = 1\n # result\n result = extrapolate(precip, velocity, timesteps)\n assert_array_almost_equal(result, expected)\n" }, { "path": "pysteps/tests/test_feature.py", "content": "import pytest\nimport numpy as np\nfrom pysteps import feature\nfrom pysteps.tests.helpers import get_precipitation_fields\n\narg_names = [\"method\", \"max_num_features\"]\narg_values = [(\"blob\", None), (\"blob\", 5), (\"shitomasi\", None), (\"shitomasi\", 5)]\n\n\n@pytest.mark.parametrize(arg_names, arg_values)\ndef test_feature(method, max_num_features):\n if method == \"blob\":\n pytest.importorskip(\"skimage\")\n if method == \"shitomasi\":\n pytest.importorskip(\"cv2\")\n\n input_field = get_precipitation_fields(\n num_prev_files=0,\n num_next_files=0,\n return_raw=True,\n metadata=False,\n upscale=None,\n source=\"mch\",\n )\n\n detector = feature.get_method(method)\n\n kwargs = {\"max_num_features\": max_num_features}\n output = detector(input_field.squeeze(), **kwargs)\n\n assert isinstance(output, np.ndarray)\n assert output.ndim == 2\n assert output.shape[0] > 0\n if max_num_features is not None:\n assert output.shape[0] <= max_num_features\n assert output.shape[1] == 2\n" }, { "path": "pysteps/tests/test_feature_tstorm.py", "content": "import numpy as np\nimport pytest\n\nfrom pysteps.feature.tstorm import detection\nfrom pysteps.utils import to_reflectivity\nfrom pysteps.tests.helpers import get_precipitation_fields\n\ntry:\n from pandas import DataFrame\nexcept ModuleNotFoundError:\n pass\n\narg_names = (\n \"source\",\n \"output_feat\",\n \"dry_input\",\n \"max_num_features\",\n \"output_split_merge\",\n)\n\narg_values = [\n (\"mch\", False, False, None, False),\n (\"mch\", False, False, 5, False),\n (\"mch\", True, False, None, False),\n (\"mch\", True, False, 5, False),\n (\"mch\", False, True, None, False),\n (\"mch\", False, True, 5, False),\n (\"mch\", False, False, None, True),\n]\n\n\n@pytest.mark.parametrize(arg_names, arg_values)\ndef test_feature_tstorm_detection(\n source, output_feat, dry_input, max_num_features, output_split_merge\n):\n pytest.importorskip(\"skimage\")\n pytest.importorskip(\"pandas\")\n\n if not dry_input:\n input, metadata = get_precipitation_fields(0, 0, True, True, None, source)\n input = input.squeeze()\n input, __ = to_reflectivity(input, metadata)\n else:\n input = np.zeros((50, 50))\n\n time = \"000\"\n output = detection(\n input,\n time=time,\n output_feat=output_feat,\n max_num_features=max_num_features,\n output_splits_merges=output_split_merge,\n )\n\n if output_feat:\n assert isinstance(output, np.ndarray)\n assert output.ndim == 2\n assert output.shape[1] == 2\n if max_num_features is not None:\n assert output.shape[0] <= max_num_features\n elif output_split_merge:\n assert isinstance(output, tuple)\n assert len(output) == 2\n assert isinstance(output[0], DataFrame)\n assert isinstance(output[1], np.ndarray)\n if max_num_features is not None:\n assert output[0].shape[0] <= max_num_features\n assert output[0].shape[1] == 15\n assert list(output[0].columns) == [\n \"ID\",\n \"time\",\n \"x\",\n \"y\",\n \"cen_x\",\n \"cen_y\",\n \"max_ref\",\n \"cont\",\n \"area\",\n \"splitted\",\n \"split_IDs\",\n \"merged\",\n \"merged_IDs\",\n \"results_from_split\",\n \"will_merge\",\n ]\n assert (output[0].time == time).all()\n assert output[1].ndim == 2\n assert output[1].shape == input.shape\n if not dry_input:\n assert output[0].shape[0] > 0\n assert sorted(list(output[0].ID)) == sorted(list(np.unique(output[1]))[1:])\n else:\n assert output[0].shape[0] == 0\n assert output[1].sum() == 0\n else:\n assert isinstance(output, tuple)\n assert len(output) == 2\n assert isinstance(output[0], DataFrame)\n assert isinstance(output[1], np.ndarray)\n if max_num_features is not None:\n assert output[0].shape[0] <= max_num_features\n assert output[0].shape[1] == 9\n assert list(output[0].columns) == [\n \"ID\",\n \"time\",\n \"x\",\n \"y\",\n \"cen_x\",\n \"cen_y\",\n \"max_ref\",\n \"cont\",\n \"area\",\n ]\n assert (output[0].time == time).all()\n assert output[1].ndim == 2\n assert output[1].shape == input.shape\n if not dry_input:\n assert output[0].shape[0] > 0\n assert sorted(list(output[0].ID)) == sorted(list(np.unique(output[1]))[1:])\n else:\n assert output[0].shape[0] == 0\n assert output[1].sum() == 0\n" }, { "path": "pysteps/tests/test_importer_decorator.py", "content": "# -*- coding: utf-8 -*-\nfrom functools import partial\n\nimport numpy as np\nimport pytest\n\nfrom pysteps.tests.helpers import get_precipitation_fields\n\ndefault_dtypes = dict(\n fmi=\"double\",\n knmi=\"double\",\n mch=\"double\",\n opera=\"double\",\n saf=\"double\",\n mrms=\"single\",\n)\n\n\n@pytest.mark.parametrize(\"source, default_dtype\", default_dtypes.items())\ndef test_postprocess_import_decorator(source, default_dtype):\n \"\"\"Test the postprocessing decorator for the importers.\"\"\"\n import_data = partial(get_precipitation_fields, return_raw=True, source=source)\n\n precip = import_data()\n invalid_mask = ~np.isfinite(precip)\n\n assert precip.dtype == default_dtype\n\n if default_dtype == \"single\":\n dtype = \"double\"\n else:\n dtype = \"single\"\n\n precip = import_data(dtype=dtype)\n\n assert precip.dtype == dtype\n\n # Test that invalid types are handled correctly\n for dtype in [\"int\", \"int64\"]:\n with pytest.raises(ValueError):\n _ = import_data(dtype=dtype)\n\n precip = import_data(fillna=-1000)\n new_invalid_mask = precip == -1000\n assert (new_invalid_mask == invalid_mask).all()\n" }, { "path": "pysteps/tests/test_interfaces.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy\nimport pytest\n\nimport pysteps\n\n\ndef _generic_interface_test(method_getter, valid_names_func_pair, invalid_names):\n for name, expected_function in valid_names_func_pair:\n error_message = \"Error getting '{}' function.\".format(name)\n assert method_getter(name) == expected_function, error_message\n if isinstance(name, str):\n assert method_getter(name.upper()) == expected_function, error_message\n\n # test invalid names\n for invalid_name in invalid_names:\n with pytest.raises(ValueError):\n method_getter(invalid_name)\n\n\ndef test_nowcasts_interface():\n \"\"\"Test the nowcasts module interface.\"\"\"\n\n from pysteps.blending import (\n linear_blending,\n steps,\n )\n\n method_getter = pysteps.nowcasts.interface.get_method\n\n valid_names_func_pair = [\n (\"linear_blending\", linear_blending.forecast),\n (\"steps\", steps.forecast),\n ]\n\n invalid_names = [\"linear\", \"step\", \"blending\", \"pysteps\"]\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n\ndef test_cascade_interface():\n \"\"\"Test the cascade module interface.\"\"\"\n\n from pysteps.cascade import decomposition, bandpass_filters\n\n method_getter = pysteps.cascade.interface.get_method\n\n valid_names_func_pair = [\n (\"fft\", (decomposition.decomposition_fft, decomposition.recompose_fft)),\n (\"gaussian\", bandpass_filters.filter_gaussian),\n (\"uniform\", bandpass_filters.filter_uniform),\n ]\n\n invalid_names = [\"gauss\", \"fourier\"]\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n\ndef test_extrapolation_interface():\n \"\"\"Test the extrapolation module interface.\"\"\"\n\n from pysteps import extrapolation\n from pysteps.extrapolation import semilagrangian\n\n from pysteps.extrapolation.interface import eulerian_persistence as eulerian\n from pysteps.extrapolation.interface import _do_nothing as do_nothing\n\n method_getter = extrapolation.interface.get_method\n\n valid_returned_objs = dict()\n valid_returned_objs[\"semilagrangian\"] = semilagrangian.extrapolate\n valid_returned_objs[\"eulerian\"] = eulerian\n valid_returned_objs[None] = do_nothing\n valid_returned_objs[\"None\"] = do_nothing\n\n valid_names_func_pair = list(valid_returned_objs.items())\n\n invalid_names = [\"euler\", \"LAGRANGIAN\"]\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n # Test eulerian persistence method\n precip = numpy.random.rand(100, 100)\n velocity = numpy.random.rand(100, 100)\n num_timesteps = 10\n for name in [\"eulerian\", \"EULERIAN\"]:\n forecaster = method_getter(name)\n forecast = forecaster(precip, velocity, num_timesteps)\n for i in range(num_timesteps):\n assert numpy.all(forecast[i] == precip)\n\n forecaster = method_getter(None)\n assert forecaster(precip, velocity, num_timesteps) is None\n\n\ndef test_io_interface():\n \"\"\"Test the io module interface.\"\"\"\n\n from pysteps.io import import_bom_rf3\n from pysteps.io import import_fmi_geotiff\n from pysteps.io import import_fmi_pgm\n from pysteps.io import import_knmi_hdf5\n from pysteps.io import import_mch_gif\n from pysteps.io import import_mch_hdf5\n from pysteps.io import import_mch_metranet\n from pysteps.io import import_mrms_grib\n from pysteps.io import import_opera_hdf5\n from pysteps.io import import_saf_crri\n\n from pysteps.io import initialize_forecast_exporter_geotiff\n from pysteps.io import initialize_forecast_exporter_kineros\n from pysteps.io import initialize_forecast_exporter_netcdf\n\n # Test importers\n valid_names_func_pair = [\n (\"bom_rf3\", import_bom_rf3),\n (\"fmi_geotiff\", import_fmi_geotiff),\n (\"fmi_pgm\", import_fmi_pgm),\n (\"knmi_hdf5\", import_knmi_hdf5),\n (\"mch_gif\", import_mch_gif),\n (\"mch_hdf5\", import_mch_hdf5),\n (\"mch_metranet\", import_mch_metranet),\n (\"mrms_grib\", import_mrms_grib),\n (\"opera_hdf5\", import_opera_hdf5),\n (\"saf_crri\", import_saf_crri),\n ]\n\n def method_getter(name):\n return pysteps.io.interface.get_method(name, \"importer\")\n\n invalid_names = [\"bom\", \"fmi\", \"knmi\", \"mch\", \"mrms\", \"opera\", \"saf\"]\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n # Test exporters\n def method_getter(name):\n return pysteps.io.interface.get_method(name, \"exporter\")\n\n valid_names_func_pair = [\n (\"geotiff\", initialize_forecast_exporter_geotiff),\n (\"kineros\", initialize_forecast_exporter_kineros),\n (\"netcdf\", initialize_forecast_exporter_netcdf),\n ]\n invalid_names = [\"hdf\"]\n\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n # Test for invalid argument type\n with pytest.raises(TypeError):\n pysteps.io.interface.get_method(\"mch_gif\", None)\n pysteps.io.interface.get_method(None, \"importer\")\n\n # Test for invalid method types\n with pytest.raises(ValueError):\n pysteps.io.interface.get_method(\"mch_gif\", \"io\")\n\n\ndef test_postprocessing_interface():\n \"\"\"Test the postprocessing module interface.\"\"\"\n\n # ensemblestats pre-installed methods\n from pysteps.postprocessing import mean, excprob, banddepth\n\n # Test ensemblestats\n valid_names_func_pair = [\n (\"mean\", mean),\n (\"excprob\", excprob),\n (\"banddepth\", banddepth),\n ]\n\n # Test for exisiting functions\n with pytest.warns(RuntimeWarning):\n pysteps.postprocessing.interface.add_postprocessor(\n \"excprob\",\n \"ensemblestat_excprob\",\n \"ensemblestats\",\n [tup[0] for tup in valid_names_func_pair],\n )\n\n # Test get method for valid and invalid names\n def method_getter(name):\n return pysteps.postprocessing.interface.get_method(name, \"ensemblestats\")\n\n invalid_names = [\n \"ensemblestat_mean\",\n \"ensemblestat_excprob\",\n \"ensemblestat_banddepth\",\n ]\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n # Test diagnostics\n def method_getter(name):\n return pysteps.postprocessing.interface.get_method(name, \"diagnostics\")\n\n valid_names_func_pair = []\n invalid_names = [\"unknown\"]\n\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n # Test for invalid argument type\n with pytest.raises(TypeError):\n pysteps.postprocessing.interface.get_method(\"mean\", None)\n with pytest.raises(TypeError):\n pysteps.postprocessing.interface.get_method(None, \"ensemblestats\")\n\n # Test for invalid method types\n with pytest.raises(ValueError):\n pysteps.postprocessing.interface.get_method(\"mean\", \"forecast\")\n\n # Test print\n pysteps.postprocessing.postprocessors_info()\n\n\ndef test_motion_interface():\n \"\"\"Test the motion module interface.\"\"\"\n\n from pysteps.motion.constant import constant\n from pysteps.motion.darts import DARTS\n from pysteps.motion.lucaskanade import dense_lucaskanade\n from pysteps.motion.proesmans import proesmans\n from pysteps.motion.vet import vet\n\n method_getter = pysteps.motion.interface.get_method\n\n valid_names_func_pair = [\n (\"constant\", constant),\n (\"darts\", DARTS),\n (\"lk\", dense_lucaskanade),\n (\"lucaskanade\", dense_lucaskanade),\n (\"proesmans\", proesmans),\n (\"vet\", vet),\n ]\n\n invalid_names = [\"dart\", \"pyvet\", \"lukascanade\", \"lucas-kanade\", \"no_method\"]\n\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n # Test default dummy method\n precip_field = method_getter(None)(numpy.random.random([2, 300, 500]))\n assert numpy.max(numpy.abs(precip_field)) == pytest.approx(0)\n\n # test not implemented names\n for name in [\"brox\", \"clg\"]:\n with pytest.raises(NotImplementedError):\n method_getter(name) # last s missing\n\n\ndef test_noise_interface():\n \"\"\"Test the noise module interface.\"\"\"\n\n from pysteps.noise.fftgenerators import (\n initialize_param_2d_fft_filter,\n generate_noise_2d_fft_filter,\n initialize_nonparam_2d_fft_filter,\n initialize_nonparam_2d_ssft_filter,\n generate_noise_2d_ssft_filter,\n initialize_nonparam_2d_nested_filter,\n )\n\n from pysteps.noise.motion import initialize_bps, generate_bps\n\n method_getter = pysteps.noise.interface.get_method\n\n valid_names_func_pair = [\n (\"parametric\", (initialize_param_2d_fft_filter, generate_noise_2d_fft_filter)),\n (\n \"nonparametric\",\n (initialize_nonparam_2d_fft_filter, generate_noise_2d_fft_filter),\n ),\n (\"ssft\", (initialize_nonparam_2d_ssft_filter, generate_noise_2d_ssft_filter)),\n (\n \"nested\",\n (initialize_nonparam_2d_nested_filter, generate_noise_2d_ssft_filter),\n ),\n (\"bps\", (initialize_bps, generate_bps)),\n ]\n\n invalid_names = [\"nest\", \"sft\", \"ssfft\"]\n\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n\ndef test_nowcasts_interface():\n \"\"\"Test the nowcasts module interface.\"\"\"\n\n from pysteps.nowcasts import (\n anvil,\n extrapolation,\n lagrangian_probability,\n linda,\n sprog,\n steps,\n sseps,\n )\n\n method_getter = pysteps.nowcasts.interface.get_method\n\n valid_names_func_pair = [\n (\"anvil\", anvil.forecast),\n (\"extrapolation\", extrapolation.forecast),\n (\"lagrangian\", extrapolation.forecast),\n (\"linda\", linda.forecast),\n (\"probability\", lagrangian_probability.forecast),\n (\"lagrangian_probability\", lagrangian_probability.forecast),\n (\"sprog\", sprog.forecast),\n (\"sseps\", sseps.forecast),\n (\"steps\", steps.forecast),\n ]\n\n invalid_names = [\"extrap\", \"step\", \"s-prog\", \"pysteps\"]\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n # Test eulerian persistence method\n precip = numpy.random.rand(100, 100)\n velocity = numpy.random.rand(100, 100)\n num_timesteps = 10\n for name in [\"eulerian\", \"EULERIAN\"]:\n forecast = method_getter(name)(precip, velocity, num_timesteps)\n for i in range(num_timesteps):\n assert numpy.all(forecast[i] == precip)\n\n\ndef test_utils_interface():\n \"\"\"Test utils module interface.\"\"\"\n\n from pysteps.utils import arrays\n from pysteps.utils import cleansing\n from pysteps.utils import conversion\n from pysteps.utils import dimension\n from pysteps.utils import images\n from pysteps.utils import interpolate\n from pysteps.utils import reprojection\n from pysteps.utils import spectral\n from pysteps.utils import tapering\n from pysteps.utils import transformation\n\n method_getter = pysteps.utils.interface.get_method\n\n valid_names_func_pair = [\n (\"centred_coord\", arrays.compute_centred_coord_array),\n (\"decluster\", cleansing.decluster),\n (\"detect_outliers\", cleansing.detect_outliers),\n (\"mm/h\", conversion.to_rainrate),\n (\"rainrate\", conversion.to_rainrate),\n (\"mm\", conversion.to_raindepth),\n (\"raindepth\", conversion.to_raindepth),\n (\"dbz\", conversion.to_reflectivity),\n (\"reflectivity\", conversion.to_reflectivity),\n (\"accumulate\", dimension.aggregate_fields_time),\n (\"clip\", dimension.clip_domain),\n (\"square\", dimension.square_domain),\n (\"upscale\", dimension.aggregate_fields_space),\n (\"morph_opening\", images.morph_opening),\n (\"rbfinterp2d\", interpolate.rbfinterp2d),\n (\"reproject_grids\", reprojection.reproject_grids),\n (\"rapsd\", spectral.rapsd),\n (\"rm_rdisc\", spectral.remove_rain_norain_discontinuity),\n (\"compute_mask_window_function\", tapering.compute_mask_window_function),\n (\"compute_window_function\", tapering.compute_window_function),\n (\"boxcox\", transformation.boxcox_transform),\n (\"box-cox\", transformation.boxcox_transform),\n (\"db\", transformation.dB_transform),\n (\"decibel\", transformation.dB_transform),\n (\"log\", transformation.boxcox_transform),\n (\"nqt\", transformation.NQ_transform),\n (\"sqrt\", transformation.sqrt_transform),\n ]\n\n invalid_names = [\"random\", \"invalid\"]\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n\ndef test_downscaling_interface():\n \"\"\"Test the downscaling module interface.\"\"\"\n\n from pysteps.downscaling import rainfarm\n\n method_getter = pysteps.downscaling.interface.get_method\n\n valid_names_func_pair = [\n (\"rainfarm\", rainfarm.downscale),\n ]\n\n invalid_names = [\"rain-farm\", \"rainfarms\"]\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n\ndef test_feature_interface():\n \"\"\"Test the feature detection module interface.\"\"\"\n\n from pysteps.feature import blob\n from pysteps.feature import tstorm\n from pysteps.feature import shitomasi\n\n method_getter = pysteps.feature.interface.get_method\n\n valid_names_func_pair = [\n (\"blob\", blob.detection),\n (\"tstorm\", tstorm.detection),\n (\"shitomasi\", shitomasi.detection),\n ]\n\n invalid_names = [\"blobs\", \"storm\", \"shi-tomasi\"]\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n\n\ndef test_tracking_interface():\n \"\"\"Test the feature tracking module interface.\"\"\"\n\n from pysteps.tracking import lucaskanade\n from pysteps.tracking import tdating\n\n method_getter = pysteps.tracking.interface.get_method\n\n valid_names_func_pair = [\n (\"lucaskanade\", lucaskanade.track_features),\n (\"tdating\", tdating.dating),\n ]\n\n invalid_names = [\"lucas-kanade\", \"dating\"]\n _generic_interface_test(method_getter, valid_names_func_pair, invalid_names)\n" }, { "path": "pysteps/tests/test_io_archive.py", "content": "from datetime import datetime\n\nimport pytest\n\nimport pysteps\n\n\ndef test_find_by_date_mch():\n pytest.importorskip(\"PIL\")\n\n date = datetime.strptime(\"201505151630\", \"%Y%m%d%H%M\")\n data_source = pysteps.rcparams.data_sources[\"mch\"]\n root_path = data_source[\"root_path\"]\n path_fmt = data_source[\"path_fmt\"]\n fn_pattern = data_source[\"fn_pattern\"]\n fn_ext = data_source[\"fn_ext\"]\n timestep = data_source[\"timestep\"]\n\n fns = pysteps.io.archive.find_by_date(\n date,\n root_path,\n path_fmt,\n fn_pattern,\n fn_ext,\n timestep=timestep,\n num_prev_files=1,\n num_next_files=1,\n )\n\n assert len(fns) == 2\n assert len(fns[0]) == 3\n assert len(fns[1]) == 3\n assert isinstance(fns[0][0], str)\n assert isinstance(fns[1][0], datetime)\n" }, { "path": "pysteps/tests/test_io_bom_rf3.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\n\nimport pytest\n\nimport pysteps\nfrom pysteps.tests.helpers import smart_assert\n\nnetCDF4 = pytest.importorskip(\"netCDF4\")\n\n# Test import_bom_rf3 function\nexpected_proj1 = (\n \"+proj=aea +lon_0=144.752 +lat_0=-37.852 \" \"+lat_1=-18.000 +lat_2=-36.000\"\n)\n\ntest_metadata_bom = [\n (\"transform\", None, None),\n (\"zerovalue\", 0.0, 0.1),\n (\"projection\", expected_proj1, None),\n (\"unit\", \"mm\", None),\n (\"accutime\", 6, 0.1),\n (\"x1\", -128000.0, 0.1),\n (\"x2\", 127500.0, 0.1),\n (\"y1\", -127500.0, 0.1),\n (\"y2\", 128000.0, 0.1),\n (\"xpixelsize\", 500.0, 0.1),\n (\"ypixelsize\", 500.0, 0.1),\n (\"cartesian_unit\", \"m\", None),\n (\"yorigin\", \"upper\", None),\n (\"institution\", \"Commonwealth of Australia, Bureau of Meteorology\", None),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_metadata_bom)\ndef test_io_import_bom_rf3_metadata(variable, expected, tolerance):\n \"\"\"Test the importer Bom RF3.\"\"\"\n root_path = pysteps.rcparams.data_sources[\"bom\"][\"root_path\"]\n rel_path = os.path.join(\"prcp-cscn\", \"2\", \"2018\", \"06\", \"16\")\n filename = os.path.join(root_path, rel_path, \"2_20180616_100000.prcp-cscn.nc\")\n precip, _, metadata = pysteps.io.import_bom_rf3(filename)\n smart_assert(metadata[variable], expected, tolerance)\n assert precip.shape == (512, 512)\n\n\n# Test _import_bom_rf3_data function\ndef test_io_import_bom_rf3_shape():\n \"\"\"Test the importer Bom RF3.\"\"\"\n root_path = pysteps.rcparams.data_sources[\"bom\"][\"root_path\"]\n rel_path = os.path.join(\"prcp-cscn\", \"2\", \"2018\", \"06\", \"16\")\n filename = os.path.join(root_path, rel_path, \"2_20180616_100000.prcp-cscn.nc\")\n precip, _ = pysteps.io.importers._import_bom_rf3_data(filename)\n assert precip.shape == (512, 512)\n\n\n# Test _import_bom_rf3_geodata function\nexpected_proj2 = (\n \"+proj=aea +lon_0=144.752 +lat_0=-37.852 \" \"+lat_1=-18.000 +lat_2=-36.000\"\n)\n# test_geodata: list of (variable,expected,tolerance) tuples\ntest_geodata_bom = [\n (\"projection\", expected_proj2, None),\n (\"unit\", \"mm\", None),\n (\"accutime\", 6, 0.1),\n (\"x1\", -128000.0, 0.1),\n (\"x2\", 127500.0, 0.1),\n (\"y1\", -127500.0, 0.1),\n (\"y2\", 128000.0, 0.1),\n (\"xpixelsize\", 500.0, 0.1),\n (\"ypixelsize\", 500.0, 0.1),\n (\"cartesian_unit\", \"m\", None),\n (\"yorigin\", \"upper\", None),\n (\"institution\", \"Commonwealth of Australia, Bureau of Meteorology\", None),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_geodata_bom)\ndef test_io_import_bom_rf3_geodata(variable, expected, tolerance):\n \"\"\"Test the importer Bom RF3.\"\"\"\n root_path = pysteps.rcparams.data_sources[\"bom\"][\"root_path\"]\n rel_path = os.path.join(\"prcp-cscn\", \"2\", \"2018\", \"06\", \"16\")\n filename = os.path.join(root_path, rel_path, \"2_20180616_100000.prcp-cscn.nc\")\n ds_rainfall = netCDF4.Dataset(filename)\n geodata = pysteps.io.importers._import_bom_rf3_geodata(ds_rainfall)\n smart_assert(geodata[variable], expected, tolerance)\n\n ds_rainfall.close()\n" }, { "path": "pysteps/tests/test_io_dwd_hdf5.py", "content": "# -*- coding: utf-8 -*-\n\nimport pytest\n\nimport pysteps\nfrom pysteps.tests.helpers import smart_assert, get_precipitation_fields\n\npytest.importorskip(\"h5py\")\n\n# Test for RADOLAN RY product\n\nprecip_ry, metadata_ry = get_precipitation_fields(\n num_prev_files=0,\n num_next_files=0,\n return_raw=False,\n metadata=True,\n source=\"dwd\",\n log_transform=False,\n importer_kwargs=dict(qty=\"RATE\"),\n)\n\n\ndef test_io_import_dwd_hdf5_ry_shape():\n \"\"\"Test the importer DWD HDF5.\"\"\"\n assert precip_ry.shape == (1200, 1100)\n\n\n# Test_metadata\n# Expected projection definition\nexpected_proj = (\n \"+proj=stere +lat_0=90 +lat_ts=60 \"\n \"+lon_0=10 +a=6378137 +b=6356752.3142451802 \"\n \"+no_defs +x_0=543196.83521776402 \"\n \"+y_0=3622588.8619310018 +units=m\"\n)\n\n# List of (variable,expected,tolerance) tuples\ntest_ry_attrs = [\n (\"projection\", expected_proj, None),\n (\"ll_lon\", 3.566994635, 1e-10),\n (\"ll_lat\", 45.69642538, 1e-10),\n (\"ur_lon\", 18.73161645, 1e-10),\n (\"ur_lat\", 55.84543856, 1e-10),\n (\"x1\", -500.0, 1e-6),\n (\"y1\", -1199500.0, 1e-6),\n (\"x2\", 1099500.0, 1e-6),\n (\"y2\", 500.0, 1e-6),\n (\"xpixelsize\", 1000.0, 1e-10),\n (\"xpixelsize\", 1000.0, 1e-10),\n (\"cartesian_unit\", \"m\", None),\n (\"yorigin\", \"upper\", None),\n (\"institution\", \"ORG:78,CTY:616,CMT:Deutscher Wetterdienst radolan@dwd.de\", None),\n (\"accutime\", 5.0, 1e-10),\n (\"unit\", \"mm/h\", None),\n (\"transform\", None, None),\n (\"zerovalue\", 0.0, 1e-6),\n (\"threshold\", 0.12, 1e-6),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_ry_attrs)\ndef test_io_import_dwd_hdf5_ry_metadata(variable, expected, tolerance):\n \"\"\"Test the importer OPERA HDF5.\"\"\"\n smart_assert(metadata_ry[variable], expected, tolerance)\n" }, { "path": "pysteps/tests/test_io_fmi_geotiff.py", "content": "import os\n\nimport pytest\n\nimport pysteps\nfrom pysteps.tests.helpers import smart_assert\n\npytest.importorskip(\"pyproj\")\npytest.importorskip(\"osgeo\")\n\nroot_path = pysteps.rcparams.data_sources[\"fmi_geotiff\"][\"root_path\"]\nfilename = os.path.join(\n root_path,\n \"20160928\",\n \"201609281600_FINUTM.tif\",\n)\nprecip, _, metadata = pysteps.io.import_fmi_geotiff(filename)\n\n\ndef test_io_import_fmi_geotiff_shape():\n \"\"\"Test the shape of the read file.\"\"\"\n assert precip.shape == (7316, 4963)\n\n\nexpected_proj = (\n \"+proj=utm +zone=35 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs\"\n)\n\n# test_geodata: list of (variable,expected,tolerance) tuples\ntest_geodata = [\n (\"projection\", expected_proj, None),\n (\"x1\", -196593.0043142295908183, 1e-10),\n (\"x2\", 1044176.9413554778, 1e-10),\n (\"y1\", 6255329.6988206729292870, 1e-10),\n (\"y2\", 8084432.005259146, 1e-10),\n (\"xpixelsize\", 250.0040188736061566, 1e-6),\n (\"ypixelsize\", 250.0139839309011904, 1e-6),\n (\"cartesian_unit\", \"m\", None),\n (\"yorigin\", \"upper\", None),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_geodata)\ndef test_io_import_fmi_pgm_geodata(variable, expected, tolerance):\n \"\"\"Test the GeoTIFF and metadata reading.\"\"\"\n smart_assert(metadata[variable], expected, tolerance)\n" }, { "path": "pysteps/tests/test_io_fmi_pgm.py", "content": "import os\n\nimport pytest\n\nimport pysteps\nfrom pysteps.tests.helpers import smart_assert\n\npytest.importorskip(\"pyproj\")\n\n\nroot_path = pysteps.rcparams.data_sources[\"fmi\"][\"root_path\"]\nfilename = os.path.join(\n root_path,\n \"20160928\",\n \"201609281600_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n)\nprecip, _, metadata = pysteps.io.import_fmi_pgm(filename, gzipped=True)\n\n\ndef test_io_import_fmi_pgm_shape():\n \"\"\"Test the importer FMI PGM.\"\"\"\n assert precip.shape == (1226, 760)\n\n\nexpected_proj = (\n \"+proj=stere +lon_0=25E +lat_0=90N \"\n \"+lat_ts=60 +a=6371288 +x_0=380886.310 \"\n \"+y_0=3395677.920 +no_defs\"\n)\n\ntest_attrs = [\n (\"projection\", expected_proj, None),\n (\"institution\", \"Finnish Meteorological Institute\", None),\n # (\"composite_area\", [\"FIN\"]),\n # (\"projection_name\", [\"SUOMI1\"]),\n # (\"radar\", [\"LUO\", \"1\", \"26.9008\", \"67.1386\"]),\n # (\"obstime\", [\"201609281600\"]),\n # (\"producttype\", [\"CAPPI\"]),\n # (\"productname\", [\"LOWEST\"]),\n # (\"param\", [\"CorrectedReflectivity\"]),\n # (\"metersperpixel_x\", [\"999.674053\"]),\n # (\"metersperpixel_y\", [\"999.62859\"]),\n # (\"projection\", [\"radar\", \"{\"]),\n # (\"type\", [\"stereographic\"]),\n # (\"centrallongitude\", [\"25\"]),\n # (\"centrallatitude\", [\"90\"]),\n # (\"truelatitude\", [\"60\"]),\n # (\"bottomleft\", [\"18.600000\", \"57.930000\"]),\n # (\"topright\", [\"34.903000\", \"69.005000\"]),\n # (\"missingval\", 255),\n (\"accutime\", 5.0, 0.1),\n (\"unit\", \"dBZ\", None),\n (\"transform\", \"dB\", None),\n (\"zerovalue\", -32.0, 0.1),\n (\"threshold\", -31.5, 0.1),\n (\"zr_a\", 223.0, 0.1),\n (\"zr_b\", 1.53, 0.1),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_attrs)\ndef test_io_import_mch_gif_dataset_attrs(variable, expected, tolerance):\n \"\"\"Test the importer FMI PMG.\"\"\"\n smart_assert(metadata[variable], expected, tolerance)\n\n\n# test_geodata: list of (variable,expected,tolerance) tuples\ntest_geodata = [\n (\"projection\", expected_proj, None),\n (\"x1\", 0.0049823258887045085, 1e-20),\n (\"x2\", 759752.2852757066, 1e-10),\n (\"y1\", 0.009731985162943602, 1e-18),\n (\"y2\", 1225544.6588913496, 1e-10),\n (\"xpixelsize\", 999.674053, 1e-6),\n (\"ypixelsize\", 999.62859, 1e-5),\n (\"cartesian_unit\", \"m\", None),\n (\"yorigin\", \"upper\", None),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_geodata)\ndef test_io_import_fmi_pgm_geodata(variable, expected, tolerance):\n \"\"\"Test the importer FMI pgm.\"\"\"\n root_path = pysteps.rcparams.data_sources[\"fmi\"][\"root_path\"]\n filename = os.path.join(\n root_path,\n \"20160928\",\n \"201609281600_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n )\n metadata = pysteps.io.importers._import_fmi_pgm_metadata(filename, gzipped=True)\n geodata = pysteps.io.importers._import_fmi_pgm_geodata(metadata)\n\n smart_assert(geodata[variable], expected, tolerance)\n" }, { "path": "pysteps/tests/test_io_knmi_hdf5.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\n\nimport pytest\n\nimport pysteps\nfrom pysteps.tests.helpers import smart_assert\n\npytest.importorskip(\"h5py\")\n\n\nroot_path = pysteps.rcparams.data_sources[\"knmi\"][\"root_path\"]\nfilename = os.path.join(root_path, \"2010/08\", \"RAD_NL25_RAP_5min_201008260000.h5\")\nprecip, _, metadata = pysteps.io.import_knmi_hdf5(filename)\n\n\ndef test_io_import_knmi_hdf5_shape():\n \"\"\"Test the importer KNMI HDF5.\"\"\"\n assert precip.shape == (765, 700)\n\n\n# test_metadata: list of (variable,expected, tolerance) tuples\n\nexpected_proj = (\n \"+proj=stere +lat_0=90 +lon_0=0.0 +lat_ts=60.0 +a=6378137 +b=6356752 +x_0=0 +y_0=0\"\n)\n\n# list of (variable,expected,tolerance) tuples\ntest_attrs = [\n (\"projection\", expected_proj, None),\n (\"x1\", 0.0, 1e-10),\n (\"y1\", -4415038.179210632, 1e-10),\n (\"x2\", 699984.2646331593, 1e-10),\n (\"y2\", -3649950.360247753, 1e-10),\n (\"xpixelsize\", 1000.0, 1e-10),\n (\"xpixelsize\", 1000.0, 1e-10),\n (\"cartesian_unit\", \"m\", None),\n (\"accutime\", 5.0, 1e-10),\n (\"yorigin\", \"upper\", None),\n (\"unit\", \"mm\", None),\n (\"institution\", \"KNMI - Royal Netherlands Meteorological Institute\", None),\n (\"transform\", None, None),\n (\"zerovalue\", 0.0, 1e-10),\n (\"threshold\", 0.01, 1e-10),\n (\"zr_a\", 200.0, None),\n (\"zr_b\", 1.6, None),\n]\n\n\n@pytest.mark.parametrize(\"variable,expected,tolerance\", test_attrs)\ndef test_io_import_knmi_hdf5_metadata(variable, expected, tolerance):\n \"\"\"Test the importer KNMI HDF5.\"\"\"\n smart_assert(metadata[variable], expected, tolerance)\n" }, { "path": "pysteps/tests/test_io_mch_gif.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\n\nimport pytest\n\nimport pysteps\nfrom pysteps.tests.helpers import smart_assert\n\npytest.importorskip(\"PIL\")\n\nroot_path = pysteps.rcparams.data_sources[\"mch\"][\"root_path\"]\nfilename = os.path.join(root_path, \"20170131\", \"AQC170310945F_00005.801.gif\")\nprecip, _, metadata = pysteps.io.import_mch_gif(filename, \"AQC\", \"mm\", 5.0)\n\n\ndef test_io_import_mch_gif_shape():\n \"\"\"Test the importer MCH GIF.\"\"\"\n assert precip.shape == (640, 710)\n\n\nexpected_proj = (\n \"+proj=somerc +lon_0=7.43958333333333 \"\n \"+lat_0=46.9524055555556 +k_0=1 \"\n \"+x_0=600000 +y_0=200000 +ellps=bessel \"\n \"+towgs84=674.374,15.056,405.346,0,0,0,0 \"\n \"+units=m +no_defs\"\n)\n\n# list of (variable,expected,tolerance) tuples\ntest_attrs = [\n (\"projection\", expected_proj, None),\n (\"institution\", \"MeteoSwiss\", None),\n (\"accutime\", 5.0, 0.1),\n (\"unit\", \"mm\", None),\n (\"transform\", None, None),\n (\"zerovalue\", 0.0, 0.1),\n (\"threshold\", 0.0009628129986471908, 1e-19),\n (\"zr_a\", 316.0, 0.1),\n (\"zr_b\", 1.5, 0.1),\n (\"x1\", 255000.0, 0.1),\n (\"y1\", -160000.0, 0.1),\n (\"x2\", 965000.0, 0.1),\n (\"y2\", 480000.0, 0.1),\n (\"xpixelsize\", 1000.0, 0.1),\n (\"ypixelsize\", 1000.0, 0.1),\n (\"cartesian_unit\", \"m\", None),\n (\"yorigin\", \"upper\", None),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_attrs)\ndef test_io_import_mch_gif_dataset_attrs(variable, expected, tolerance):\n \"\"\"Test the importer MCH GIF.\"\"\"\n smart_assert(metadata[variable], expected, tolerance)\n\n\n# test_geodata: list of (variable,expected,tolerance) tuples\ntest_geodata = [\n (\"projection\", expected_proj, None),\n (\"x1\", 255000.0, 0.1),\n (\"y1\", -160000.0, 0.1),\n (\"x2\", 965000.0, 0.1),\n (\"y2\", 480000.0, 0.1),\n (\"xpixelsize\", 1000.0, 0.1),\n (\"ypixelsize\", 1000.0, 0.1),\n (\"cartesian_unit\", \"m\", None),\n (\"yorigin\", \"upper\", None),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_geodata)\ndef test_io_import_mch_geodata(variable, expected, tolerance):\n \"\"\"Test the importer MCH geodata.\"\"\"\n geodata = pysteps.io.importers._import_mch_geodata()\n smart_assert(geodata[variable], expected, tolerance)\n" }, { "path": "pysteps/tests/test_io_mrms_grib.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nimport pysteps\n\npytest.importorskip(\"pygrib\")\n\n\ndef test_io_import_mrms_grib():\n \"\"\"Test the importer for NSSL data.\"\"\"\n\n root_path = pysteps.rcparams.data_sources[\"mrms\"][\"root_path\"]\n filename = os.path.join(\n root_path, \"2019/06/10/\", \"PrecipRate_00.00_20190610-000000.grib2\"\n )\n precip, _, metadata = pysteps.io.import_mrms_grib(filename, fillna=0, window_size=1)\n\n assert precip.shape == (3500, 7000)\n assert precip.dtype == \"single\"\n\n expected_metadata = {\n \"institution\": \"NOAA National Severe Storms Laboratory\",\n \"xpixelsize\": 0.01,\n \"ypixelsize\": 0.01,\n \"unit\": \"mm/h\",\n \"transform\": None,\n \"zerovalue\": 0,\n \"projection\": \"+proj=longlat +ellps=IAU76\",\n \"yorigin\": \"upper\",\n \"threshold\": 0.1,\n \"x1\": -129.99999999999997,\n \"x2\": -60.00000199999991,\n \"y1\": 20.000001,\n \"y2\": 55.00000000000001,\n \"cartesian_unit\": \"degrees\",\n }\n\n for key, value in expected_metadata.items():\n if isinstance(value, float):\n assert_array_almost_equal(metadata[key], expected_metadata[key])\n else:\n assert metadata[key] == expected_metadata[key]\n\n x = np.arange(metadata[\"x1\"], metadata[\"x2\"], metadata[\"xpixelsize\"])\n y = np.arange(metadata[\"y1\"], metadata[\"y2\"], metadata[\"ypixelsize\"])\n\n assert y.size == precip.shape[0]\n assert x.size == precip.shape[1]\n\n # The full latitude range is (20.005, 54.995)\n # The full longitude range is (230.005, 299.995)\n\n # Test that if the bounding box is larger than the domain, all the points are returned.\n precip2, _, _ = pysteps.io.import_mrms_grib(\n filename, fillna=0, extent=(220, 300, 20, 55), window_size=1\n )\n assert precip2.shape == (3500, 7000)\n\n assert_array_almost_equal(precip, precip2)\n\n del precip2\n\n # Test that a portion of the domain is returned correctly\n precip3, _, _ = pysteps.io.import_mrms_grib(\n filename, fillna=0, extent=(250, 260, 30, 35), window_size=1\n )\n\n assert precip3.shape == (500, 1000)\n assert_array_almost_equal(precip3, precip[2000:2500, 2000:3000])\n del precip3\n\n precip4, _, _ = pysteps.io.import_mrms_grib(filename, dtype=\"double\", fillna=0)\n assert precip4.dtype == \"double\"\n del precip4\n\n precip5, _, _ = pysteps.io.import_mrms_grib(filename, dtype=\"single\", fillna=0)\n assert precip5.dtype == \"single\"\n del precip5\n" }, { "path": "pysteps/tests/test_io_nowcast_importers.py", "content": "import numpy as np\nimport pytest\n\nfrom pysteps import io\nfrom pysteps.tests.helpers import get_precipitation_fields\n\nprecip, metadata = get_precipitation_fields(\n num_prev_files=1,\n num_next_files=0,\n return_raw=False,\n metadata=True,\n upscale=2000,\n)\n\n\n@pytest.mark.parametrize(\n \"precip, metadata\",\n [(precip, metadata), (np.zeros_like(precip), metadata)],\n)\ndef test_import_netcdf(precip, metadata, tmp_path):\n\n pytest.importorskip(\"pyproj\")\n\n field_shape = (precip.shape[1], precip.shape[2])\n startdate = metadata[\"timestamps\"][-1]\n timestep = metadata[\"accutime\"]\n exporter = io.exporters.initialize_forecast_exporter_netcdf(\n outpath=tmp_path.as_posix(),\n outfnprefix=\"test\",\n startdate=startdate,\n timestep=timestep,\n n_timesteps=precip.shape[0],\n shape=field_shape,\n metadata=metadata,\n )\n io.exporters.export_forecast_dataset(precip, exporter)\n io.exporters.close_forecast_files(exporter)\n\n tmp_file = tmp_path / \"test.nc\"\n precip_netcdf, metadata_netcdf = io.import_netcdf_pysteps(tmp_file, dtype=\"float64\")\n\n assert isinstance(precip_netcdf, np.ndarray)\n assert isinstance(metadata_netcdf, dict)\n assert precip_netcdf.ndim == precip.ndim, \"Wrong number of dimensions\"\n assert precip_netcdf.shape[0] == precip.shape[0], \"Wrong number of lead times\"\n assert precip_netcdf.shape[1:] == field_shape, \"Wrong field shape\"\n assert np.allclose(precip_netcdf, precip)\n" }, { "path": "pysteps/tests/test_io_opera_hdf5.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\n\nimport pytest\n\nimport pysteps\nfrom pysteps.tests.helpers import smart_assert\n\npytest.importorskip(\"h5py\")\n\n# tests for three OPERA products:\n# Odyssey rain rate composite (production discontinued on October 30th 2024)\n# CIRRUS max. reflectivity composites\n# NIMBUS rain rate composites\n\nroot_path = pysteps.rcparams.data_sources[\"opera\"][\"root_path\"]\n\nfilename = os.path.join(root_path, \"20180824\", \"T_PAAH21_C_EUOC_20180824180000.hdf\")\nprecip_odyssey, _, metadata_odyssey = pysteps.io.import_opera_hdf5(filename, qty=\"RATE\")\n\nfilename = os.path.join(\n root_path, \"20241126\", \"CIRRUS\", \"T_PABV21_C_EUOC_20241126010000.hdf\"\n)\nprecip_cirrus, _, metadata_cirrus = pysteps.io.import_opera_hdf5(filename, qty=\"DBZH\")\n\nfilename = os.path.join(\n root_path, \"20241126\", \"NIMBUS\", \"T_PAAH22_C_EUOC_20241126010000.hdf\"\n)\nprecip_nimbus_rain_rate, _, metadata_nimbus_rain_rate = pysteps.io.import_opera_hdf5(\n filename, qty=\"RATE\"\n)\n\nfilename = os.path.join(\n root_path, \"20241126\", \"NIMBUS\", \"T_PASH22_C_EUOC_20241126010000.hdf\"\n)\nprecip_nimbus_rain_accum, _, metadata_nimbus_rain_accum = pysteps.io.import_opera_hdf5(\n filename, qty=\"ACRR\"\n)\n\n\ndef test_io_import_opera_hdf5_odyssey_shape():\n \"\"\"Test the importer OPERA HDF5.\"\"\"\n assert precip_odyssey.shape == (2200, 1900)\n\n\ndef test_io_import_opera_hdf5_cirrus_shape():\n \"\"\"Test the importer OPERA HDF5.\"\"\"\n assert precip_cirrus.shape == (4400, 3800)\n\n\ndef test_io_import_opera_hdf5_nimbus_rain_rate_shape():\n \"\"\"Test the importer OPERA HDF5.\"\"\"\n assert precip_nimbus_rain_rate.shape == (2200, 1900)\n\n\ndef test_io_import_opera_hdf5_nimbus_rain_accum_shape():\n \"\"\"Test the importer OPERA HDF5.\"\"\"\n assert precip_nimbus_rain_accum.shape == (2200, 1900)\n\n\n# test_metadata: list of (variable,expected, tolerance) tuples\nexpected_proj = (\n \"+proj=laea +lat_0=55.0 +lon_0=10.0 \"\n \"+x_0=1950000.0 \"\n \"+y_0=-2100000.0 \"\n \"+units=m +ellps=WGS84\"\n)\n\n# list of (variable,expected,tolerance) tuples\ntest_odyssey_attrs = [\n (\"projection\", expected_proj, None),\n (\"ll_lon\", -10.434576838640398, 1e-10),\n (\"ll_lat\", 31.746215319325056, 1e-10),\n (\"ur_lon\", 57.81196475014995, 1e-10),\n (\"ur_lat\", 67.62103710275053, 1e-10),\n (\"x1\", -0.0004161088727414608, 1e-6),\n (\"y1\", -4400000.001057557, 1e-10),\n (\"x2\", 3800000.0004256153, 1e-10),\n (\"y2\", -0.0004262728616595268, 1e-6),\n (\"xpixelsize\", 2000.0, 1e-10),\n (\"xpixelsize\", 2000.0, 1e-10),\n (\"cartesian_unit\", \"m\", None),\n (\"accutime\", 15.0, 1e-10),\n (\"yorigin\", \"upper\", None),\n (\"unit\", \"mm/h\", None),\n (\"institution\", \"Odyssey datacentre\", None),\n (\"transform\", None, None),\n (\"zerovalue\", 0.0, 1e-10),\n (\"threshold\", 0.01, 1e-10),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_odyssey_attrs)\ndef test_io_import_opera_hdf5_odyssey_dataset_attrs(variable, expected, tolerance):\n \"\"\"Test the importer OPERA HDF5.\"\"\"\n smart_assert(metadata_odyssey[variable], expected, tolerance)\n\n\n# list of (variable,expected,tolerance) tuples\ntest_cirrus_attrs = [\n (\"projection\", expected_proj, None),\n (\"ll_lon\", -10.4345768386404, 1e-10),\n (\"ll_lat\", 31.7462153182675, 1e-10),\n (\"ur_lon\", 57.8119647501499, 1e-10),\n (\"ur_lat\", 67.6210371071631, 1e-10),\n (\"x1\", -0.00027143326587975025, 1e-6),\n (\"y1\", -4400000.00116988, 1e-10),\n (\"x2\", 3800000.0000817003, 1e-10),\n (\"y2\", -8.761277422308922e-05, 1e-6),\n (\"xpixelsize\", 1000.0, 1e-10),\n (\"ypixelsize\", 1000.0, 1e-10),\n (\"cartesian_unit\", \"m\", None),\n (\"accutime\", 15.0, 1e-10),\n (\"yorigin\", \"upper\", None),\n (\"unit\", \"dBZ\", None),\n (\"institution\", \"Odyssey datacentre\", None),\n (\"transform\", \"dB\", None),\n (\"zerovalue\", -32.0, 1e-10),\n (\"threshold\", -31.5, 1e-10),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_cirrus_attrs)\ndef test_io_import_opera_hdf5_cirrus_dataset_attrs(variable, expected, tolerance):\n \"\"\"Test OPERA HDF5 importer: max. reflectivity composites from CIRRUS.\"\"\"\n smart_assert(metadata_cirrus[variable], expected, tolerance)\n\n\n# list of (variable,expected,tolerance) tuples\ntest_nimbus_rain_rate_attrs = [\n (\"projection\", expected_proj, None),\n (\"ll_lon\", -10.434599999137568, 1e-10),\n (\"ll_lat\", 31.74619995126678, 1e-10),\n (\"ur_lon\", 57.8119032106317, 1e-10),\n (\"ur_lat\", 67.62104536996274, 1e-10),\n (\"x1\", -2.5302714337594807, 1e-6),\n (\"y1\", -4400001.031169886, 1e-10),\n (\"x2\", 3799997.4700817037, 1e-10),\n (\"y2\", -1.0300876162946224, 1e-6),\n (\"xpixelsize\", 2000.0, 1e-10),\n (\"ypixelsize\", 2000.0, 1e-10),\n (\"cartesian_unit\", \"m\", None),\n (\"accutime\", 15.0, 1e-10),\n (\"yorigin\", \"upper\", None),\n (\"unit\", \"mm/h\", None),\n (\"institution\", \"Odyssey datacentre\", None),\n (\"transform\", None, None),\n (\"zerovalue\", 0.0, 1e-10),\n (\"threshold\", 0.01, 1e-10),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_nimbus_rain_rate_attrs)\ndef test_io_import_opera_hdf5_nimbus_rain_rate_dataset_attrs(\n variable, expected, tolerance\n):\n \"\"\"Test OPERA HDF5 importer: rain rate composites from NIMBUS.\"\"\"\n smart_assert(metadata_nimbus_rain_rate[variable], expected, tolerance)\n\n\n# list of (variable,expected,tolerance) tuples\ntest_nimbus_rain_accum_attrs = [\n (\"projection\", expected_proj, None),\n (\"ll_lon\", -10.434599999137568, 1e-10),\n (\"ll_lat\", 31.74619995126678, 1e-10),\n (\"ur_lon\", 57.8119032106317, 1e-10),\n (\"ur_lat\", 67.62104536996274, 1e-10),\n (\"x1\", -2.5302714337594807, 1e-6),\n (\"y1\", -4400001.031169886, 1e-10),\n (\"x2\", 3799997.4700817037, 1e-10),\n (\"y2\", -1.0300876162946224, 1e-6),\n (\"xpixelsize\", 2000.0, 1e-10),\n (\"ypixelsize\", 2000.0, 1e-10),\n (\"cartesian_unit\", \"m\", None),\n (\"accutime\", 15.0, 1e-10),\n (\"yorigin\", \"upper\", None),\n (\"unit\", \"mm\", None),\n (\"institution\", \"Odyssey datacentre\", None),\n (\"transform\", None, None),\n (\"zerovalue\", 0.0, 1e-10),\n (\"threshold\", 0.01, 1e-10),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_nimbus_rain_accum_attrs)\ndef test_io_import_opera_hdf5_nimbus_rain_accum_dataset_attrs(\n variable, expected, tolerance\n):\n \"\"\"Test OPERA HDF5 importer: rain accumulation composites from NIMBUS.\"\"\"\n smart_assert(metadata_nimbus_rain_accum[variable], expected, tolerance)\n" }, { "path": "pysteps/tests/test_io_readers.py", "content": "from datetime import datetime\n\nimport numpy as np\nimport pytest\n\nimport pysteps\n\n\ndef test_read_timeseries_mch():\n pytest.importorskip(\"PIL\")\n\n date = datetime.strptime(\"201505151630\", \"%Y%m%d%H%M\")\n data_source = pysteps.rcparams.data_sources[\"mch\"]\n root_path = data_source[\"root_path\"]\n path_fmt = data_source[\"path_fmt\"]\n fn_pattern = data_source[\"fn_pattern\"]\n fn_ext = data_source[\"fn_ext\"]\n importer_name = data_source[\"importer\"]\n importer_kwargs = data_source[\"importer_kwargs\"]\n timestep = data_source[\"timestep\"]\n\n fns = pysteps.io.archive.find_by_date(\n date,\n root_path,\n path_fmt,\n fn_pattern,\n fn_ext,\n timestep=timestep,\n num_prev_files=1,\n num_next_files=1,\n )\n\n importer = pysteps.io.get_method(importer_name, \"importer\")\n precip, _, metadata = pysteps.io.read_timeseries(fns, importer, **importer_kwargs)\n\n assert isinstance(precip, np.ndarray)\n assert isinstance(metadata, dict)\n assert precip.shape[0] == 3\n" }, { "path": "pysteps/tests/test_io_saf_crri.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\n\nimport pytest\n\nimport pysteps\nfrom pysteps.tests.helpers import smart_assert\n\npytest.importorskip(\"netCDF4\")\n\n\nexpected_proj = (\n \"+proj=geos +a=6378137.000000 +b=6356752.300000 \"\n \"+lon_0=0.000000 +h=35785863.000000\"\n)\ntest_geodata_crri = [\n (\"projection\", expected_proj, None),\n (\"x1\", -3301500.0, 0.1),\n (\"x2\", 3298500.0, 0.1),\n (\"y1\", 2512500.0, 0.1),\n (\"y2\", 5569500.0, 0.1),\n (\"xpixelsize\", 3000.0, 0.1),\n (\"ypixelsize\", 3000.0, 0.1),\n (\"cartesian_unit\", \"m\", None),\n (\"yorigin\", \"upper\", None),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_geodata_crri)\ndef test_io_import_saf_crri_geodata(variable, expected, tolerance):\n \"\"\"Test the importer SAF CRRI.\"\"\"\n root_path = pysteps.rcparams.data_sources[\"saf\"][\"root_path\"]\n rel_path = \"20180601/CRR\"\n filename = os.path.join(\n root_path, rel_path, \"S_NWC_CRR_MSG4_Europe-VISIR_20180601T070000Z.nc\"\n )\n geodata = pysteps.io.importers._import_saf_crri_geodata(filename)\n smart_assert(geodata[variable], expected, tolerance)\n\n\nroot_path = pysteps.rcparams.data_sources[\"saf\"][\"root_path\"]\nrel_path = \"20180601/CRR\"\nfilename = os.path.join(\n root_path, rel_path, \"S_NWC_CRR_MSG4_Europe-VISIR_20180601T070000Z.nc\"\n)\n_, _, metadata = pysteps.io.import_saf_crri(filename)\n\n# list of (variable,expected,tolerance) tuples\ntest_attrs = [\n (\"projection\", expected_proj, None),\n (\"institution\", \"Agencia Estatal de Meteorología (AEMET)\", None),\n (\"transform\", None, None),\n (\"zerovalue\", 0.0, 0.1),\n (\"unit\", \"mm/h\", None),\n (\"accutime\", None, None),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected, tolerance\", test_attrs)\ndef test_io_import_saf_crri_attrs(variable, expected, tolerance):\n \"\"\"Test the importer SAF CRRI.\"\"\"\n smart_assert(metadata[variable], expected, tolerance)\n\n\ntest_extent_crri = [\n (None, (-3301500.0, 3298500.0, 2512500.0, 5569500.0), (1019, 2200), None),\n (\n (-1980000.0, 1977000.0, 2514000.0, 4818000.0),\n (-1978500.0, 1975500.0, 2515500.0, 4816500.0),\n (767, 1318),\n None,\n ),\n]\n\n\n@pytest.mark.parametrize(\n \"extent, expected_extent, expected_shape, tolerance\", test_extent_crri\n)\ndef test_io_import_saf_crri_extent(extent, expected_extent, expected_shape, tolerance):\n \"\"\"Test the importer SAF CRRI.\"\"\"\n root_path = pysteps.rcparams.data_sources[\"saf\"][\"root_path\"]\n rel_path = \"20180601/CRR\"\n filename = os.path.join(\n root_path, rel_path, \"S_NWC_CRR_MSG4_Europe-VISIR_20180601T070000Z.nc\"\n )\n precip, _, metadata = pysteps.io.import_saf_crri(filename, extent=extent)\n extent_out = (metadata[\"x1\"], metadata[\"x2\"], metadata[\"y1\"], metadata[\"y2\"])\n smart_assert(extent_out, expected_extent, tolerance)\n smart_assert(precip.shape, expected_shape, tolerance)\n" }, { "path": "pysteps/tests/test_motion.py", "content": "# coding: utf-8\n\n\"\"\"\nTest the convergence of the optical flow methods available in\npySTEPS using idealized motion fields.\n\nTo test the convergence, using an example precipitation field we will:\n\n- Read precipitation field from a file\n- Morph the precipitation field using a given motion field (linear or rotor) to\n generate a sequence of moving precipitation patterns.\n- Using the available optical flow methods, retrieve the motion field from the\n precipitation time sequence (synthetic precipitation observations).\n\nThis tests check that the retrieved motion fields are within reasonable values.\nAlso, they will fail if any modification on the code decrease the quality of\nthe retrieval.\n\"\"\"\n\nfrom contextlib import contextmanager\n\nimport numpy as np\nimport pytest\nfrom functools import partial\nfrom scipy.ndimage import uniform_filter\n\nimport pysteps as stp\nfrom pysteps import motion\nfrom pysteps.motion.vet import morph\nfrom pysteps.tests.helpers import get_precipitation_fields\n\n\n@contextmanager\ndef not_raises(_exception):\n try:\n yield\n except _exception:\n raise pytest.fail(\"DID RAISE {0}\".format(_exception))\n\n\nreference_field = get_precipitation_fields(num_prev_files=0)\n\n\ndef _create_motion_field(input_precip, motion_type):\n \"\"\"\n Create idealized motion fields to be applied to the reference image.\n\n Parameters\n ----------\n\n input_precip: numpy array (lat, lon)\n\n motion_type : str\n The supported motion fields are:\n\n - linear_x: (u=2, v=0)\n - linear_y: (u=0, v=2)\n\n Returns\n -------\n ideal_motion : numpy array (u, v)\n \"\"\"\n\n # Create an imaginary grid on the image and create a motion field to be\n # applied to the image.\n ny, nx = input_precip.shape\n\n ideal_motion = np.zeros((2, nx, ny))\n\n if motion_type == \"linear_x\":\n ideal_motion[0, :] = 2 # Motion along x\n elif motion_type == \"linear_y\":\n ideal_motion[1, :] = 2 # Motion along y\n else:\n raise ValueError(\"motion_type not supported.\")\n\n # We need to swap the axes because the optical flow methods expect\n # (lat, lon) or (y,x) indexing convention.\n ideal_motion = ideal_motion.swapaxes(1, 2)\n return ideal_motion\n\n\ndef _create_observations(input_precip, motion_type, num_times=9):\n \"\"\"\n Create synthetic precipitation observations by displacing the input field\n using an ideal motion field.\n\n Parameters\n ----------\n\n input_precip: numpy array (lat, lon)\n Input precipitation field.\n\n motion_type : str\n The supported motion fields are:\n\n - linear_x: (u=2, v=0)\n - linear_y: (u=0, v=2)\n\n num_times: int, optional\n Length of the observations sequence.\n\n\n Returns\n -------\n synthetic_observations : numpy array\n Sequence of observations\n \"\"\"\n\n ideal_motion = _create_motion_field(input_precip, motion_type)\n\n # The morph function expects (lon, lat) or (x, y) dimensions.\n # Hence, we need to swap the lat,lon axes.\n\n # NOTE: The motion field passed to the morph function can't have any NaNs.\n # Otherwise, it can produce a segmentation fault.\n morphed_field, mask = morph(\n input_precip.swapaxes(0, 1), ideal_motion.swapaxes(1, 2)\n )\n\n mask = np.array(mask, dtype=bool)\n\n synthetic_observations = np.ma.MaskedArray(morphed_field, mask=mask)\n synthetic_observations = synthetic_observations[np.newaxis, :]\n\n for t in range(1, num_times):\n morphed_field, mask = morph(\n synthetic_observations[t - 1], ideal_motion.swapaxes(1, 2)\n )\n mask = np.array(mask, dtype=bool)\n\n morphed_field = np.ma.MaskedArray(\n morphed_field[np.newaxis, :], mask=mask[np.newaxis, :]\n )\n\n synthetic_observations = np.ma.concatenate(\n [synthetic_observations, morphed_field], axis=0\n )\n\n # Swap back to (lat, lon)\n synthetic_observations = synthetic_observations.swapaxes(1, 2)\n\n synthetic_observations = np.ma.masked_invalid(synthetic_observations)\n\n synthetic_observations.data[np.ma.getmaskarray(synthetic_observations)] = 0\n\n return ideal_motion, synthetic_observations\n\n\nconvergence_arg_names = (\n \"input_precip, optflow_method_name, motion_type, \" \"num_times, max_rel_rmse\"\n)\n\nconvergence_arg_values = [\n (reference_field, \"lk\", \"linear_x\", 2, 0.1),\n (reference_field, \"lk\", \"linear_y\", 2, 0.1),\n (reference_field, \"lk\", \"linear_x\", 3, 0.1),\n (reference_field, \"lk\", \"linear_y\", 3, 0.1),\n (reference_field, \"vet\", \"linear_x\", 2, 0.1),\n # (reference_field, 'vet', 'linear_x', 3, 9),\n # (reference_field, 'vet', 'linear_y', 2, 9),\n (reference_field, \"vet\", \"linear_y\", 3, 0.1),\n (reference_field, \"proesmans\", \"linear_x\", 2, 0.45),\n (reference_field, \"proesmans\", \"linear_y\", 2, 0.45),\n (reference_field, \"darts\", \"linear_x\", 9, 20),\n (reference_field, \"darts\", \"linear_y\", 9, 20),\n (reference_field, \"farneback\", \"linear_x\", 2, 28),\n (reference_field, \"farneback\", \"linear_y\", 2, 28),\n]\n\n\n@pytest.mark.parametrize(convergence_arg_names, convergence_arg_values)\ndef test_optflow_method_convergence(\n input_precip, optflow_method_name, motion_type, num_times, max_rel_rmse\n):\n \"\"\"\n Test the convergence to the actual solution of the optical flow method used.\n\n We measure the error in the retrieved field by using the\n Relative RMSE = Rel_RMSE = sqrt(Relative MSE)\n\n - Rel_RMSE = 0%: no error\n - Rel_RMSE = 100%: The retrieved motion field has an average error\n equal in magnitude to the motion field.\n\n Relative RMSE is computed only un a region surrounding the precipitation\n field, were we have enough information to retrieve the motion field.\n The precipitation region includes the precipitation pattern plus a margin\n of approximately 20 grid points.\n\n\n Parameters\n ----------\n\n input_precip: numpy array (lat, lon)\n Input precipitation field.\n\n optflow_method_name: str\n Optical flow method name\n\n motion_type : str\n The supported motion fields are:\n\n - linear_x: (u=2, v=0)\n - linear_y: (u=0, v=2)\n \"\"\"\n if optflow_method_name == \"lk\":\n pytest.importorskip(\"cv2\")\n\n ideal_motion, precip_obs = _create_observations(\n input_precip.copy(), motion_type, num_times=num_times\n )\n\n oflow_method = motion.get_method(optflow_method_name)\n\n if optflow_method_name == \"vet\":\n # By default, the maximum number of iteration in the VET minimization\n # is maxiter=100.\n # To increase the stability of the tests to we increase this value to\n # maxiter=150.\n retrieved_motion = oflow_method(\n precip_obs, verbose=False, options=dict(maxiter=150)\n )\n elif optflow_method_name == \"proesmans\":\n retrieved_motion = oflow_method(precip_obs)\n else:\n retrieved_motion = oflow_method(precip_obs, verbose=False)\n\n precip_data, _ = stp.utils.dB_transform(precip_obs.max(axis=0), inverse=True)\n precip_data.data[precip_data.mask] = 0\n\n precip_mask = (uniform_filter(precip_data, size=20) > 0.1) & ~precip_obs.mask.any(\n axis=0\n )\n\n # To evaluate the accuracy of the computed_motion vectors, we will use\n # a relative RMSE measure.\n # Relative MSE = < (expected_motion - computed_motion)^2 > / \n # Relative RMSE = sqrt(Relative MSE)\n\n mse = ((ideal_motion - retrieved_motion)[:, precip_mask] ** 2).mean()\n\n rel_mse = mse / (ideal_motion[:, precip_mask] ** 2).mean()\n rel_rmse = np.sqrt(rel_mse) * 100\n print(\n f\"method:{optflow_method_name} ; \"\n f\"motion:{motion_type} ; times: {num_times} ; \"\n f\"rel_rmse:{rel_rmse:.2f}%\"\n )\n assert rel_rmse < max_rel_rmse\n\n\nno_precip_args_names = \"optflow_method_name, num_times\"\nno_precip_args_values = [\n (\"lk\", 2),\n (\"lk\", 3),\n (\"vet\", 2),\n (\"vet\", 3),\n (\"darts\", 9),\n (\"proesmans\", 2),\n (\"farneback\", 2),\n]\n\n\n@pytest.mark.parametrize(no_precip_args_names, no_precip_args_values)\ndef test_no_precipitation(optflow_method_name, num_times):\n \"\"\"\n Test that the motion methods work well with a zero precipitation in the\n domain.\n\n The expected result is a zero motion vector.\n\n Parameters\n ----------\n\n optflow_method_name: str\n Optical flow method name\n\n num_times : int\n Number of precipitation frames (times) used as input for the optical\n flow methods.\n \"\"\"\n if optflow_method_name == \"lk\":\n pytest.importorskip(\"cv2\")\n zero_precip = np.zeros((num_times,) + reference_field.shape)\n motion_method = motion.get_method(optflow_method_name)\n uv_motion = motion_method(zero_precip, verbose=False)\n\n assert np.abs(uv_motion).max() < 0.01\n\n\ninput_tests_args_names = (\n \"optflow_method_name\",\n \"minimum_input_frames\",\n \"maximum_input_frames\",\n)\ninput_tests_args_values = [\n (\"lk\", 2, np.inf),\n (\"vet\", 2, 3),\n (\"darts\", 9, 9),\n (\"proesmans\", 2, 2),\n (\"farneback\", 2, np.inf),\n]\n\n\n@pytest.mark.parametrize(input_tests_args_names, input_tests_args_values)\ndef test_input_shape_checks(\n optflow_method_name, minimum_input_frames, maximum_input_frames\n):\n if optflow_method_name in (\"lk\", \"farneback\"):\n pytest.importorskip(\"cv2\")\n image_size = 100\n motion_method = motion.get_method(optflow_method_name)\n\n if maximum_input_frames == np.inf:\n maximum_input_frames = minimum_input_frames + 10\n\n with not_raises(Exception):\n for frames in range(minimum_input_frames, maximum_input_frames + 1):\n motion_method(np.zeros((frames, image_size, image_size)), verbose=False)\n\n with pytest.raises(ValueError):\n motion_method(np.zeros((2,)))\n motion_method(np.zeros((2, 2)))\n for frames in range(minimum_input_frames):\n motion_method(np.zeros((frames, image_size, image_size)), verbose=False)\n for frames in range(maximum_input_frames + 1, maximum_input_frames + 4):\n motion_method(np.zeros((frames, image_size, image_size)), verbose=False)\n\n\ndef test_vet_padding():\n \"\"\"\n Test that the padding functionality in vet works correctly with ndarrays and\n masked arrays.\n \"\"\"\n\n _, precip_obs = _create_observations(\n reference_field.copy(), \"linear_y\", num_times=2\n )\n\n # Use a small region to speed up the test\n precip_obs = precip_obs[:, 200:427, 250:456]\n # precip_obs.shape == (227 , 206)\n # 227 is a prime number ; 206 = 2*103\n # Using this shape will force vet to internally pad the input array for the sector's\n # blocks to divide exactly the input shape.\n # NOTE: This \"internal padding\" is different from the padding keyword being test next.\n\n for padding in [0, 3, 10]:\n # No padding\n vet_method = partial(\n motion.get_method(\"vet\"),\n verbose=False,\n sectors=((16, 4, 2), (16, 4, 2)),\n options=dict(maxiter=5),\n padding=padding,\n # We use only a few iterations since\n # we don't care about convergence in this test\n )\n\n assert precip_obs.shape == vet_method(precip_obs).shape\n assert precip_obs.shape == vet_method(np.ma.masked_invalid(precip_obs)).shape\n\n\ndef test_vet_cost_function():\n \"\"\"\n Test that the vet cost_function computation gives always the same result\n with the same input.\n\n Useful to test if the parallelization in VET produce undesired results.\n \"\"\"\n\n from pysteps.motion import vet\n\n ideal_motion, precip_obs = _create_observations(\n reference_field.copy(), \"linear_y\", num_times=2\n )\n\n mask_2d = np.ma.getmaskarray(precip_obs).any(axis=0).astype(\"int8\")\n\n returned_values = np.zeros(20)\n\n for i in range(20):\n returned_values[i] = vet.vet_cost_function(\n ideal_motion.ravel(), # sector_displacement_1d\n precip_obs.data, # input_images\n ideal_motion.shape[1:], # blocks_shape (same as 2D grid)\n mask_2d, # Mask\n 1e6, # smooth_gain\n debug=False,\n )\n\n tolerance = 1e-12\n errors = np.abs(returned_values - returned_values[0])\n # errors should contain all zeros\n assert (errors < tolerance).any()\n assert (returned_values[0] - 1548250.87627097) < 0.001\n\n\n@pytest.mark.parametrize(\n \"method,kwargs\",\n [\n (\"LK\", {\"fd_kwargs\": {\"buffer_mask\": 20}, \"verbose\": False}),\n (\"farneback\", {\"verbose\": False}),\n ],\n)\ndef test_motion_masked_array(method, kwargs):\n \"\"\"\n Passing a ndarray with NaNs or a masked array should produce the same results.\n Tests for both LK and Farneback motion estimation methods.\n \"\"\"\n pytest.importorskip(\"cv2\")\n\n __, precip_obs = _create_observations(\n reference_field.copy(), \"linear_y\", num_times=2\n )\n motion_method = motion.get_method(method)\n\n # ndarray with nans\n np.ma.set_fill_value(precip_obs, -15)\n ndarray = precip_obs.filled()\n ndarray[ndarray == -15] = np.nan\n uv_ndarray = motion_method(ndarray, **kwargs)\n\n # masked array\n mdarray = np.ma.masked_invalid(ndarray)\n mdarray.data[mdarray.mask] = -15\n uv_mdarray = motion_method(mdarray, **kwargs)\n\n assert np.abs(uv_mdarray - uv_ndarray).max() < 0.01\n" }, { "path": "pysteps/tests/test_motion_farneback.py", "content": "import pytest\nimport numpy as np\n\nfrom pysteps.motion import farneback\nfrom pysteps.exceptions import MissingOptionalDependency\nfrom pysteps.tests.helpers import get_precipitation_fields\n\nfb_arg_names = (\n \"pyr_scale\",\n \"levels\",\n \"winsize\",\n \"iterations\",\n \"poly_n\",\n \"poly_sigma\",\n \"flags\",\n \"size_opening\",\n \"sigma\",\n \"verbose\",\n)\n\nfb_arg_values = [\n (0.5, 3, 15, 3, 5, 1.1, 0, 3, 60.0, False), # default\n (0.5, 1, 5, 1, 7, 1.5, 0, 0, 0.0, True), # minimal settings, sigma=0, verbose\n (\n 0.3,\n 5,\n 30,\n 10,\n 7,\n 1.5,\n 1,\n 5,\n 10.0,\n False,\n ), # maximal settings, flags=1, big opening\n (0.5, 3, 15, 3, 5, 1.1, 0, 0, 60.0, True), # no opening, verbose\n]\n\n\n@pytest.mark.parametrize(fb_arg_names, fb_arg_values)\ndef test_farneback_params(\n pyr_scale,\n levels,\n winsize,\n iterations,\n poly_n,\n poly_sigma,\n flags,\n size_opening,\n sigma,\n verbose,\n):\n \"\"\"Test Farneback with various parameters and input types.\"\"\"\n pytest.importorskip(\"cv2\")\n # Input: realistic precipitation fields\n precip, metadata = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n return_raw=False,\n metadata=True,\n upscale=2000,\n )\n precip = precip.filled()\n\n output = farneback.farneback(\n precip,\n pyr_scale=pyr_scale,\n levels=levels,\n winsize=winsize,\n iterations=iterations,\n poly_n=poly_n,\n poly_sigma=poly_sigma,\n flags=flags,\n size_opening=size_opening,\n sigma=sigma,\n verbose=verbose,\n )\n\n assert isinstance(output, np.ndarray)\n assert output.shape[0] == 2\n assert output.shape[1:] == precip[0].shape\n assert np.isfinite(output).all() or np.isnan(output).any()\n\n\ndef test_farneback_invalid_shape():\n \"\"\"Test error when input is wrong shape.\"\"\"\n pytest.importorskip(\"cv2\")\n arr = np.random.rand(64, 64)\n with pytest.raises(ValueError):\n farneback.farneback(arr)\n\n\ndef test_farneback_nan_input():\n \"\"\"Test NaN handling in input.\"\"\"\n pytest.importorskip(\"cv2\")\n arr = np.random.rand(2, 64, 64)\n arr[0, 0, 0] = np.nan\n arr[1, 10, 10] = np.inf\n result = farneback.farneback(arr)\n assert result.shape == (2, 64, 64)\n\n\ndef test_farneback_cv2_missing(monkeypatch):\n \"\"\"Test MissingOptionalDependency when cv2 is not injected.\"\"\"\n monkeypatch.setattr(farneback, \"CV2_IMPORTED\", False)\n arr = np.random.rand(2, 64, 64)\n with pytest.raises(MissingOptionalDependency):\n farneback.farneback(arr)\n monkeypatch.setattr(farneback, \"CV2_IMPORTED\", True) # restore\n\n\ndef test_farneback_sigma_zero():\n \"\"\"Test sigma=0 disables smoothing logic.\"\"\"\n pytest.importorskip(\"cv2\")\n arr = np.random.rand(2, 32, 32)\n result = farneback.farneback(arr, sigma=0.0)\n assert isinstance(result, np.ndarray)\n assert result.shape == (2, 32, 32)\n\n\ndef test_farneback_small_window():\n \"\"\"Test winsize edge case behavior.\"\"\"\n pytest.importorskip(\"cv2\")\n arr = np.random.rand(2, 16, 16)\n result = farneback.farneback(arr, winsize=3)\n assert result.shape == (2, 16, 16)\n\n\ndef test_farneback_verbose(capsys):\n \"\"\"Test that verbose produces output (side-effect only).\"\"\"\n pytest.importorskip(\"cv2\")\n arr = np.random.rand(2, 16, 16)\n farneback.farneback(arr, verbose=True)\n out = capsys.readouterr().out\n assert \"Farneback method\" in out or \"mult factor\" in out or \"---\" in out\n" }, { "path": "pysteps/tests/test_motion_lk.py", "content": "# coding: utf-8\n\n\"\"\" \"\"\"\n\nimport pytest\nimport numpy as np\n\nfrom pysteps import motion, verification\nfrom pysteps.tests.helpers import get_precipitation_fields\n\nlk_arg_names = (\n \"lk_kwargs\",\n \"fd_method\",\n \"dense\",\n \"nr_std_outlier\",\n \"k_outlier\",\n \"size_opening\",\n \"decl_scale\",\n \"verbose\",\n)\n\nlk_arg_values = [\n ({}, \"shitomasi\", True, 3, 30, 3, 20, False), # defaults\n ({}, \"shitomasi\", False, 3, 30, 3, 20, True), # sparse ouput, verbose\n ({}, \"shitomasi\", False, 0, 30, 3, 20, False), # sparse ouput, all outliers\n (\n {},\n \"shitomasi\",\n True,\n 3,\n None,\n 0,\n 0,\n False,\n ), # global outlier detection, no filtering, no declutering\n ({}, \"shitomasi\", True, 0, 30, 3, 20, False), # all outliers\n ({}, \"blob\", True, 3, 30, 3, 20, False), # blob detection\n ({}, \"tstorm\", True, 3, 30, 3, 20, False), # tstorm detection\n]\n\n\n@pytest.mark.parametrize(lk_arg_names, lk_arg_values)\ndef test_lk(\n lk_kwargs,\n fd_method,\n dense,\n nr_std_outlier,\n k_outlier,\n size_opening,\n decl_scale,\n verbose,\n):\n \"\"\"Tests Lucas-Kanade optical flow.\"\"\"\n\n pytest.importorskip(\"cv2\")\n if fd_method == \"blob\":\n pytest.importorskip(\"skimage\")\n if fd_method == \"tstorm\":\n pytest.importorskip(\"skimage\")\n pytest.importorskip(\"pandas\")\n\n # inputs\n precip, metadata = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n return_raw=False,\n metadata=True,\n upscale=2000,\n )\n precip = precip.filled()\n\n # Retrieve motion field\n oflow_method = motion.get_method(\"LK\")\n output = oflow_method(\n precip,\n lk_kwargs=lk_kwargs,\n fd_method=fd_method,\n dense=dense,\n nr_std_outlier=nr_std_outlier,\n k_outlier=k_outlier,\n size_opening=size_opening,\n decl_scale=decl_scale,\n verbose=verbose,\n )\n\n # Check format of ouput\n if dense:\n assert isinstance(output, np.ndarray)\n assert output.ndim == 3\n assert output.shape[0] == 2\n assert output.shape[1:] == precip[0].shape\n if nr_std_outlier == 0:\n assert output.sum() == 0\n else:\n assert isinstance(output, tuple)\n assert len(output) == 2\n assert isinstance(output[0], np.ndarray)\n assert isinstance(output[1], np.ndarray)\n assert output[0].ndim == 2\n assert output[1].ndim == 2\n assert output[0].shape[1] == 2\n assert output[1].shape[1] == 2\n assert output[0].shape[0] == output[1].shape[0]\n if nr_std_outlier == 0:\n assert output[0].shape[0] == 0\n assert output[1].shape[0] == 0\n" }, { "path": "pysteps/tests/test_noise_fftgenerators.py", "content": "import numpy as np\n\nfrom pysteps.noise import fftgenerators\nfrom pysteps.tests.helpers import get_precipitation_fields\n\nPRECIP = get_precipitation_fields(\n num_prev_files=0,\n num_next_files=0,\n return_raw=False,\n metadata=False,\n upscale=2000,\n)\nPRECIP = PRECIP.filled()\n\n\ndef test_noise_param_2d_fft_filter():\n fft_filter = fftgenerators.initialize_param_2d_fft_filter(PRECIP)\n\n assert isinstance(fft_filter, dict)\n assert all([key in fft_filter for key in [\"field\", \"input_shape\", \"model\", \"pars\"]])\n\n out = fftgenerators.generate_noise_2d_fft_filter(fft_filter)\n\n assert isinstance(out, np.ndarray)\n assert out.shape == PRECIP.shape\n\n\ndef test_noise_nonparam_2d_fft_filter():\n fft_filter = fftgenerators.initialize_nonparam_2d_fft_filter(PRECIP)\n\n assert isinstance(fft_filter, dict)\n assert all([key in fft_filter for key in [\"field\", \"input_shape\"]])\n\n out = fftgenerators.generate_noise_2d_fft_filter(fft_filter)\n\n assert isinstance(out, np.ndarray)\n assert out.shape == PRECIP.shape\n\n\ndef test_noise_nonparam_2d_ssft_filter():\n fft_filter = fftgenerators.initialize_nonparam_2d_ssft_filter(PRECIP)\n\n assert isinstance(fft_filter, dict)\n assert all([key in fft_filter for key in [\"field\", \"input_shape\"]])\n\n out = fftgenerators.generate_noise_2d_ssft_filter(fft_filter)\n\n assert isinstance(out, np.ndarray)\n assert out.shape == PRECIP.shape\n\n\ndef test_noise_nonparam_2d_nested_filter():\n fft_filter = fftgenerators.initialize_nonparam_2d_nested_filter(PRECIP)\n\n assert isinstance(fft_filter, dict)\n assert all([key in fft_filter for key in [\"field\", \"input_shape\"]])\n\n out = fftgenerators.generate_noise_2d_ssft_filter(fft_filter)\n\n assert isinstance(out, np.ndarray)\n assert out.shape == PRECIP.shape\n" }, { "path": "pysteps/tests/test_noise_motion.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.noise.motion import generate_bps\nfrom pysteps.noise.motion import get_default_params_bps_par\nfrom pysteps.noise.motion import get_default_params_bps_perp\nfrom pysteps.noise.motion import initialize_bps\n\n\ndef test_noise_motion_get_default_params_bps_par():\n \"\"\"Tests default BPS velocity parameters.\"\"\"\n expected = (10.88, 0.23, -7.68)\n result = get_default_params_bps_par()\n assert_array_almost_equal(result, expected)\n\n\ndef test_noise_motion_get_default_params_bps_perp():\n \"\"\"Tests default BPS velocity perturbation.\"\"\"\n expected = (5.76, 0.31, -2.72)\n result = get_default_params_bps_perp()\n assert_array_almost_equal(result, expected)\n\n\nvv = np.ones((8, 8)) * np.sqrt(2) * 0.5\ntest_init_bps_vars = [\n (\"vsf\", 60),\n (\"eps_par\", -0.2042896366299448),\n (\"eps_perp\", 1.6383482042624593),\n (\"p_par\", (10.88, 0.23, -7.68)),\n (\"p_perp\", (5.76, 0.31, -2.72)),\n (\"V_par\", np.stack([vv, vv])),\n (\"V_perp\", np.stack([-vv, vv])),\n]\n\n\n@pytest.mark.parametrize(\"variable, expected\", test_init_bps_vars)\ndef test_initialize_bps(variable, expected):\n \"\"\"Tests initialation BPS velocity perturbation method.\"\"\"\n seed = 42\n timestep = 1\n pixelsperkm = 1\n v = np.ones((8, 8))\n velocity = np.stack([v, v])\n perturbator = initialize_bps(velocity, pixelsperkm, timestep, seed=seed)\n assert_array_almost_equal(perturbator[variable], expected)\n\n\ndef test_generate_bps():\n \"\"\"Tests generation BPS velocity perturbation method.\"\"\"\n seed = 42\n timestep = 1\n pixelsperkm = 1\n v = np.ones((8, 8))\n velocity = np.stack([v, v])\n perturbator = initialize_bps(velocity, pixelsperkm, timestep, seed=seed)\n new_vv = generate_bps(perturbator, timestep)\n expected = np.stack([v * -0.066401, v * 0.050992])\n assert_array_almost_equal(new_vv, expected)\n" }, { "path": "pysteps/tests/test_nowcasts_anvil.py", "content": "import numpy as np\nimport pytest\n\nfrom pysteps import motion, nowcasts, verification\nfrom pysteps.tests.helpers import get_precipitation_fields\n\nanvil_arg_names = (\n \"n_cascade_levels\",\n \"ar_order\",\n \"ar_window_radius\",\n \"timesteps\",\n \"min_csi\",\n \"apply_rainrate_mask\",\n \"measure_time\",\n)\n\nanvil_arg_values = [\n (8, 1, 50, 3, 0.6, True, False),\n (8, 1, 50, [3], 0.6, False, True),\n]\n\n\ndef test_default_anvil_norain():\n \"\"\"Tests anvil nowcast with default params and all-zero inputs.\"\"\"\n\n # Define dummy nowcast input data\n precip_input = np.zeros((4, 100, 100))\n\n pytest.importorskip(\"cv2\")\n oflow_method = motion.get_method(\"LK\")\n retrieved_motion = oflow_method(precip_input)\n\n nowcast_method = nowcasts.get_method(\"anvil\")\n precip_forecast = nowcast_method(\n precip_input,\n retrieved_motion,\n timesteps=3,\n )\n\n assert precip_forecast.ndim == 3\n assert precip_forecast.shape[0] == 3\n assert precip_forecast.sum() == 0.0\n\n\n@pytest.mark.parametrize(anvil_arg_names, anvil_arg_values)\ndef test_anvil_rainrate(\n n_cascade_levels,\n ar_order,\n ar_window_radius,\n timesteps,\n min_csi,\n apply_rainrate_mask,\n measure_time,\n):\n \"\"\"Tests ANVIL nowcast using rain rate precipitation fields.\"\"\"\n # inputs\n precip_input = get_precipitation_fields(\n num_prev_files=4,\n num_next_files=0,\n return_raw=False,\n metadata=False,\n upscale=2000,\n )\n precip_input = precip_input.filled()\n\n precip_obs = get_precipitation_fields(\n num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000\n )[1:, :, :]\n precip_obs = precip_obs.filled()\n\n pytest.importorskip(\"cv2\")\n oflow_method = motion.get_method(\"LK\")\n retrieved_motion = oflow_method(precip_input)\n\n nowcast_method = nowcasts.get_method(\"anvil\")\n\n output = nowcast_method(\n precip_input[-(ar_order + 2) :],\n retrieved_motion,\n timesteps=timesteps,\n rainrate=None, # no R(VIL) conversion is done\n n_cascade_levels=n_cascade_levels,\n ar_order=ar_order,\n ar_window_radius=ar_window_radius,\n apply_rainrate_mask=apply_rainrate_mask,\n measure_time=measure_time,\n )\n if measure_time:\n precip_forecast, __, __ = output\n else:\n precip_forecast = output\n\n assert precip_forecast.ndim == 3\n assert precip_forecast.shape[0] == (\n timesteps if isinstance(timesteps, int) else len(timesteps)\n )\n\n result = verification.det_cat_fct(\n precip_forecast[-1], precip_obs[-1], thr=0.1, scores=\"CSI\"\n )[\"CSI\"]\n assert result > min_csi, f\"CSI={result:.2f}, required > {min_csi:.2f}\"\n\n\nif __name__ == \"__main__\":\n for n in range(len(anvil_arg_values)):\n test_args = zip(anvil_arg_names, anvil_arg_values[n])\n test_anvil_rainrate(**dict((x, y) for x, y in test_args))\n" }, { "path": "pysteps/tests/test_nowcasts_lagrangian_probability.py", "content": "# -*- coding: utf-8 -*-\nimport numpy as np\nimport pytest\n\nfrom pysteps.nowcasts.lagrangian_probability import forecast\nfrom pysteps.tests.helpers import get_precipitation_fields\nfrom pysteps.motion.lucaskanade import dense_lucaskanade\n\n\ndef test_numerical_example():\n \"\"\"\"\"\"\n precip = np.zeros((20, 20))\n precip[5:10, 5:10] = 1\n velocity = np.zeros((2, *precip.shape))\n timesteps = 4\n thr = 0.5\n slope = 1 # pixels / timestep\n\n # compute probability forecast\n fct = forecast(precip, velocity, timesteps, thr, slope=slope)\n\n assert fct.ndim == 3\n assert fct.shape[0] == timesteps\n assert fct.shape[1:] == precip.shape\n assert fct.max() <= 1.0\n assert fct.min() >= 0.0\n\n # slope = 0 should return a binary field\n fct = forecast(precip, velocity, timesteps, thr, slope=0)\n ref = (np.repeat(precip[None, ...], timesteps, axis=0) >= thr).astype(float)\n assert np.allclose(fct, fct.astype(bool))\n assert np.allclose(fct, ref)\n\n\ndef test_numerical_example_with_float_slope_and_float_list_timesteps():\n \"\"\"\"\"\"\n precip = np.zeros((20, 20))\n precip[5:10, 5:10] = 1\n velocity = np.zeros((2, *precip.shape))\n timesteps = [1.0, 2.0, 5.0, 12.0]\n thr = 0.5\n slope = 1.0 # pixels / timestep\n\n # compute probability forecast\n fct = forecast(precip, velocity, timesteps, thr, slope=slope)\n\n assert fct.ndim == 3\n assert fct.shape[0] == len(timesteps)\n assert fct.shape[1:] == precip.shape\n assert fct.max() <= 1.0\n assert fct.min() >= 0.0\n\n\ndef test_real_case():\n \"\"\"\"\"\"\n pytest.importorskip(\"cv2\")\n\n # inputs\n precip, metadata = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n return_raw=False,\n metadata=True,\n upscale=2000,\n )\n\n # motion\n motion = dense_lucaskanade(precip)\n\n # parameters\n timesteps = [1, 2, 3]\n thr = 1 # mm / h\n slope = 1 * metadata[\"accutime\"] # min-1\n\n # compute probability forecast\n extrap_kwargs = dict(allow_nonfinite_values=True)\n fct = forecast(\n precip[-1], motion, timesteps, thr, slope=slope, extrap_kwargs=extrap_kwargs\n )\n\n assert fct.ndim == 3\n assert fct.shape[0] == len(timesteps)\n assert fct.shape[1:] == precip.shape[1:]\n assert np.nanmax(fct) <= 1.0\n assert np.nanmin(fct) >= 0.0\n\n\ndef test_wrong_inputs():\n # dummy inputs\n precip = np.zeros((3, 3))\n velocity = np.zeros((2, *precip.shape))\n\n # timesteps must be > 0\n with pytest.raises(ValueError):\n forecast(precip, velocity, 0, 1)\n\n # timesteps must be a sorted list\n with pytest.raises(ValueError):\n forecast(precip, velocity, [2, 1], 1)\n" }, { "path": "pysteps/tests/test_nowcasts_linda.py", "content": "from datetime import timedelta\nimport os\nimport numpy as np\nimport pytest\n\nfrom pysteps import io, motion, nowcasts, verification\nfrom pysteps.nowcasts.linda import forecast\nfrom pysteps.tests.helpers import get_precipitation_fields\n\nlinda_arg_names = (\n \"timesteps\",\n \"add_perturbations\",\n \"kernel_type\",\n \"vel_pert_method\",\n \"num_workers\",\n \"measure_time\",\n \"min_csi\",\n \"max_crps\",\n)\n\nlinda_arg_values = [\n (3, False, \"anisotropic\", None, 1, False, 0.5, None),\n ([3], False, \"anisotropic\", None, 1, False, 0.5, None),\n (3, False, \"isotropic\", None, 5, True, 0.5, None),\n (3, True, \"anisotropic\", None, 1, True, None, 0.3),\n (3, True, \"isotropic\", \"bps\", 5, True, None, 0.3),\n]\n\n\ndef test_default_linda_norain():\n \"\"\"Tests linda nowcast with default params and all-zero inputs.\"\"\"\n\n # Define dummy nowcast input data\n precip_input = np.zeros((3, 100, 100))\n\n pytest.importorskip(\"cv2\")\n oflow_method = motion.get_method(\"LK\")\n retrieved_motion = oflow_method(precip_input)\n\n nowcast_method = nowcasts.get_method(\"linda\")\n precip_forecast = nowcast_method(\n precip_input,\n retrieved_motion,\n n_ens_members=3,\n timesteps=3,\n kmperpixel=1,\n timestep=5,\n )\n\n assert precip_forecast.ndim == 4\n assert precip_forecast.shape[0] == 3\n assert precip_forecast.shape[1] == 3\n assert precip_forecast.sum() == 0.0\n\n\n@pytest.mark.parametrize(linda_arg_names, linda_arg_values)\ndef test_linda(\n timesteps,\n add_perturbations,\n kernel_type,\n vel_pert_method,\n num_workers,\n measure_time,\n min_csi,\n max_crps,\n):\n \"\"\"Tests LINDA nowcast.\"\"\"\n\n pytest.importorskip(\"cv2\")\n pytest.importorskip(\"skimage\")\n\n # inputs\n precip_input, metadata = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n metadata=True,\n clip=(354000, 866000, -96000, 416000),\n upscale=4000,\n log_transform=False,\n )\n\n precip_obs = get_precipitation_fields(\n num_prev_files=0,\n num_next_files=3,\n clip=(354000, 866000, -96000, 416000),\n upscale=4000,\n log_transform=False,\n )[1:, :, :]\n\n oflow_method = motion.get_method(\"LK\")\n retrieved_motion = oflow_method(precip_input)\n\n precip_forecast = forecast(\n precip_input,\n retrieved_motion,\n timesteps,\n kernel_type=kernel_type,\n vel_pert_method=vel_pert_method,\n feature_kwargs={\"threshold\": 1.5, \"min_sigma\": 2, \"max_sigma\": 10},\n add_perturbations=add_perturbations,\n kmperpixel=4.0,\n timestep=metadata[\"accutime\"],\n measure_time=measure_time,\n n_ens_members=5,\n num_workers=num_workers,\n seed=42,\n )\n num_nowcast_timesteps = timesteps if isinstance(timesteps, int) else len(timesteps)\n if measure_time:\n assert len(precip_forecast) == num_nowcast_timesteps\n assert isinstance(precip_forecast[1], float)\n precip_forecast = precip_forecast[0]\n\n if not add_perturbations:\n assert precip_forecast.ndim == 3\n assert precip_forecast.shape[0] == num_nowcast_timesteps\n assert precip_forecast.shape[1:] == precip_input.shape[1:]\n\n csi = verification.det_cat_fct(\n precip_forecast[-1], precip_obs[-1], thr=1.0, scores=\"CSI\"\n )[\"CSI\"]\n assert csi > min_csi, f\"CSI={csi:.1f}, required > {min_csi:.1f}\"\n else:\n assert precip_forecast.ndim == 4\n assert precip_forecast.shape[0] == 5\n assert precip_forecast.shape[1] == num_nowcast_timesteps\n assert precip_forecast.shape[2:] == precip_input.shape[1:]\n\n crps = verification.probscores.CRPS(precip_forecast[:, -1], precip_obs[-1])\n assert crps < max_crps, f\"CRPS={crps:.2f}, required < {max_crps:.2f}\"\n\n\ndef test_linda_wrong_inputs():\n # dummy inputs\n precip = np.zeros((3, 3, 3))\n velocity = np.zeros((2, 3, 3))\n\n # vel_pert_method is set but kmperpixel is None\n with pytest.raises(ValueError):\n forecast(precip, velocity, 1, vel_pert_method=\"bps\", kmperpixel=None)\n\n # vel_pert_method is set but timestep is None\n with pytest.raises(ValueError):\n forecast(\n precip, velocity, 1, vel_pert_method=\"bps\", kmperpixel=1, timestep=None\n )\n\n # ari_order 1 or 2 required\n with pytest.raises(ValueError):\n forecast(precip, velocity, 1, ari_order=3)\n\n # precip_fields must be a three-dimensional array\n with pytest.raises(ValueError):\n forecast(np.zeros((3, 3, 3, 3)), velocity, 1)\n\n # precip_fields.shape[0] < ari_order+2\n with pytest.raises(ValueError):\n forecast(np.zeros((2, 3, 3)), velocity, 1, ari_order=1)\n\n # advection_field must be a three-dimensional array\n with pytest.raises(ValueError):\n forecast(precip, velocity[0], 1)\n\n # dimension mismatch between precip_fields and advection_field\n with pytest.raises(ValueError):\n forecast(np.zeros((3, 2, 3)), velocity, 1)\n\n\ndef test_linda_callback(tmp_path):\n \"\"\"Test LINDA callback functionality to export the output as a netcdf.\"\"\"\n\n pytest.importorskip(\"skimage\")\n\n n_ens_members = 2\n n_timesteps = 3\n\n precip_input, metadata = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n return_raw=False,\n metadata=True,\n upscale=2000,\n )\n precip_input = precip_input.filled()\n field_shape = (precip_input.shape[1], precip_input.shape[2])\n startdate = metadata[\"timestamps\"][-1]\n timestep = metadata[\"accutime\"]\n\n motion_field = np.zeros((2, *field_shape))\n\n exporter = io.initialize_forecast_exporter_netcdf(\n outpath=tmp_path.as_posix(),\n outfnprefix=\"test_linda\",\n startdate=startdate,\n timestep=timestep,\n n_timesteps=n_timesteps,\n shape=field_shape,\n n_ens_members=n_ens_members,\n metadata=metadata,\n incremental=\"timestep\",\n )\n\n def callback(array):\n return io.export_forecast_dataset(array, exporter)\n\n precip_output = nowcasts.get_method(\"linda\")(\n precip_input,\n motion_field,\n timesteps=n_timesteps,\n add_perturbations=False,\n n_ens_members=n_ens_members,\n kmperpixel=4.0,\n timestep=metadata[\"accutime\"],\n callback=callback,\n return_output=True,\n )\n io.close_forecast_files(exporter)\n\n # assert that netcdf exists and its size is not zero\n tmp_file = os.path.join(tmp_path, \"test_linda.nc\")\n assert os.path.exists(tmp_file) and os.path.getsize(tmp_file) > 0\n\n # assert that the file can be read by the nowcast importer\n precip_netcdf, metadata_netcdf = io.import_netcdf_pysteps(tmp_file, dtype=\"float64\")\n\n # assert that the dimensionality of the array is as expected\n assert precip_netcdf.ndim == 4, \"Wrong number of dimensions\"\n assert precip_netcdf.shape[0] == n_ens_members, \"Wrong ensemble size\"\n assert precip_netcdf.shape[1] == n_timesteps, \"Wrong number of lead times\"\n assert precip_netcdf.shape[2:] == field_shape, \"Wrong field shape\"\n\n # assert that the saved output is the same as the original output\n assert np.allclose(\n precip_netcdf, precip_output, equal_nan=True\n ), \"Wrong output values\"\n\n # assert that leadtimes and timestamps are as expected\n td = timedelta(minutes=timestep)\n leadtimes = [(i + 1) * timestep for i in range(n_timesteps)]\n timestamps = [startdate + (i + 1) * td for i in range(n_timesteps)]\n assert (metadata_netcdf[\"leadtimes\"] == leadtimes).all(), \"Wrong leadtimes\"\n assert (metadata_netcdf[\"timestamps\"] == timestamps).all(), \"Wrong timestamps\"\n" }, { "path": "pysteps/tests/test_nowcasts_sprog.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\n\nfrom pysteps import motion, nowcasts, verification\nfrom pysteps.tests.helpers import get_precipitation_fields\n\nsprog_arg_names = (\n \"n_cascade_levels\",\n \"ar_order\",\n \"probmatching_method\",\n \"domain\",\n \"timesteps\",\n \"min_csi\",\n)\n\nsprog_arg_values = [\n (6, 1, None, \"spatial\", 3, 0.5),\n (6, 1, None, \"spatial\", [3], 0.5),\n (6, 2, None, \"spatial\", 3, 0.5),\n (6, 2, \"cdf\", \"spatial\", 3, 0.5),\n (6, 2, \"mean\", \"spatial\", 3, 0.5),\n (6, 2, \"cdf\", \"spectral\", 3, 0.5),\n]\n\n\ndef test_default_sprog_norain():\n \"\"\"Tests SPROG nowcast with default params and all-zero inputs.\"\"\"\n\n # Define dummy nowcast input data\n precip_input = np.zeros((3, 100, 100))\n\n pytest.importorskip(\"cv2\")\n oflow_method = motion.get_method(\"LK\")\n retrieved_motion = oflow_method(precip_input)\n\n nowcast_method = nowcasts.get_method(\"sprog\")\n precip_forecast = nowcast_method(\n precip_input,\n retrieved_motion,\n timesteps=3,\n precip_thr=0.1,\n )\n\n assert precip_forecast.ndim == 3\n assert precip_forecast.shape[0] == 3\n assert precip_forecast.sum() == 0.0\n\n\n@pytest.mark.parametrize(sprog_arg_names, sprog_arg_values)\ndef test_sprog(\n n_cascade_levels, ar_order, probmatching_method, domain, timesteps, min_csi\n):\n \"\"\"Tests SPROG nowcast.\"\"\"\n # inputs\n precip_input, metadata = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n return_raw=False,\n metadata=True,\n upscale=2000,\n )\n precip_input = precip_input.filled()\n\n precip_obs = get_precipitation_fields(\n num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000\n )[1:, :, :]\n precip_obs = precip_obs.filled()\n\n pytest.importorskip(\"cv2\")\n oflow_method = motion.get_method(\"LK\")\n retrieved_motion = oflow_method(precip_input)\n\n nowcast_method = nowcasts.get_method(\"sprog\")\n\n precip_forecast = nowcast_method(\n precip_input,\n retrieved_motion,\n timesteps=timesteps,\n precip_thr=metadata[\"threshold\"],\n n_cascade_levels=n_cascade_levels,\n ar_order=ar_order,\n probmatching_method=probmatching_method,\n domain=domain,\n )\n\n assert precip_forecast.ndim == 3\n assert precip_forecast.shape[0] == (\n timesteps if isinstance(timesteps, int) else len(timesteps)\n )\n\n result = verification.det_cat_fct(\n precip_forecast[-1], precip_obs[-1], thr=0.1, scores=\"CSI\"\n )[\"CSI\"]\n assert result > min_csi, f\"CSI={result:.1f}, required > {min_csi:.1f}\"\n\n\nif __name__ == \"__main__\":\n for n in range(len(sprog_arg_values)):\n test_args = zip(sprog_arg_names, sprog_arg_values[n])\n test_sprog(**dict((x, y) for x, y in test_args))\n" }, { "path": "pysteps/tests/test_nowcasts_sseps.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\n\nfrom pysteps import motion, nowcasts, verification\nfrom pysteps.tests.helpers import get_precipitation_fields\n\nsseps_arg_names = (\n \"n_ens_members\",\n \"n_cascade_levels\",\n \"ar_order\",\n \"mask_method\",\n \"probmatching_method\",\n \"win_size\",\n \"timesteps\",\n \"max_crps\",\n)\n\nsseps_arg_values = [\n (5, 6, 2, \"incremental\", \"cdf\", 200, 3, 0.60),\n (5, 6, 2, \"incremental\", \"cdf\", 200, [3], 0.60),\n]\n\n\ndef test_default_sseps_norain():\n \"\"\"Tests SSEPS nowcast with default params and all-zero inputs.\"\"\"\n\n # Define dummy nowcast input data\n precip_input = np.zeros((3, 100, 100))\n metadata = {\n \"accutime\": 5,\n \"xpixelsize\": 1000,\n \"threshold\": 0.1,\n \"zerovalue\": 0,\n }\n\n pytest.importorskip(\"cv2\")\n oflow_method = motion.get_method(\"LK\")\n retrieved_motion = oflow_method(precip_input)\n\n nowcast_method = nowcasts.get_method(\"sseps\")\n precip_forecast = nowcast_method(\n precip_input,\n metadata,\n retrieved_motion,\n n_ens_members=3,\n timesteps=3,\n )\n\n assert precip_forecast.ndim == 4\n assert precip_forecast.shape[0] == 3\n assert precip_forecast.shape[1] == 3\n assert precip_forecast.sum() == 0.0\n\n\n@pytest.mark.parametrize(sseps_arg_names, sseps_arg_values)\ndef test_sseps(\n n_ens_members,\n n_cascade_levels,\n ar_order,\n mask_method,\n probmatching_method,\n win_size,\n timesteps,\n max_crps,\n):\n \"\"\"Tests SSEPS nowcast.\"\"\"\n # inputs\n precip_input, metadata = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n return_raw=False,\n metadata=True,\n upscale=2000,\n )\n precip_input = precip_input.filled()\n\n precip_obs = get_precipitation_fields(\n num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000\n )[1:, :, :]\n precip_obs = precip_obs.filled()\n\n pytest.importorskip(\"cv2\")\n oflow_method = motion.get_method(\"LK\")\n retrieved_motion = oflow_method(precip_input)\n\n nowcast_method = nowcasts.get_method(\"sseps\")\n\n precip_forecast = nowcast_method(\n precip_input,\n metadata,\n retrieved_motion,\n win_size=win_size,\n timesteps=timesteps,\n n_ens_members=n_ens_members,\n n_cascade_levels=n_cascade_levels,\n ar_order=ar_order,\n seed=42,\n mask_method=mask_method,\n probmatching_method=probmatching_method,\n )\n\n assert precip_forecast.ndim == 4\n assert precip_forecast.shape[0] == n_ens_members\n assert precip_forecast.shape[1] == (\n timesteps if isinstance(timesteps, int) else len(timesteps)\n )\n\n crps = verification.probscores.CRPS(precip_forecast[:, -1], precip_obs[-1])\n assert crps < max_crps, f\"CRPS={crps:.2f}, required < {max_crps:.2f}\"\n\n\nif __name__ == \"__main__\":\n for n in range(len(sseps_arg_values)):\n test_args = zip(sseps_arg_names, sseps_arg_values[n])\n test_sseps(**dict((x, y) for x, y in test_args))\n" }, { "path": "pysteps/tests/test_nowcasts_steps.py", "content": "import os\nfrom datetime import timedelta\n\nimport numpy as np\nimport pytest\n\nfrom pysteps import io, motion, nowcasts, verification\nfrom pysteps.tests.helpers import get_precipitation_fields\n\nsteps_arg_names = (\n \"n_ens_members\",\n \"n_cascade_levels\",\n \"ar_order\",\n \"mask_method\",\n \"probmatching_method\",\n \"domain\",\n \"timesteps\",\n \"max_crps\",\n)\n\nsteps_arg_values = [\n (5, 6, 2, None, None, \"spatial\", 3, 1.30),\n (5, 6, 2, None, None, \"spatial\", [3], 1.30),\n (5, 6, 2, \"incremental\", None, \"spatial\", 3, 7.32),\n (5, 6, 2, \"sprog\", None, \"spatial\", 3, 8.4),\n (5, 6, 2, \"obs\", None, \"spatial\", 3, 8.37),\n (5, 6, 2, None, \"cdf\", \"spatial\", 3, 0.60),\n (5, 6, 2, None, \"mean\", \"spatial\", 3, 1.35),\n (5, 6, 2, \"incremental\", \"cdf\", \"spectral\", 3, 0.60),\n]\n\n\ndef test_default_steps_norain():\n \"\"\"Tests STEPS nowcast with default params and all-zero inputs.\"\"\"\n\n # Define dummy nowcast input data\n precip_input = np.zeros((3, 100, 100))\n\n pytest.importorskip(\"cv2\")\n oflow_method = motion.get_method(\"LK\")\n retrieved_motion = oflow_method(precip_input)\n\n nowcast_method = nowcasts.get_method(\"steps\")\n precip_forecast = nowcast_method(\n precip_input,\n retrieved_motion,\n n_ens_members=3,\n timesteps=3,\n precip_thr=0.1,\n kmperpixel=1,\n timestep=5,\n )\n\n assert precip_forecast.ndim == 4\n assert precip_forecast.shape[0] == 3\n assert precip_forecast.shape[1] == 3\n assert precip_forecast.sum() == 0.0\n\n\n@pytest.mark.parametrize(steps_arg_names, steps_arg_values)\ndef test_steps_skill(\n n_ens_members,\n n_cascade_levels,\n ar_order,\n mask_method,\n probmatching_method,\n domain,\n timesteps,\n max_crps,\n):\n \"\"\"Tests STEPS nowcast skill.\"\"\"\n # inputs\n precip_input, metadata = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n return_raw=False,\n metadata=True,\n upscale=2000,\n )\n precip_input = precip_input.filled()\n\n precip_obs = get_precipitation_fields(\n num_prev_files=0, num_next_files=3, return_raw=False, upscale=2000\n )[1:, :, :]\n precip_obs = precip_obs.filled()\n\n pytest.importorskip(\"cv2\")\n oflow_method = motion.get_method(\"LK\")\n retrieved_motion = oflow_method(precip_input)\n\n nowcast_method = nowcasts.get_method(\"steps\")\n\n precip_forecast = nowcast_method(\n precip_input,\n retrieved_motion,\n timesteps=timesteps,\n precip_thr=metadata[\"threshold\"],\n kmperpixel=2.0,\n timestep=metadata[\"accutime\"],\n seed=42,\n n_ens_members=n_ens_members,\n n_cascade_levels=n_cascade_levels,\n ar_order=ar_order,\n mask_method=mask_method,\n probmatching_method=probmatching_method,\n domain=domain,\n )\n\n assert precip_forecast.ndim == 4\n assert precip_forecast.shape[0] == n_ens_members\n assert precip_forecast.shape[1] == (\n timesteps if isinstance(timesteps, int) else len(timesteps)\n )\n\n crps = verification.probscores.CRPS(precip_forecast[:, -1], precip_obs[-1])\n assert crps < max_crps, f\"CRPS={crps:.2f}, required < {max_crps:.2f}\"\n\n\ndef test_steps_callback(tmp_path):\n \"\"\"Test STEPS callback functionality to export the output as a netcdf.\"\"\"\n\n pytest.importorskip(\"netCDF4\")\n\n n_ens_members = 2\n n_timesteps = 3\n\n precip_input, metadata = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n return_raw=False,\n metadata=True,\n upscale=2000,\n )\n precip_input = precip_input.filled()\n field_shape = (precip_input.shape[1], precip_input.shape[2])\n startdate = metadata[\"timestamps\"][-1]\n timestep = metadata[\"accutime\"]\n\n motion_field = np.zeros((2, *field_shape))\n\n exporter = io.initialize_forecast_exporter_netcdf(\n outpath=tmp_path.as_posix(),\n outfnprefix=\"test_steps\",\n startdate=startdate,\n timestep=timestep,\n n_timesteps=n_timesteps,\n shape=field_shape,\n n_ens_members=n_ens_members,\n metadata=metadata,\n incremental=\"timestep\",\n )\n\n def callback(array):\n return io.export_forecast_dataset(array, exporter)\n\n precip_output = nowcasts.get_method(\"steps\")(\n precip_input,\n motion_field,\n timesteps=n_timesteps,\n precip_thr=metadata[\"threshold\"],\n kmperpixel=2.0,\n timestep=timestep,\n seed=42,\n n_ens_members=n_ens_members,\n vel_pert_method=None,\n callback=callback,\n return_output=True,\n )\n io.close_forecast_files(exporter)\n\n # assert that netcdf exists and its size is not zero\n tmp_file = os.path.join(tmp_path, \"test_steps.nc\")\n assert os.path.exists(tmp_file) and os.path.getsize(tmp_file) > 0\n\n # assert that the file can be read by the nowcast importer\n precip_netcdf, metadata_netcdf = io.import_netcdf_pysteps(tmp_file, dtype=\"float64\")\n\n # assert that the dimensionality of the array is as expected\n assert precip_netcdf.ndim == 4, \"Wrong number of dimensions\"\n assert precip_netcdf.shape[0] == n_ens_members, \"Wrong ensemble size\"\n assert precip_netcdf.shape[1] == n_timesteps, \"Wrong number of lead times\"\n assert precip_netcdf.shape[2:] == field_shape, \"Wrong field shape\"\n\n # assert that the saved output is the same as the original output\n assert np.allclose(\n precip_netcdf, precip_output, equal_nan=True\n ), \"Wrong output values\"\n\n # assert that leadtimes and timestamps are as expected\n td = timedelta(minutes=timestep)\n leadtimes = [(i + 1) * timestep for i in range(n_timesteps)]\n timestamps = [startdate + (i + 1) * td for i in range(n_timesteps)]\n assert (metadata_netcdf[\"leadtimes\"] == leadtimes).all(), \"Wrong leadtimes\"\n assert (metadata_netcdf[\"timestamps\"] == timestamps).all(), \"Wrong timestamps\"\n" }, { "path": "pysteps/tests/test_nowcasts_utils.py", "content": "import numpy as np\nimport pytest\n\nfrom pysteps import motion\nfrom pysteps.nowcasts import utils as nowcast_utils\nfrom pysteps.tests.helpers import get_precipitation_fields\n\nmain_loop_arg_names = (\n \"timesteps\",\n \"ensemble\",\n \"num_ensemble_members\",\n \"velocity_perturbations\",\n)\n\n# TODO: add tests for callback and other untested options\nmain_loop_arg_values = [\n (6, False, 0, False),\n ([0.5, 1.5], False, 0, False),\n (6, True, 2, False),\n (6, True, 2, True),\n]\n\n\n@pytest.mark.parametrize(main_loop_arg_names, main_loop_arg_values)\ndef test_nowcast_main_loop(\n timesteps, ensemble, num_ensemble_members, velocity_perturbations\n):\n \"\"\"Test the nowcast_main_loop function.\"\"\"\n precip = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n return_raw=False,\n metadata=False,\n upscale=2000,\n )\n precip = precip.filled()\n\n oflow_method = motion.get_method(\"LK\")\n velocity = oflow_method(precip)\n\n precip = precip[-1]\n\n state = {\"input\": precip}\n extrap_method = \"semilagrangian\"\n\n def func(state, params):\n if not ensemble:\n precip_out = state[\"input\"]\n else:\n precip_out = state[\"input\"][np.newaxis, :]\n\n return precip_out, state\n\n nowcast_utils.nowcast_main_loop(\n precip,\n velocity,\n state,\n timesteps,\n extrap_method,\n func,\n ensemble=ensemble,\n num_ensemble_members=num_ensemble_members,\n )\n" }, { "path": "pysteps/tests/test_paramsrc.py", "content": "# -*- coding: utf-8 -*-\nimport os\n\nfrom tempfile import NamedTemporaryFile\n\nimport pysteps\nfrom pysteps import load_config_file\n\nminimal_pystepsrc_file = \"\"\"\n// pysteps configuration\n{\n \"silent_import\": false,\n \"outputs\": {\n \"path_outputs\": \"./\"\n },\n \"plot\": {\n \"motion_plot\": \"quiver\",\n \"colorscale\": \"pysteps\"\n },\n \"data_sources\": {\n \"bom\": {\n \"root_path\": \"./radar/bom\",\n \"path_fmt\": \"prcp-cscn/2/%Y/%m/%d\",\n \"fn_pattern\": \"2_%Y%m%d_%H%M00.prcp-cscn\",\n \"fn_ext\": \"nc\",\n \"importer\": \"bom_rf3\",\n \"timestep\": 6,\n \"importer_kwargs\": {\n \"gzipped\": true\n }\n } \n }\n}\n\"\"\"\n\n\ndef test_read_paramsrc():\n \"\"\"\n Test that the parameter file is read correctly and the resulting\n pysteps.paramsrc dict can be accessed by attributes too.\n \"\"\"\n\n with NamedTemporaryFile(mode=\"w\", delete=False) as tmp_paramsrc:\n tmp_paramsrc.write(minimal_pystepsrc_file)\n tmp_paramsrc.flush()\n\n # Perform a dry run that does not update\n # the internal pysteps.rcparams values.\n rcparams = load_config_file(tmp_paramsrc.name, dryrun=True, verbose=False)\n os.unlink(tmp_paramsrc.name)\n # Test item and attribute getters\n assert rcparams[\"data_sources\"][\"bom\"][\"fn_ext\"] == \"nc\"\n assert rcparams.data_sources.bom.fn_ext == \"nc\"\n\n bom_datasource_as_dict = rcparams[\"data_sources\"][\"bom\"]\n bom_datasource_as_attr = rcparams.data_sources.bom\n assert bom_datasource_as_dict is bom_datasource_as_attr\n bom_datasource = bom_datasource_as_attr\n\n timestep_as_dict = bom_datasource[\"timestep\"]\n timestep_as_attr = bom_datasource.timestep\n assert timestep_as_dict == 6\n assert timestep_as_attr == 6\n assert timestep_as_dict is timestep_as_attr\n\n importer_kwargs_dict = bom_datasource[\"importer_kwargs\"]\n importer_kwargs_attr = bom_datasource.importer_kwargs\n assert importer_kwargs_attr is importer_kwargs_dict\n\n assert importer_kwargs_attr[\"gzipped\"] is importer_kwargs_attr.gzipped\n assert importer_kwargs_attr[\"gzipped\"] is True\n\n # Test item and attribute setters\n rcparams.test = 4\n assert rcparams.test == 4\n assert rcparams.test is rcparams[\"test\"]\n\n rcparams[\"test2\"] = 4\n assert rcparams.test2 == 4\n assert rcparams.test2 is rcparams[\"test2\"]\n\n rcparams.test = dict(a=1, b=\"test\")\n assert rcparams.test == dict(a=1, b=\"test\")\n assert rcparams.test[\"a\"] == 1\n assert rcparams.test[\"b\"] == \"test\"\n\n assert rcparams.test[\"a\"] is rcparams[\"test\"].a\n assert rcparams.test[\"b\"] is rcparams[\"test\"].b\n" }, { "path": "pysteps/tests/test_plt_animate.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\n\nimport numpy as np\nimport pytest\nfrom unittest.mock import patch\n\nfrom pysteps.tests.helpers import get_precipitation_fields\nfrom pysteps.visualization.animations import animate\n\nPRECIP, METADATA = get_precipitation_fields(\n num_prev_files=2,\n num_next_files=0,\n return_raw=True,\n metadata=True,\n upscale=2000,\n)\n\nVALID_ARGS = (\n ([PRECIP], {}),\n ([PRECIP], {\"title\": \"title\"}),\n ([PRECIP], {\"timestamps_obs\": METADATA[\"timestamps\"]}),\n ([PRECIP], {\"geodata\": METADATA, \"map_kwargs\": {\"plot_map\": None}}),\n ([PRECIP], {\"motion_field\": np.ones((2, *PRECIP.shape[1:]))}),\n (\n [PRECIP],\n {\"precip_kwargs\": {\"units\": \"mm/h\", \"colorbar\": True, \"colorscale\": \"pysteps\"}},\n ),\n ([PRECIP, PRECIP], {}),\n ([PRECIP, PRECIP], {\"title\": \"title\"}),\n ([PRECIP, PRECIP], {\"timestamps_obs\": METADATA[\"timestamps\"]}),\n ([PRECIP, PRECIP], {\"timestamps_obs\": METADATA[\"timestamps\"], \"timestep_min\": 5}),\n ([PRECIP, PRECIP], {\"ptype\": \"prob\", \"prob_thr\": 1}),\n ([PRECIP, PRECIP], {\"ptype\": \"mean\"}),\n ([PRECIP, np.stack((PRECIP, PRECIP))], {\"ptype\": \"ensemble\"}),\n)\n\n\n@pytest.mark.parametrize([\"anim_args\", \"anim_kwargs\"], VALID_ARGS)\ndef test_animate(anim_args, anim_kwargs):\n with patch(\"matplotlib.pyplot.show\"):\n animate(*anim_args, **anim_kwargs)\n\n\nVALUEERROR_ARGS = (\n ([PRECIP], {\"timestamps_obs\": METADATA[\"timestamps\"][:2]}),\n ([PRECIP], {\"motion_plot\": \"test\"}),\n ([PRECIP, PRECIP], {\"ptype\": \"prob\"}),\n)\n\n\n@pytest.mark.parametrize([\"anim_args\", \"anim_kwargs\"], VALUEERROR_ARGS)\ndef test_animate_valueerrors(anim_args, anim_kwargs):\n with pytest.raises(ValueError):\n animate(*anim_args, **anim_kwargs)\n\n\nTYPEERROR_ARGS = (\n ([PRECIP], {\"timestamps\": METADATA[\"timestamps\"]}),\n ([PRECIP], {\"plotanimation\": True}),\n ([PRECIP], {\"units\": \"mm/h\"}),\n ([PRECIP], {\"colorbar\": True}),\n ([PRECIP], {\"colorscale\": \"pysteps\"}),\n ([PRECIP, PRECIP], {\"type\": \"ensemble\"}),\n)\n\n\n@pytest.mark.parametrize([\"anim_args\", \"anim_kwargs\"], TYPEERROR_ARGS)\ndef test_animate_typeerrors(anim_args, anim_kwargs):\n with pytest.raises(TypeError):\n animate(*anim_args, **anim_kwargs)\n\n\ndef test_animate_save(tmp_path):\n animate(\n PRECIP,\n np.stack((PRECIP, PRECIP)),\n display_animation=False,\n savefig=True,\n path_outputs=tmp_path,\n fig_dpi=10,\n )\n assert len(os.listdir(tmp_path)) == 9\n" }, { "path": "pysteps/tests/test_plt_cartopy.py", "content": "# -*- coding: utf-8 -*-\n\nimport pytest\n\nfrom pysteps.visualization import plot_precip_field\nfrom pysteps.utils import to_rainrate\nfrom pysteps.tests.helpers import get_precipitation_fields\nimport matplotlib.pyplot as plt\n\nplt_arg_names = (\"source\", \"map_kwargs\", \"pass_geodata\")\n\nplt_arg_values = [\n (\"mch\", {\"drawlonlatlines\": False, \"lw\": 0.5, \"plot_map\": None}, False),\n (\"mch\", {\"drawlonlatlines\": False, \"lw\": 0.5, \"plot_map\": \"cartopy\"}, False),\n (\"mch\", {\"drawlonlatlines\": False, \"lw\": 0.5}, True),\n (\"mch\", {\"drawlonlatlines\": True, \"lw\": 1.0}, True),\n (\"bom\", {\"drawlonlatlines\": True, \"lw\": 0.5}, True),\n (\"fmi\", {\"drawlonlatlines\": True, \"lw\": 0.5}, True),\n (\"knmi\", {\"drawlonlatlines\": True, \"lw\": 0.5}, True),\n (\"opera\", {\"drawlonlatlines\": True, \"lw\": 0.5}, True),\n (\"mrms\", {\"drawlonlatlines\": True, \"lw\": 0.5}, True),\n (\"saf\", {\"drawlonlatlines\": True, \"lw\": 0.5}, True),\n]\n\n\n@pytest.mark.parametrize(plt_arg_names, plt_arg_values)\ndef test_visualization_plot_precip_field(source, map_kwargs, pass_geodata):\n field, metadata = get_precipitation_fields(0, 0, True, True, None, source)\n field = field.squeeze()\n field, __ = to_rainrate(field, metadata)\n\n if not pass_geodata:\n metadata = None\n\n plot_precip_field(field, ptype=\"intensity\", geodata=metadata, map_kwargs=map_kwargs)\n\n\nif __name__ == \"__main__\":\n for i, args in enumerate(plt_arg_values):\n test_visualization_plot_precip_field(*args)\n plt.show()\n" }, { "path": "pysteps/tests/test_plt_motionfields.py", "content": "# -*- coding: utf-8 -*-\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pytest\n\nfrom pysteps import motion\nfrom pysteps.visualization import plot_precip_field, quiver, streamplot\nfrom pysteps.tests.helpers import get_precipitation_fields\n\narg_names_quiver = (\n \"source\",\n \"axis\",\n \"step\",\n \"quiver_kwargs\",\n \"map_kwargs\",\n \"upscale\",\n \"pass_geodata\",\n)\n\narg_values_quiver = [\n (None, \"off\", 10, {}, {\"drawlonlatlines\": False, \"lw\": 0.5}, None, False),\n (\"bom\", \"on\", 10, {}, {\"drawlonlatlines\": False, \"lw\": 0.5}, 4000, False),\n (\"bom\", \"on\", 10, {}, {\"drawlonlatlines\": True, \"lw\": 0.5}, 4000, True),\n (\"mch\", \"on\", 20, {}, {\"drawlonlatlines\": False, \"lw\": 0.5}, 2000, True),\n]\n\n\n@pytest.mark.parametrize(arg_names_quiver, arg_values_quiver)\ndef test_visualization_motionfields_quiver(\n source, axis, step, quiver_kwargs, map_kwargs, upscale, pass_geodata\n):\n pytest.importorskip(\"cv2\")\n if source is not None:\n fields, geodata = get_precipitation_fields(0, 2, False, True, upscale, source)\n if not pass_geodata:\n geodata = None\n ax = plot_precip_field(fields[-1], geodata=geodata)\n oflow_method = motion.get_method(\"LK\")\n UV = oflow_method(fields)\n\n else:\n shape = (100, 100)\n geodata = None\n ax = None\n u = np.ones(shape[1]) * shape[0]\n v = np.arange(0, shape[0])\n U, V = np.meshgrid(u, v)\n UV = np.concatenate([U[None, :], V[None, :]])\n\n UV_orig = UV.copy()\n __ = quiver(UV, ax, geodata, axis, step, quiver_kwargs, map_kwargs=map_kwargs)\n\n # Check that quiver does not modify the input data\n assert np.array_equal(UV, UV_orig)\n\n\narg_names_streamplot = (\n \"source\",\n \"axis\",\n \"streamplot_kwargs\",\n \"map_kwargs\",\n \"upscale\",\n \"pass_geodata\",\n)\n\narg_values_streamplot = [\n (None, \"off\", {}, {\"drawlonlatlines\": False, \"lw\": 0.5}, None, False),\n (\"bom\", \"on\", {}, {\"drawlonlatlines\": False, \"lw\": 0.5}, 4000, False),\n (\"bom\", \"on\", {\"density\": 0.5}, {\"drawlonlatlines\": True, \"lw\": 0.5}, 4000, True),\n]\n\n\n@pytest.mark.parametrize(arg_names_streamplot, arg_values_streamplot)\ndef test_visualization_motionfields_streamplot(\n source, axis, streamplot_kwargs, map_kwargs, upscale, pass_geodata\n):\n pytest.importorskip(\"cv2\")\n if source is not None:\n fields, geodata = get_precipitation_fields(0, 2, False, True, upscale, source)\n if not pass_geodata:\n pass_geodata = None\n ax = plot_precip_field(fields[-1], geodata=geodata)\n oflow_method = motion.get_method(\"LK\")\n UV = oflow_method(fields)\n\n else:\n shape = (100, 100)\n geodata = None\n ax = None\n u = np.ones(shape[1]) * shape[0]\n v = np.arange(0, shape[0])\n U, V = np.meshgrid(u, v)\n UV = np.concatenate([U[None, :], V[None, :]])\n\n UV_orig = UV.copy()\n __ = streamplot(UV, ax, geodata, axis, streamplot_kwargs, map_kwargs=map_kwargs)\n\n # Check that streamplot does not modify the input data\n assert np.array_equal(UV, UV_orig)\n\n\nif __name__ == \"__main__\":\n for i, args in enumerate(arg_values_quiver):\n test_visualization_motionfields_quiver(*args)\n plt.show()\n\n for i, args in enumerate(arg_values_streamplot):\n test_visualization_motionfields_streamplot(*args)\n plt.show()\n" }, { "path": "pysteps/tests/test_plt_precipfields.py", "content": "# -*- coding: utf-8 -*-\n\nimport pytest\n\nfrom pysteps.visualization import plot_precip_field\nfrom pysteps.utils import conversion\nfrom pysteps.postprocessing import ensemblestats\nfrom pysteps.tests.helpers import get_precipitation_fields\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nplt_arg_names = (\n \"source\",\n \"plot_type\",\n \"bbox\",\n \"colorscale\",\n \"probthr\",\n \"title\",\n \"colorbar\",\n \"axis\",\n)\n\nplt_arg_values = [\n (\"mch\", \"intensity\", None, \"pysteps\", None, None, False, \"off\"),\n (\"mch\", \"depth\", None, \"pysteps\", None, \"Title\", True, \"on\"),\n (\"mch\", \"prob\", None, \"pysteps\", 0.1, None, True, \"on\"),\n (\"mch\", \"intensity\", None, \"STEPS-BE\", None, None, True, \"on\"),\n (\"mch\", \"intensity\", None, \"BOM-RF3\", None, None, True, \"on\"),\n (\"bom\", \"intensity\", None, \"pysteps\", None, None, True, \"on\"),\n (\"fmi\", \"intensity\", None, \"pysteps\", None, None, True, \"on\"),\n (\"knmi\", \"intensity\", None, \"pysteps\", None, None, True, \"on\"),\n (\"knmi\", \"intensity\", None, \"STEPS-NL\", None, None, True, \"on\"),\n (\"knmi\", \"intensity\", [300, 300, 500, 500], \"pysteps\", None, None, True, \"on\"),\n (\"opera\", \"intensity\", None, \"pysteps\", None, None, True, \"on\"),\n (\"saf\", \"intensity\", None, \"pysteps\", None, None, True, \"on\"),\n]\n\n\n@pytest.mark.parametrize(plt_arg_names, plt_arg_values)\ndef test_visualization_plot_precip_field(\n source, plot_type, bbox, colorscale, probthr, title, colorbar, axis\n):\n if plot_type == \"intensity\":\n field, metadata = get_precipitation_fields(0, 0, True, True, None, source)\n field = field.squeeze()\n field, metadata = conversion.to_rainrate(field, metadata)\n\n elif plot_type == \"depth\":\n field, metadata = get_precipitation_fields(0, 0, True, True, None, source)\n field = field.squeeze()\n field, metadata = conversion.to_raindepth(field, metadata)\n\n elif plot_type == \"prob\":\n field, metadata = get_precipitation_fields(0, 10, True, True, None, source)\n field, metadata = conversion.to_rainrate(field, metadata)\n field = ensemblestats.excprob(field, probthr)\n\n field_orig = field.copy()\n ax = plot_precip_field(\n field.copy(),\n ptype=plot_type,\n bbox=bbox,\n geodata=None,\n colorscale=colorscale,\n probthr=probthr,\n units=metadata[\"unit\"],\n title=title,\n colorbar=colorbar,\n axis=axis,\n )\n\n # Check that plot_precip_field does not modify the input data\n field_orig = np.ma.masked_invalid(field_orig)\n field_orig.data[field_orig.mask] = -100\n field = np.ma.masked_invalid(field)\n field.data[field.mask] = -100\n assert np.array_equal(field_orig.data, field.data)\n\n\nif __name__ == \"__main__\":\n for i, args in enumerate(plt_arg_values):\n test_visualization_plot_precip_field(*args)\n plt.show()\n" }, { "path": "pysteps/tests/test_plugins_support.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nScript to test the plugin support.\n\nhttps://github.com/pySTEPS/cookiecutter-pysteps-plugin\n\"\"\"\n\nimport os\nimport pytest\nimport subprocess\nimport sys\nimport tempfile\n\n__ = pytest.importorskip(\"cookiecutter\")\nfrom cookiecutter.main import cookiecutter\n\nPLUGIN_TEMPLATE_URL = \"https://github.com/pysteps/cookiecutter-pysteps-plugin\"\n\nfrom contextlib import contextmanager\nfrom pysteps import io, postprocessing\n\n\ndef _check_installed_importer_plugin(import_func_name):\n # reload the pysteps module to detect the installed plugin\n io.discover_importers()\n print(io.importers_info())\n import_func_name = import_func_name.replace(\"importer_\", \"import_\")\n assert hasattr(io.importers, import_func_name)\n func_name = import_func_name.replace(\"import_\", \"\")\n assert func_name in io.interface._importer_methods\n importer = getattr(io.importers, import_func_name)\n importer(\"filename\")\n\n\ndef _check_installed_diagnostic_plugin(diagnostic_func_name):\n # reload the pysteps module to detect the installed plugin\n postprocessing.discover_postprocessors()\n assert hasattr(postprocessing.diagnostics, diagnostic_func_name)\n assert diagnostic_func_name in postprocessing.interface._diagnostics_methods\n diagnostic = getattr(postprocessing.diagnostics, diagnostic_func_name)\n diagnostic(\"filename\")\n\n\n@contextmanager\ndef _create_and_install_plugin(project_name, plugin_type):\n with tempfile.TemporaryDirectory() as tmpdirname:\n print(f\"Installing plugin {project_name} providing a {plugin_type} module\")\n cookiecutter(\n PLUGIN_TEMPLATE_URL,\n no_input=True,\n overwrite_if_exists=True,\n extra_context={\n \"project_name\": project_name,\n \"plugin_type\": plugin_type,\n },\n output_dir=tmpdirname,\n )\n # Install the plugin\n subprocess.check_call(\n [\n sys.executable,\n \"-m\",\n \"pip\",\n \"install\",\n \"--force-reinstall\",\n os.path.join(tmpdirname, project_name),\n ]\n )\n\n # The block below, together with the decorator used in this function are used\n # to create a context manager that uninstall the plugin packages after the\n # tests finish (even if they fail).\n # https://docs.pytest.org/en/stable/fixture.html?highlight=context#fixture-finalization-executing-teardown-code\n try:\n yield project_name\n finally:\n _uninstall_plugin(project_name)\n\n\ndef _uninstall_plugin(project_name):\n # Install the plugin\n subprocess.check_call(\n [sys.executable, \"-m\", \"pip\", \"uninstall\", \"-y\", project_name]\n )\n\n\ndef test_importers_plugins():\n with _create_and_install_plugin(\"pysteps-importer-institution-fun\", \"importer\"):\n _check_installed_importer_plugin(\"importer_institution_fun\")\n\n\ndef test_diagnostic_plugins():\n with _create_and_install_plugin(\"pysteps-diagnostic-fun\", \"diagnostic\"):\n _check_installed_diagnostic_plugin(\"diagnostic_fun\")\n" }, { "path": "pysteps/tests/test_postprocessing_ensemblestats.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.postprocessing.ensemblestats import excprob, mean, banddepth\n\n# CREATE DATASETS TO TEST\n\na = np.arange(9, dtype=float).reshape(3, 3)\nb = np.tile(a, (4, 1, 1))\nb1 = b.copy()\nb1[3] = np.nan\na1 = a.copy()\na1[:] = np.nan\na2 = a.copy()\na2[0, :] = np.nan\n\n# test data\ntest_data = [\n (a, False, None, a),\n (b, False, None, a),\n (b1, True, None, a),\n (b1, False, None, a1),\n (b, False, 0.0, a),\n (b, False, 3.0, a2),\n (b, True, 3.0, a2),\n (b1, True, 3.0, a2),\n]\n\n\n@pytest.mark.parametrize(\"X, ignore_nan, X_thr, expected\", test_data)\ndef test_ensemblestats_mean(X, ignore_nan, X_thr, expected):\n \"\"\"\n Test ensemblestats mean.\"\"\"\n assert_array_almost_equal(mean(X, ignore_nan, X_thr), expected)\n\n\n# test exceptions\ntest_exceptions = [(0), (None), (a[0, :]), (np.tile(a, (4, 1, 1, 1)))]\n\n\n@pytest.mark.parametrize(\"X\", test_exceptions)\ndef test_exceptions_mean(X):\n with pytest.raises(Exception):\n mean(X)\n\n\n# test data\nb2 = b.copy()\nb2[2, 2, 2] = np.nan\n\ntest_data = [\n (b, 2.0, False, np.array([[0.0, 0.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]])),\n (b2, 2.0, False, np.array([[0.0, 0.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, np.nan]])),\n (b2, 2.0, True, np.array([[0.0, 0.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]])),\n]\n\n\n@pytest.mark.parametrize(\"X, X_thr, ignore_nan, expected\", test_data)\ndef test_ensemblestats_excprob(X, X_thr, ignore_nan, expected):\n \"\"\"Test ensemblestats excprob.\"\"\"\n assert_array_almost_equal(excprob(X, X_thr, ignore_nan), expected)\n\n\n# test exceptions\ntest_exceptions = [(0), (None), (a[0, :]), (a)]\n\n\n@pytest.mark.parametrize(\"X\", test_exceptions)\ndef test_exceptions_excprob(X):\n with pytest.raises(Exception):\n excprob(X, 2.0)\n\n\n# test data\nb3 = np.tile(a, (5, 1, 1)) + 1\nb3 *= np.arange(1, 6)[:, None, None]\nb3[2, 2, 2] = np.nan\n\ntest_data = [\n (b3, 1, True, np.array([0.0, 0.75, 1.0, 0.75, 0.0])),\n (b3, None, False, np.array([0.4, 0.7, 0.8, 0.7, 0.4])),\n]\n\n\n@pytest.mark.parametrize(\"X, thr, norm, expected\", test_data)\ndef test_ensemblestats_banddepth(X, thr, norm, expected):\n \"\"\"Test ensemblestats banddepth.\"\"\"\n assert_array_almost_equal(banddepth(X, thr, norm), expected)\n" }, { "path": "pysteps/tests/test_postprocessing_probmatching.py", "content": "import numpy as np\nimport pytest\n\nfrom pysteps.postprocessing.probmatching import (\n nonparam_match_empirical_cdf,\n resample_distributions,\n)\n\n\nclass TestResampleDistributions:\n\n @pytest.fixture(autouse=True)\n def setup(self):\n # Set the seed for reproducibility\n np.random.seed(42)\n\n def test_valid_inputs(self):\n first_array = np.array([1, 3, 5, 7, 9])\n second_array = np.array([2, 4, 6, 8, 10])\n probability_first_array = 0.6\n result = resample_distributions(\n first_array, second_array, probability_first_array\n )\n expected_result = np.array([9, 8, 6, 3, 1]) # Expected result based on the seed\n assert result.shape == first_array.shape\n assert np.array_equal(result, expected_result)\n\n def test_probability_zero(self):\n first_array = np.array([1, 3, 5, 7, 9])\n second_array = np.array([2, 4, 6, 8, 10])\n probability_first_array = 0.0\n result = resample_distributions(\n first_array, second_array, probability_first_array\n )\n assert np.array_equal(result, np.sort(second_array)[::-1])\n\n def test_probability_one(self):\n first_array = np.array([1, 3, 5, 7, 9])\n second_array = np.array([2, 4, 6, 8, 10])\n probability_first_array = 1.0\n result = resample_distributions(\n first_array, second_array, probability_first_array\n )\n assert np.array_equal(result, np.sort(first_array)[::-1])\n\n def test_nan_in_arr1_prob_1(self):\n array_with_nan = np.array([1, 3, np.nan, 7, 9])\n array_without_nan = np.array([2.0, 4, 6, 8, 10])\n probability_first_array = 1.0\n result = resample_distributions(\n array_with_nan, array_without_nan, probability_first_array\n )\n expected_result = np.array([np.nan, 9, 7, 3, 1], dtype=float)\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_nan_in_arr1_prob_0(self):\n array_with_nan = np.array([1, 3, np.nan, 7, 9])\n array_without_nan = np.array([2, 4, 6, 8, 10])\n probability_first_array = 0.0\n result = resample_distributions(\n array_with_nan, array_without_nan, probability_first_array\n )\n expected_result = np.array([np.nan, 10, 8, 4, 2], dtype=float)\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_nan_in_arr2_prob_1(self):\n array_without_nan = np.array([1, 3, 5, 7, 9])\n array_with_nan = np.array([2.0, 4, 6, np.nan, 10])\n probability_first_array = 1.0\n result = resample_distributions(\n array_without_nan, array_with_nan, probability_first_array\n )\n expected_result = np.array([np.nan, 9, 5, 3, 1], dtype=float)\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_nan_in_arr2_prob_0(self):\n array_without_nan = np.array([1, 3, 5, 7, 9])\n array_with_nan = np.array([2, 4, 6, np.nan, 10])\n probability_first_array = 0.0\n result = resample_distributions(\n array_without_nan, array_with_nan, probability_first_array\n )\n expected_result = np.array([np.nan, 10, 6, 4, 2], dtype=float)\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_nan_in_both_prob_1(self):\n array1_with_nan = np.array([1, np.nan, np.nan, 7, 9])\n array2_with_nan = np.array([2.0, 4, np.nan, np.nan, 10])\n probability_first_array = 1.0\n result = resample_distributions(\n array1_with_nan, array2_with_nan, probability_first_array\n )\n expected_result = np.array([np.nan, np.nan, np.nan, 9, 1], dtype=float)\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_nan_in_both_prob_0(self):\n array1_with_nan = np.array([1, np.nan, np.nan, 7, 9])\n array2_with_nan = np.array([2.0, 4, np.nan, np.nan, 10])\n probability_first_array = 0.0\n result = resample_distributions(\n array1_with_nan, array2_with_nan, probability_first_array\n )\n expected_result = np.array([np.nan, np.nan, np.nan, 10, 2], dtype=float)\n assert np.allclose(result, expected_result, equal_nan=True)\n\n\nclass TestNonparamMatchEmpiricalCDF:\n @pytest.fixture(autouse=True)\n def setup(self):\n # Set the seed for reproducibility\n np.random.seed(42)\n\n def test_ignore_indices_with_nans_both(self):\n initial_array = np.array([np.nan, np.nan, 6, 2, 0, 0, 0, 0, 0, 0])\n target_array = np.array([np.nan, np.nan, 9, 5, 4, 0, 0, 0, 0, 0])\n result = nonparam_match_empirical_cdf(\n initial_array, target_array, ignore_indices=np.isnan(initial_array)\n )\n expected_result = np.array([np.nan, np.nan, 9, 5, 0, 0, 0, 0, 0, 0])\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_zeroes_initial(self):\n initial_array = np.zeros(10)\n target_array = np.array([0, 2, 3, 4, 5, 6, 7, 8, 9, 10])\n result = nonparam_match_empirical_cdf(initial_array, target_array)\n expected_result = np.zeros(10)\n assert np.allclose(result, expected_result)\n\n def test_nans_initial(self):\n initial_array = np.array(\n [0, 1, 2, 3, 4, np.nan, np.nan, np.nan, np.nan, np.nan]\n )\n target_array = np.array([0, 2, 3, 4, 5, 6, 7, 8, 9, 10])\n with pytest.raises(\n ValueError,\n match=\"Initial array contains non-finite values outside ignore_indices mask.\",\n ):\n nonparam_match_empirical_cdf(initial_array, target_array)\n\n def test_all_nans_initial(self):\n initial_array = np.full(10, np.nan)\n target_array = np.array([0, 2, 3, 4, 5, 6, 7, 8, 9, 10])\n with pytest.raises(ValueError, match=\"Initial array contains only nans.\"):\n nonparam_match_empirical_cdf(initial_array, target_array)\n\n def test_ignore_indices_nans_initial(self):\n initial_array = np.array(\n [0, 1, 2, 3, 4, np.nan, np.nan, np.nan, np.nan, np.nan]\n )\n target_array = np.array([0, 2, 3, 4, 5, 6, 7, 8, 9, 10])\n result = nonparam_match_empirical_cdf(\n initial_array, target_array, ignore_indices=np.isnan(initial_array)\n )\n expected_result = np.array(\n [0, 7, 8, 9, 10, np.nan, np.nan, np.nan, np.nan, np.nan]\n )\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_ignore_indices_nans_target(self):\n # We expect the initial_array values for which ignore_indices is true to be conserved as-is.\n initial_array = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n target_array = np.array(\n [0, 2, 3, 4, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan]\n )\n result = nonparam_match_empirical_cdf(\n initial_array, target_array, ignore_indices=np.isnan(target_array)\n )\n expected_result = np.array([0, 2, 3, 4, 4, 5, 6, 7, 8, 9])\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_more_zeroes_in_initial(self):\n initial_array = np.array([1, 4, 0, 0, 0, 0, 0, 0, 0, 0])\n target_array = np.array([10, 8, 6, 4, 2, 0, 0, 0, 0, 0])\n result = nonparam_match_empirical_cdf(\n initial_array, target_array, ignore_indices=np.isnan(initial_array)\n )\n expected_result = np.array([8, 10, 0, 0, 0, 0, 0, 0, 0, 0])\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_more_zeroes_in_initial_unsrt(self):\n initial_array = np.array([1, 4, 0, 0, 0, 0, 0, 0, 0, 0])\n target_array = np.array([6, 4, 2, 0, 0, 0, 0, 0, 10, 8])\n result = nonparam_match_empirical_cdf(\n initial_array, target_array, ignore_indices=np.isnan(initial_array)\n )\n expected_result = np.array([8, 10, 0, 0, 0, 0, 0, 0, 0, 0])\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_more_zeroes_in_target(self):\n initial_array = np.array([1, 3, 7, 5, 0, 0, 0, 0, 0, 0])\n target_array = np.array([10, 8, 0, 0, 0, 0, 0, 0, 0, 0])\n result = nonparam_match_empirical_cdf(\n initial_array, target_array, ignore_indices=np.isnan(initial_array)\n )\n expected_result = np.array([0, 0, 10, 8, 0, 0, 0, 0, 0, 0])\n assert np.allclose(result, expected_result, equal_nan=True)\n\n def test_2dim_array(self):\n initial_array = np.array([[1, 3, 5], [11, 9, 7]])\n target_array = np.array([[2, 4, 6], [8, 10, 12]])\n result = nonparam_match_empirical_cdf(initial_array, target_array)\n expected_result = np.array([[2, 4, 6], [12, 10, 8]])\n assert np.allclose(result, expected_result, equal_nan=True)\n" }, { "path": "pysteps/tests/test_timeseries_autoregression.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\nimport numpy as np\n\nimport pytest\n\nimport pysteps\nfrom pysteps.timeseries import autoregression, correlation\n\npytest.importorskip(\"pyproj\")\n\n\ndef test_estimate_ar_params_ols():\n R = _create_data_univariate()\n\n for p in range(1, 4):\n phi = autoregression.estimate_ar_params_ols(R[-(p + 1) :], p)\n assert len(phi) == p + 1\n for i in range(len(phi)):\n assert np.isscalar(phi[i])\n\n phi = autoregression.estimate_ar_params_ols(\n R[-(p + 1) :], p, include_constant_term=True\n )\n assert len(phi) == p + 2\n for i in range(len(phi)):\n assert np.isscalar(phi[i])\n\n phi = autoregression.estimate_ar_params_ols(\n R[-(p + 2) :], p, include_constant_term=True, d=1\n )\n assert len(phi) == p + 3\n for i in range(len(phi)):\n assert np.isscalar(phi[i])\n\n\ndef test_estimate_ar_params_yw():\n R = _create_data_univariate()\n\n for p in range(1, 4):\n gamma = correlation.temporal_autocorrelation(R[-(p + 1) :])\n phi = autoregression.estimate_ar_params_yw(gamma)\n assert len(phi) == p + 1\n for i in range(len(phi)):\n assert np.isscalar(phi[i])\n\n\ndef test_estimate_ar_params_yw_localized():\n R = _create_data_univariate()\n\n for p in range(1, 4):\n gamma = correlation.temporal_autocorrelation(\n R[-(p + 1) :], window=\"gaussian\", window_radius=25\n )\n phi = autoregression.estimate_ar_params_yw_localized(gamma)\n assert len(phi) == p + 1\n for i in range(len(phi)):\n assert phi[i].shape == R.shape[1:]\n\n\ndef test_estimate_ar_params_ols_localized():\n R = _create_data_univariate()\n\n for p in range(1, 4):\n phi = autoregression.estimate_ar_params_ols_localized(R[-(p + 1) :], p, 25)\n assert len(phi) == p + 1\n for i in range(len(phi)):\n assert phi[i].shape == R.shape[1:]\n\n phi = autoregression.estimate_ar_params_ols_localized(\n R[-(p + 1) :], p, 25, include_constant_term=True\n )\n assert len(phi) == p + 2\n for i in range(len(phi)):\n assert phi[i].shape == R.shape[1:]\n\n phi = autoregression.estimate_ar_params_ols_localized(\n R[-(p + 2) :], p, 25, include_constant_term=True, d=1\n )\n assert len(phi) == p + 3\n for i in range(len(phi)):\n assert phi[i].shape == R.shape[1:]\n\n\ndef test_estimate_var_params_ols():\n R = _create_data_multivariate()\n q = R.shape[1]\n\n for p in range(1, 4):\n phi = autoregression.estimate_var_params_ols(R[-(p + 1) :], p)\n assert len(phi) == p + 1\n for i in range(len(phi)):\n assert phi[i].shape == (q, q)\n\n phi = autoregression.estimate_var_params_ols(\n R[-(p + 1) :], p, include_constant_term=True\n )\n assert len(phi) == p + 2\n assert phi[0].shape == (q,)\n for i in range(1, len(phi)):\n assert phi[i].shape == (q, q)\n\n phi = autoregression.estimate_var_params_ols(\n R[-(p + 2) :], p, include_constant_term=True, d=1\n )\n assert len(phi) == p + 3\n assert phi[0].shape == (q,)\n for i in range(1, len(phi)):\n assert phi[i].shape == (q, q)\n\n\ndef test_estimate_var_params_ols_localized():\n R = _create_data_multivariate()\n q = R.shape[1]\n\n for p in range(1, 4):\n phi = autoregression.estimate_var_params_ols_localized(R[-(p + 1) :], p, 25)\n assert len(phi) == p + 1\n for i in range(len(phi)):\n assert phi[i].shape == (R.shape[2], R.shape[3], q, q)\n\n phi = autoregression.estimate_var_params_ols_localized(\n R[-(p + 1) :], p, 25, include_constant_term=True\n )\n assert len(phi) == p + 2\n assert phi[0].shape == (R.shape[2], R.shape[3], q)\n for i in range(1, len(phi)):\n assert phi[i].shape == (R.shape[2], R.shape[3], q, q)\n\n phi = autoregression.estimate_var_params_ols_localized(\n R[-(p + 2) :], p, 25, include_constant_term=True, d=1\n )\n assert len(phi) == p + 3\n assert phi[0].shape == (R.shape[2], R.shape[3], q)\n for i in range(1, len(phi)):\n assert phi[i].shape == (R.shape[2], R.shape[3], q, q)\n\n\ndef test_estimate_var_params_yw():\n R = _create_data_multivariate()\n\n for p in range(1, 4):\n gamma = correlation.temporal_autocorrelation_multivariate(R[-(p + 1) :])\n phi = autoregression.estimate_var_params_yw(gamma)\n assert len(phi) == p + 1\n for i in range(len(phi)):\n assert phi[i].shape == (R.shape[1], R.shape[1])\n\n\ndef test_estimate_var_params_yw_localized():\n R = _create_data_multivariate()\n q = R.shape[1]\n\n for p in range(1, 4):\n gamma = correlation.temporal_autocorrelation_multivariate(\n R[-(p + 1) :], window=\"gaussian\", window_radius=25\n )\n phi = autoregression.estimate_var_params_yw_localized(gamma)\n assert len(phi) == p + 1\n for i in range(len(phi)):\n assert phi[i].shape == (R.shape[2], R.shape[3], q, q)\n\n\ndef test_iterate_ar():\n R = _create_data_univariate()\n p = 2\n\n phi = autoregression.estimate_ar_params_ols(R[-(p + 1) :], p)\n autoregression.iterate_ar_model(R, phi)\n\n\ndef test_iterate_ar_localized():\n R = _create_data_univariate()\n p = 2\n\n phi = autoregression.estimate_ar_params_ols_localized(R[-(p + 1) :], p, 25)\n R_ = autoregression.iterate_ar_model(R, phi)\n assert R_.shape == R.shape\n\n\ndef test_iterate_var():\n R = _create_data_multivariate()\n p = 2\n\n phi = autoregression.estimate_var_params_ols(R[-(p + 1) :], p)\n R_ = autoregression.iterate_var_model(R, phi)\n assert R_.shape == R.shape\n\n\ndef test_iterate_var_localized():\n R = _create_data_multivariate()\n p = 2\n\n phi = autoregression.estimate_var_params_ols_localized(R[-(p + 1) :], p, 25)\n R_ = autoregression.iterate_var_model(R, phi)\n assert R_.shape == R.shape\n\n\ndef _create_data_multivariate():\n root_path = pysteps.rcparams.data_sources[\"fmi\"][\"root_path\"]\n\n filenames = [\n \"201609281600_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n \"201609281605_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n \"201609281610_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n \"201609281615_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n \"201609281620_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n ]\n\n R = []\n for fn in filenames:\n filename = os.path.join(root_path, \"20160928\", fn)\n R_, _, _ = pysteps.io.import_fmi_pgm(filename, gzipped=True)\n R_[~np.isfinite(R_)] = 0.0\n R.append(np.stack([R_, np.roll(R_, 5, axis=0)]))\n\n R = np.stack(R)\n R = R[:, :, 575:800, 255:480]\n\n return R\n\n\ndef _create_data_univariate():\n root_path = pysteps.rcparams.data_sources[\"fmi\"][\"root_path\"]\n\n filenames = [\n \"201609281600_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n \"201609281605_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n \"201609281610_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n \"201609281615_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n \"201609281620_fmi.radar.composite.lowest_FIN_SUOMI1.pgm.gz\",\n ]\n\n R = []\n for fn in filenames:\n filename = os.path.join(root_path, \"20160928\", fn)\n R_, _, _ = pysteps.io.import_fmi_pgm(filename, gzipped=True)\n R_[~np.isfinite(R_)] = 0.0\n R.append(R_)\n\n R = np.stack(R)\n R = R[:, 575:800, 255:480]\n\n return R\n" }, { "path": "pysteps/tests/test_tracking_tdating.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\n\nfrom pysteps.tracking.tdating import dating\nfrom pysteps.utils import to_reflectivity\nfrom pysteps.tests.helpers import get_precipitation_fields\n\narg_names = (\"source\", \"dry_input\", \"output_splits_merges\")\n\narg_values = [\n (\"mch\", False, False),\n (\"mch\", False, False),\n (\"mch\", True, False),\n (\"mch\", False, True),\n]\n\narg_names_multistep = (\"source\", \"len_timesteps\", \"output_splits_merges\")\narg_values_multistep = [\n (\"mch\", 6, False),\n (\"mch\", 6, True),\n]\n\n\n@pytest.mark.parametrize(arg_names_multistep, arg_values_multistep)\ndef test_tracking_tdating_dating_multistep(source, len_timesteps, output_splits_merges):\n pytest.importorskip(\"skimage\")\n\n input_fields, metadata = get_precipitation_fields(\n 0, len_timesteps, True, True, 4000, source\n )\n input_fields, __ = to_reflectivity(input_fields, metadata)\n\n timelist = metadata[\"timestamps\"]\n\n # First half of timesteps\n tracks_1, cells, labels = dating(\n input_fields[0 : len_timesteps // 2],\n timelist[0 : len_timesteps // 2],\n mintrack=1,\n output_splits_merges=output_splits_merges,\n )\n # Second half of timesteps\n tracks_2, cells, _ = dating(\n input_fields[len_timesteps // 2 - 2 :],\n timelist[len_timesteps // 2 - 2 :],\n mintrack=1,\n start=2,\n cell_list=cells,\n label_list=labels,\n output_splits_merges=output_splits_merges,\n )\n\n # Since we are adding cells, number of tracks should increase\n assert len(tracks_1) <= len(tracks_2)\n\n # Tracks should be continuous in time so time difference should not exceed timestep\n max_track_step = max([t.time.diff().max().seconds for t in tracks_2 if len(t) > 1])\n timestep = np.diff(timelist).max().seconds\n assert max_track_step <= timestep\n\n # IDs of unmatched cells should increase in every timestep\n for prev_df, cur_df in zip(cells[:-1], cells[1:]):\n prev_ids = set(prev_df.ID)\n cur_ids = set(cur_df.ID)\n new_ids = list(cur_ids - prev_ids)\n prev_unmatched = list(prev_ids - cur_ids)\n if len(prev_unmatched):\n assert np.all(np.array(new_ids) > max(prev_unmatched))\n\n\n@pytest.mark.parametrize(arg_names, arg_values)\ndef test_tracking_tdating_dating(source, dry_input, output_splits_merges):\n pytest.importorskip(\"skimage\")\n pandas = pytest.importorskip(\"pandas\")\n\n if not dry_input:\n input, metadata = get_precipitation_fields(0, 2, True, True, 4000, source)\n input, __ = to_reflectivity(input, metadata)\n else:\n input = np.zeros((3, 50, 50))\n metadata = {\"timestamps\": [\"00\", \"01\", \"02\"]}\n\n timelist = metadata[\"timestamps\"]\n\n cell_column_length = 9\n if output_splits_merges:\n cell_column_length = 15\n\n output = dating(\n input, timelist, mintrack=1, output_splits_merges=output_splits_merges\n )\n\n # Check output format\n assert isinstance(output, tuple)\n assert len(output) == 3\n assert isinstance(output[0], list)\n assert isinstance(output[1], list)\n assert isinstance(output[2], list)\n assert len(output[1]) == input.shape[0]\n assert len(output[2]) == input.shape[0]\n assert isinstance(output[1][0], pandas.DataFrame)\n assert isinstance(output[2][0], np.ndarray)\n assert output[1][0].shape[1] == cell_column_length\n assert output[2][0].shape == input.shape[1:]\n if not dry_input:\n assert len(output[0]) > 0\n assert isinstance(output[0][0], pandas.DataFrame)\n assert output[0][0].shape[1] == cell_column_length\n else:\n assert len(output[0]) == 0\n assert output[1][0].shape[0] == 0\n assert output[2][0].sum() == 0\n" }, { "path": "pysteps/tests/test_utils_arrays.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_equal\n\nfrom pysteps.utils import arrays\n\n# compute_centred_coord_array\ntest_data = [\n (2, 2, [np.array([[-1, 0]]).T, np.array([[-1, 0]])]),\n (3, 3, [np.array([[-1, 0, 1]]).T, np.array([[-1, 0, 1]])]),\n (3, 2, [np.array([[-1, 0, 1]]).T, np.array([[-1, 0]])]),\n (2, 3, [np.array([[-1, 0]]).T, np.array([[-1, 0, 1]])]),\n]\n\n\n@pytest.mark.parametrize(\"M, N, expected\", test_data)\ndef test_compute_centred_coord_array(M, N, expected):\n \"\"\"Test the compute_centred_coord_array.\"\"\"\n assert_array_equal(arrays.compute_centred_coord_array(M, N)[0], expected[0])\n assert_array_equal(arrays.compute_centred_coord_array(M, N)[1], expected[1])\n" }, { "path": "pysteps/tests/test_utils_cleansing.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\n\nfrom pysteps.utils import cleansing\n\n\ndef test_decluster_empty():\n \"\"\"\n Decluster an empty input\"\"\"\n\n X = np.empty((0, 2))\n V = np.empty((0, 2))\n X_dec, V_dec = cleansing.decluster(X, V, 20, 1)\n\n assert X_dec.ndim == 2\n assert V_dec.ndim == 2\n assert X_dec.shape[0] == 0\n assert V_dec.shape[0] == 0\n\n\ndef test_decluster_single():\n \"\"\"decluster a single vector\"\"\"\n\n X = np.array([[0.0, 0.0]])\n V = np.array([[1.0, 1.0]])\n X_dec, V_dec = cleansing.decluster(X, V, 20, 1)\n\n assert X_dec.ndim == 2\n assert V_dec.ndim == 2\n assert np.all(X_dec == X)\n assert np.all(X_dec == X)\n\n X_dec, V_dec = cleansing.decluster(X, V, 20, 2)\n assert X_dec.ndim == 2\n assert V_dec.ndim == 2\n assert X_dec.shape[0] == 0\n assert V_dec.shape[0] == 0\n\n\ndef test_decluster():\n \"\"\"decluster an input with duplicated vectors\"\"\"\n\n X = np.tile(np.random.randint(100, size=(10, 2)), (3, 1))\n V = np.tile(np.random.randint(100, size=(10, 2)), (3, 1))\n\n X_dec, V_dec = cleansing.decluster(X, V, 20, 1)\n\n assert X_dec.ndim == 2\n assert V_dec.ndim == 2\n assert X_dec.shape[0] <= V_dec.shape[0]\n assert X_dec.shape[0] <= 10\n assert V_dec.shape[0] <= 10\n\n X_dec, V_dec = cleansing.decluster(X, V, 100, 1)\n assert X_dec.ndim == 2\n assert V_dec.ndim == 2\n assert X_dec.shape[0] == 1\n assert V_dec.shape[0] == 1\n assert np.all(X_dec == np.median(X, axis=0))\n assert np.all(V_dec == np.median(V, axis=0))\n\n\ndef test_decluster_value_error_is_raise_when_input_has_nan():\n coords = np.ones((3, 1))\n input_array = np.ones((3, 1))\n\n input_array[1, 0] = np.nan\n with pytest.raises(ValueError):\n cleansing.decluster(coords, input_array, scale=20)\n\n\ndef test_detect_outlier_constant():\n \"\"\"Test that a constant input produces no outliers and that warnings are raised\"\"\"\n\n V = np.zeros(20) # this will trigger a runtime warning\n with pytest.warns(RuntimeWarning):\n outliers = cleansing.detect_outliers(V, 1)\n assert outliers.size == V.shape[0]\n assert outliers.sum() == 0\n\n V = np.zeros((20, 3)) # this will trigger a singular matrix warning\n with pytest.warns(UserWarning):\n outliers = cleansing.detect_outliers(V, 1)\n assert outliers.size == V.shape[0]\n assert outliers.sum() == 0\n\n V = np.zeros((20, 3)) # this will trigger a singular matrix warning\n X = np.random.randint(100, size=(20, 3))\n with pytest.warns(UserWarning):\n outliers = cleansing.detect_outliers(V, 1, coord=X, k=10)\n assert outliers.size == V.shape[0]\n assert outliers.sum() == 0\n\n\ndef test_detect_outlier_univariate_global():\n \"\"\"Test that\"\"\"\n\n # test input with no outliers at all\n V = np.random.randn(200)\n V = V[np.abs(V) < 1.5]\n outliers = cleansing.detect_outliers(V, 4)\n assert outliers.sum() == 0\n\n # test a postive outlier\n V[-1] = 10\n outliers = cleansing.detect_outliers(V, 4)\n assert outliers.sum() == 1\n\n # test a negative outlier\n V[-1] = -10\n outliers = cleansing.detect_outliers(V, 4)\n assert outliers.sum() == 1\n\n\ndef test_detect_outlier_multivariate_global():\n \"\"\"Test that\"\"\"\n\n # test input with no outliers at all\n V = np.random.multivariate_normal([0, 0], [[1, 0.5], [0.5, 1]], size=200)\n V = V[np.all(np.abs(V) < 1.5, axis=1), :]\n V = V[np.abs(V[:, 1] - V[:, 0]) < 0.5, :]\n outliers = cleansing.detect_outliers(V, 4)\n assert outliers.sum() == 0\n\n # test postive outliers\n V[-2, :] = (10, 0)\n V[-1, :] = (3, -3)\n outliers = cleansing.detect_outliers(V, 4)\n assert outliers.sum() == 2\n\n # test negative outliers\n V[-2] = (-10, 0)\n V[-1] = (-3, 3)\n outliers = cleansing.detect_outliers(V, 4)\n assert outliers.sum() == 2\n\n\ndef test_detect_outlier_univariate_local():\n \"\"\"Test that\"\"\"\n\n # test input with no outliers at all\n V = np.random.randn(200)\n X = np.random.randint(100, size=200)\n X = X[np.abs(V) < 1.5]\n V = V[np.abs(V) < 1.5]\n outliers = cleansing.detect_outliers(V, 4, coord=X, k=50)\n assert outliers.sum() == 0\n\n # test a postive outlier\n V[-1] = 10\n outliers = cleansing.detect_outliers(V, 4, coord=X, k=50)\n assert outliers.sum() == 1\n\n # test a negative outlier\n V[-1] = -10\n outliers = cleansing.detect_outliers(V, 4, coord=X, k=50)\n assert outliers.sum() == 1\n\n\ndef test_detect_outlier_multivariate_local():\n \"\"\"Test that\"\"\"\n\n # test input with no outliers at all\n V = np.random.multivariate_normal([0, 0], [[1, 0.5], [0.5, 1]], size=200)\n X = np.random.randint(100, size=(200, 3))\n idx = np.abs(V[:, 1] - V[:, 0]) < 1\n idx = idx & np.all(np.abs(V) < 1.5, axis=1)\n X = X[idx, :]\n V = V[idx, :]\n outliers = cleansing.detect_outliers(V, 4, coord=X, k=50)\n assert outliers.sum() == 0\n\n # test postive outliers\n V[-2, :] = (10, 0)\n V[-1, :] = (3, -3)\n outliers = cleansing.detect_outliers(V, 4, coord=X, k=50)\n assert outliers.sum() == 2\n\n # test negative outliers\n V[-2] = (-10, 0)\n V[-1] = (-3, 3)\n outliers = cleansing.detect_outliers(V, 4, coord=X, k=50)\n assert outliers.sum() == 2\n\n\ndef test_detect_outlier_wrong_input_dims_raise_error():\n input_array = np.zeros((20, 3, 2))\n thr_std_devs = 1\n with pytest.raises(ValueError):\n cleansing.detect_outliers(input_array, thr_std_devs)\n" }, { "path": "pysteps/tests/test_utils_conversion.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.utils import conversion\n\n# to_rainrate\ntest_data = [\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([1]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([12]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"dB\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([1.25892541]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"dB\",\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([15.10710494]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"dB\",\n \"unit\": \"dBZ\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([0.04210719]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"log\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([2.71828183]),\n ),\n (\n np.array([1.0]),\n {\n \"accutime\": 5,\n \"transform\": \"log\",\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([32.61938194]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"sqrt\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([1]),\n ),\n (\n np.array([1.0]),\n {\n \"accutime\": 5,\n \"transform\": \"sqrt\",\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([12.0]),\n ),\n]\n\n\n@pytest.mark.parametrize(\"R, metadata, expected\", test_data)\ndef test_to_rainrate(R, metadata, expected):\n \"\"\"Test the to_rainrate.\"\"\"\n assert_array_almost_equal(conversion.to_rainrate(R, metadata)[0], expected)\n\n\n# to_raindepth\ntest_data = [\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([0.08333333]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([1]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"dB\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([0.10491045]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"dB\",\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([1.25892541]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"dB\",\n \"unit\": \"dBZ\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([0.00350893]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"log\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([0.22652349]),\n ),\n (\n np.array([1.0]),\n {\n \"accutime\": 5,\n \"transform\": \"log\",\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([2.71828183]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"sqrt\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([0.08333333]),\n ),\n (\n np.array([1.0]),\n {\n \"accutime\": 5,\n \"transform\": \"sqrt\",\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([1.0]),\n ),\n]\n\n\n@pytest.mark.parametrize(\"R, metadata, expected\", test_data)\ndef test_to_raindepth(R, metadata, expected):\n \"\"\"Test the to_raindepth.\"\"\"\n assert_array_almost_equal(conversion.to_raindepth(R, metadata)[0], expected)\n\n\n# to_reflectivity\ntest_data = [\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([23.01029996]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([40.27719989]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"dB\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([24.61029996]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"dB\",\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([41.87719989]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"dB\",\n \"unit\": \"dBZ\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([1]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"log\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([29.95901167]),\n ),\n (\n np.array([1.0]),\n {\n \"accutime\": 5,\n \"transform\": \"log\",\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([47.2259116]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"sqrt\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([23.01029996]),\n ),\n (\n np.array([1.0]),\n {\n \"accutime\": 5,\n \"transform\": \"sqrt\",\n \"unit\": \"mm\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n np.array([40.27719989]),\n ),\n]\n\n\n@pytest.mark.parametrize(\"R, metadata, expected\", test_data)\ndef test_to_reflectivity(R, metadata, expected):\n \"\"\"Test the to_reflectivity.\"\"\"\n assert_array_almost_equal(conversion.to_reflectivity(R, metadata)[0], expected)\n" }, { "path": "pysteps/tests/test_utils_dimension.py", "content": "# -*- coding: utf-8 -*-\n\nimport datetime as dt\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_equal\nfrom pytest import raises\n\nfrom pysteps.utils import dimension\n\ntest_data_not_trim = (\n # \"data, window_size, axis, method, expected\"\n (np.arange(6), 2, 0, \"mean\", np.array([0.5, 2.5, 4.5])),\n (\n np.arange(4 * 6).reshape(4, 6),\n (2, 3),\n (0, 1),\n \"sum\",\n np.array([[24, 42], [96, 114]]),\n ),\n (\n np.arange(4 * 6).reshape(4, 6),\n (2, 2),\n (0, 1),\n \"sum\",\n np.array([[14, 22, 30], [62, 70, 78]]),\n ),\n (\n np.arange(4 * 6).reshape(4, 6),\n 2,\n (0, 1),\n \"sum\",\n np.array([[14, 22, 30], [62, 70, 78]]),\n ),\n (\n np.arange(4 * 6).reshape(4, 6),\n (2, 3),\n (0, 1),\n \"mean\",\n np.array([[4.0, 7.0], [16.0, 19.0]]),\n ),\n (\n np.arange(4 * 6).reshape(4, 6),\n (2, 2),\n (0, 1),\n \"mean\",\n np.array([[3.5, 5.5, 7.5], [15.5, 17.5, 19.5]]),\n ),\n (\n np.arange(4 * 6).reshape(4, 6),\n 2,\n (0, 1),\n \"mean\",\n np.array([[3.5, 5.5, 7.5], [15.5, 17.5, 19.5]]),\n ),\n)\n\n\n@pytest.mark.parametrize(\n \"data, window_size, axis, method, expected\", test_data_not_trim\n)\ndef test_aggregate_fields(data, window_size, axis, method, expected):\n \"\"\"\n Test the aggregate_fields function.\n The windows size must divide exactly the data dimensions.\n Internally, additional test are generated for situations where the\n windows size does not divide the data dimensions.\n The length of each dimension should be larger than 2.\n \"\"\"\n\n assert_array_equal(\n dimension.aggregate_fields(data, window_size, axis=axis, method=method),\n expected,\n )\n\n # Test the trimming capabilities.\n data = np.pad(data, (0, 1))\n assert_array_equal(\n dimension.aggregate_fields(\n data, window_size, axis=axis, method=method, trim=True\n ),\n expected,\n )\n\n with raises(ValueError):\n dimension.aggregate_fields(data, window_size, axis=axis, method=method)\n\n\ndef test_aggregate_fields_errors():\n \"\"\"\n Test that the errors are correctly captured in the aggregate_fields\n function.\n \"\"\"\n data = np.arange(4 * 6).reshape(4, 6)\n\n with raises(ValueError):\n dimension.aggregate_fields(data, -1, axis=0)\n with raises(ValueError):\n dimension.aggregate_fields(data, 0, axis=0)\n with raises(ValueError):\n dimension.aggregate_fields(data, 1, method=\"invalid\")\n\n with raises(TypeError):\n dimension.aggregate_fields(data, (1, 1), axis=0)\n\n\n# aggregate_fields_time\ntimestamps = [dt.datetime.now() + dt.timedelta(minutes=t) for t in range(10)]\ntest_data = [\n (\n np.ones((10, 1, 1)),\n {\"unit\": \"mm/h\", \"timestamps\": timestamps},\n 2,\n False,\n np.ones((5, 1, 1)),\n ),\n (\n np.ones((10, 1, 1)),\n {\"unit\": \"mm\", \"timestamps\": timestamps},\n 2,\n False,\n 2 * np.ones((5, 1, 1)),\n ),\n]\n\n\n@pytest.mark.parametrize(\n \"R, metadata, time_window_min, ignore_nan, expected\", test_data\n)\ndef test_aggregate_fields_time(R, metadata, time_window_min, ignore_nan, expected):\n \"\"\"Test the aggregate_fields_time.\"\"\"\n assert_array_equal(\n dimension.aggregate_fields_time(R, metadata, time_window_min, ignore_nan)[0],\n expected,\n )\n\n\n# aggregate_fields_space\ntest_data = [\n (\n np.ones((1, 10, 10)),\n {\"unit\": \"mm/h\", \"xpixelsize\": 1, \"ypixelsize\": 1},\n 2,\n False,\n np.ones((1, 5, 5)),\n ),\n (\n np.ones((1, 10, 10)),\n {\"unit\": \"mm\", \"xpixelsize\": 1, \"ypixelsize\": 1},\n 2,\n False,\n np.ones((1, 5, 5)),\n ),\n (\n np.ones((1, 10, 10)),\n {\"unit\": \"mm/h\", \"xpixelsize\": 1, \"ypixelsize\": 2},\n (2, 4),\n False,\n np.ones((1, 5, 5)),\n ),\n]\n\n\n@pytest.mark.parametrize(\"R, metadata, space_window, ignore_nan, expected\", test_data)\ndef test_aggregate_fields_space(R, metadata, space_window, ignore_nan, expected):\n \"\"\"Test the aggregate_fields_space.\"\"\"\n assert_array_equal(\n dimension.aggregate_fields_space(R, metadata, space_window, ignore_nan)[0],\n expected,\n )\n\n\n# clip_domain\nR = np.zeros((4, 4))\nR[:2, :] = 1\ntest_data = [\n (\n R,\n {\n \"x1\": 0,\n \"x2\": 4,\n \"y1\": 0,\n \"y2\": 4,\n \"xpixelsize\": 1,\n \"ypixelsize\": 1,\n \"zerovalue\": 0,\n \"yorigin\": \"upper\",\n },\n None,\n R,\n ),\n (\n R,\n {\n \"x1\": 0,\n \"x2\": 4,\n \"y1\": 0,\n \"y2\": 4,\n \"xpixelsize\": 1,\n \"ypixelsize\": 1,\n \"zerovalue\": 0,\n \"yorigin\": \"lower\",\n },\n (2, 4, 2, 4),\n np.zeros((2, 2)),\n ),\n (\n R,\n {\n \"x1\": 0,\n \"x2\": 4,\n \"y1\": 0,\n \"y2\": 4,\n \"xpixelsize\": 1,\n \"ypixelsize\": 1,\n \"zerovalue\": 0,\n \"yorigin\": \"upper\",\n },\n (2, 4, 2, 4),\n np.ones((2, 2)),\n ),\n]\n\n\n@pytest.mark.parametrize(\"R, metadata, extent, expected\", test_data)\ndef test_clip_domain(R, metadata, extent, expected):\n \"\"\"Test the clip_domain.\"\"\"\n assert_array_equal(dimension.clip_domain(R, metadata, extent)[0], expected)\n\n\n# square_domain\nR = np.zeros((4, 2))\ntest_data = [\n # square by padding\n (\n R,\n {\"x1\": 0, \"x2\": 2, \"y1\": 0, \"y2\": 4, \"xpixelsize\": 1, \"ypixelsize\": 1},\n \"pad\",\n False,\n np.zeros((4, 4)),\n ),\n # square by cropping\n (\n R,\n {\"x1\": 0, \"x2\": 2, \"y1\": 0, \"y2\": 4, \"xpixelsize\": 1, \"ypixelsize\": 1},\n \"crop\",\n False,\n np.zeros((2, 2)),\n ),\n # inverse square by padding\n (\n np.zeros((4, 4)),\n {\n \"x1\": -1,\n \"x2\": 3,\n \"y1\": 0,\n \"y2\": 4,\n \"xpixelsize\": 1,\n \"ypixelsize\": 1,\n \"orig_domain\": (4, 2),\n \"square_method\": \"pad\",\n },\n \"pad\",\n True,\n R,\n ),\n # inverse square by cropping\n (\n np.zeros((2, 2)),\n {\n \"x1\": 0,\n \"x2\": 2,\n \"y1\": 1,\n \"y2\": 3,\n \"xpixelsize\": 1,\n \"ypixelsize\": 1,\n \"orig_domain\": (4, 2),\n \"square_method\": \"crop\",\n },\n \"crop\",\n True,\n R,\n ),\n]\n\n\n@pytest.mark.parametrize(\"R, metadata, method, inverse, expected\", test_data)\ndef test_square_domain(R, metadata, method, inverse, expected):\n \"\"\"Test the square_domain.\"\"\"\n assert_array_equal(\n dimension.square_domain(R, metadata, method, inverse)[0], expected\n )\n" }, { "path": "pysteps/tests/test_utils_interpolate.py", "content": "# -*- coding: utf-8 -*-\nimport numpy as np\nimport pytest\n\nfrom pysteps.utils import get_method\n\ninterp_methods = (\n \"idwinterp2d\",\n \"rbfinterp2d\",\n)\n\n\n@pytest.mark.parametrize(\"interp_method\", interp_methods)\ndef test_interp_univariate(interp_method):\n coord = np.random.rand(10, 2)\n input_array = np.random.rand(10)\n xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10)\n\n interp = get_method(interp_method)\n output = interp(coord, input_array, xgrid, ygrid)\n\n assert isinstance(output, np.ndarray)\n assert output.ndim == 2\n assert output.shape == (ygrid.size, xgrid.size)\n assert np.isfinite(output).all()\n\n\n@pytest.mark.parametrize(\"interp_method\", interp_methods)\ndef test_interp_multivariate(interp_method):\n coord = np.random.rand(10, 2)\n input_array = np.random.rand(10, 2)\n xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10)\n\n interp = get_method(interp_method)\n output = interp(coord, input_array, xgrid, ygrid)\n\n assert isinstance(output, np.ndarray)\n assert output.ndim == 3\n assert output.shape[0] == 2\n assert output.shape[1:] == (ygrid.size, xgrid.size)\n assert np.isfinite(output).all()\n\n\n@pytest.mark.parametrize(\"interp_method\", interp_methods)\ndef test_wrong_inputs(interp_method):\n coord = np.random.rand(10, 2)\n input_array = np.random.rand(10, 2)\n xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10)\n\n interp = get_method(interp_method)\n\n # nan in the input values\n with pytest.raises(ValueError):\n input_with_nans = input_array.copy()\n input_with_nans[0, 0] = np.nan\n interp(coord, input_with_nans, xgrid, ygrid)\n\n # nan in the input coordinates\n with pytest.raises(ValueError):\n coord_with_nans = coord.copy()\n coord_with_nans[0, 0] = np.nan\n interp(coord_with_nans, input_array, xgrid, ygrid)\n\n # too many dimensions in the input values\n with pytest.raises(ValueError):\n interp(coord, np.random.rand(10, 2, 1), xgrid, ygrid)\n\n # wrong dimension size in the input coordinates\n with pytest.raises(ValueError):\n interp(np.random.rand(10, 1), input_array, xgrid, ygrid)\n\n # wrong number of dimensions in the input coordinates\n with pytest.raises(ValueError):\n interp(np.random.rand(10, 2, 1), input_array, xgrid, ygrid)\n\n # wrong number of coordinates\n with pytest.raises(ValueError):\n interp(np.random.rand(9, 2), input_array, xgrid, ygrid)\n\n\n@pytest.mark.parametrize(\"interp_method\", interp_methods)\ndef test_one_sample_input(interp_method):\n coord = np.random.rand(1, 2)\n input_array = np.array([1, 2])[None, :]\n xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10)\n\n interp = get_method(interp_method)\n\n # one sample returns uniform grids\n output = interp(coord, input_array, xgrid, ygrid)\n assert np.isfinite(output).all()\n assert output[0, ...].max() == output[0, ...].min() == 1\n assert output[1, ...].max() == output[1, ...].min() == 2\n\n\n@pytest.mark.parametrize(\"interp_method\", interp_methods)\ndef test_uniform_input(interp_method):\n coord = np.random.rand(10, 2)\n xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10)\n\n interp = get_method(interp_method)\n\n # same value across all variables\n input_array = np.ones((10, 2))\n output = interp(coord, input_array, xgrid, ygrid)\n assert np.isfinite(output).all()\n assert output.max() == output.min() == input_array.ravel()[0]\n\n # # same value in one variable only\n # input_array = np.vstack((np.ones(10), np.random.rand(10))).T\n # output = interp(coord, input_array, xgrid, ygrid)\n # assert output[0,].max() == output[0,].min() == input_array[0,0]\n\n\ndef test_idwinterp2d_k1():\n coord = np.random.rand(10, 2)\n input_array = np.random.rand(10, 2)\n xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10)\n\n interp = get_method(\"idwinterp2d\")\n output = interp(coord, input_array, xgrid, ygrid, k=1)\n\n assert isinstance(output, np.ndarray)\n assert output.ndim == 3\n assert output.shape[0] == 2\n assert output.shape[1:] == (ygrid.size, xgrid.size)\n assert np.isfinite(output).all()\n\n\ndef test_idwinterp2d_kNone():\n coord = np.random.rand(10, 2)\n input_array = np.random.rand(10, 2)\n xgrid, ygrid = np.linspace(0, 1, 10), np.linspace(0, 1, 10)\n\n interp = get_method(\"idwinterp2d\")\n output = interp(coord, input_array, xgrid, ygrid, k=None)\n\n assert isinstance(output, np.ndarray)\n assert output.ndim == 3\n assert output.shape[0] == 2\n assert output.shape[1:] == (ygrid.size, xgrid.size)\n assert np.isfinite(output).all()\n" }, { "path": "pysteps/tests/test_utils_pca.py", "content": "# -*- coding: utf-8 -*-\n\nimport pytest\nimport numpy as np\nfrom pysteps.utils import pca\n\npca_arg_values = (\n (10, 10),\n (20, 20),\n (10, 5),\n (20, 15),\n)\n\npca_arg_names = (\"len_y\", \"n_components\")\n\n\n@pytest.mark.parametrize(pca_arg_names, pca_arg_values)\ndef test_pca(len_y, n_components):\n\n pytest.importorskip(\"sklearn\")\n\n precip_field = np.zeros((len_y, 200, 200))\n for i in range(len_y):\n a = 3 * i\n b = 2 * i\n precip_field[i, 20 + b : 160 - b, 30 + a : 180 - a] = 0.1\n precip_field[i, 22 + b : 162 - b, 35 + a : 178 - a] = 0.1\n precip_field[i, 24 + b : 164 - b, 40 + a : 176 - a] = 1.0\n precip_field[i, 26 + b : 166 - b, 45 + a : 174 - a] = 5.0\n precip_field[i, 28 + b : 168 - b, 50 + a : 172 - a] = 5.0\n precip_field[i, 30 + b : 170 - b, 35 + a : 170 - a] = 4.5\n precip_field[i, 32 + b : 172 - b, 40 + a : 168 - a] = 4.5\n precip_field[i, 34 + b : 174 - b, 45 + a : 166 - a] = 4.0\n precip_field[i, 36 + b : 176 - b, 50 + a : 164 - a] = 2.0\n precip_field[i, 38 + b : 178 - b, 55 + a : 162 - a] = 1.0\n precip_field[i, 40 + b : 180 - b, 60 + a : 160 - a] = 0.5\n precip_field[i, 42 + b : 182 - b, 65 + a : 158 - a] = 0.1\n\n precip_field = precip_field.reshape(\n len_y, precip_field.shape[1] * precip_field.shape[2]\n )\n\n kwargs = {\"n_components\": n_components, \"svd_solver\": \"full\"}\n precip_field_pc, pca_params = pca.pca_transform(\n forecast_ens=precip_field, get_params=True, **kwargs\n )\n\n assert precip_field_pc.shape == (len_y, n_components)\n assert pca_params[\"principal_components\"].shape[1] == precip_field.shape[1]\n assert pca_params[\"mean\"].shape[0] == precip_field.shape[1]\n\n precip_field_backtransformed = pca.pca_backtransform(\n precip_field_pc, pca_params=pca_params\n )\n\n # These fields are only equal if the full PCA is computed\n if len_y == n_components:\n assert np.sum(np.abs(precip_field_backtransformed - precip_field)) < 1e-6\n" }, { "path": "pysteps/tests/test_utils_reprojection.py", "content": "# -*- coding: utf-8 -*-\n\nimport os\nimport numpy as np\nimport pytest\nimport pysteps\nfrom pysteps.utils import reprojection as rpj\n\npytest.importorskip(\"rasterio\")\npytest.importorskip(\"pyproj\")\n\nroot_path_radar = pysteps.rcparams.data_sources[\"rmi\"][\"root_path\"]\n\nrel_path_radar = \"20210704\" # Different date, but that does not matter for the tester\n\nfilename_radar = os.path.join(\n root_path_radar, rel_path_radar, \"20210704180500.rad.best.comp.rate.qpe.hdf\"\n)\n\n# Open the radar data\nradar_array, _, metadata_dst = pysteps.io.importers.import_odim_hdf5(filename_radar)\n\n# Initialise dummy NWP data\nnwp_array = np.zeros((24, 564, 564))\n\nfor t in range(nwp_array.shape[0]):\n nwp_array[t, 30 + t : 185 + t, 30 + 2 * t] = 0.1\n nwp_array[t, 30 + t : 185 + t, 31 + 2 * t] = 0.1\n nwp_array[t, 30 + t : 185 + t, 32 + 2 * t] = 1.0\n nwp_array[t, 30 + t : 185 + t, 33 + 2 * t] = 5.0\n nwp_array[t, 30 + t : 185 + t, 34 + 2 * t] = 5.0\n nwp_array[t, 30 + t : 185 + t, 35 + 2 * t] = 4.5\n nwp_array[t, 30 + t : 185 + t, 36 + 2 * t] = 4.5\n nwp_array[t, 30 + t : 185 + t, 37 + 2 * t] = 4.0\n nwp_array[t, 30 + t : 185 + t, 38 + 2 * t] = 2.0\n nwp_array[t, 30 + t : 185 + t, 39 + 2 * t] = 1.0\n nwp_array[t, 30 + t : 185 + t, 40 + 2 * t] = 0.5\n nwp_array[t, 30 + t : 185 + t, 41 + 2 * t] = 0.1\n\nnwp_proj = (\n \"+proj=lcc +lon_0=4.55 +lat_1=50.8 +lat_2=50.8 \"\n \"+a=6371229 +es=0 +lat_0=50.8 +x_0=365950 +y_0=-365950.000000001\"\n)\n\nmetadata_src = dict(\n projection=nwp_proj,\n institution=\"Royal Meteorological Institute of Belgium\",\n transform=None,\n zerovalue=0.0,\n threshold=0,\n unit=\"mm\",\n accutime=5,\n xpixelsize=1300.0,\n ypixelsize=1300.0,\n yorigin=\"upper\",\n cartesian_unit=\"m\",\n x1=0.0,\n x2=731900.0,\n y1=-731900.0,\n y2=0.0,\n)\n\nsteps_arg_names = (\n \"radar_array\",\n \"nwp_array\",\n \"metadata_src\",\n \"metadata_dst\",\n)\n\nsteps_arg_values = [\n (radar_array, nwp_array, metadata_src, metadata_dst),\n]\n\n\n@pytest.mark.parametrize(steps_arg_names, steps_arg_values)\ndef test_utils_reproject_grids(\n radar_array,\n nwp_array,\n metadata_src,\n metadata_dst,\n):\n # Reproject\n nwp_array_reproj, metadata_reproj = rpj.reproject_grids(\n nwp_array, radar_array, metadata_src, metadata_dst\n )\n\n # The tests\n assert (\n nwp_array_reproj.shape[0] == nwp_array.shape[0]\n ), \"Time dimension has not the same length as source\"\n assert (\n nwp_array_reproj.shape[1] == radar_array.shape[0]\n ), \"y dimension has not the same length as radar composite\"\n assert (\n nwp_array_reproj.shape[2] == radar_array.shape[1]\n ), \"x dimension has not the same length as radar composite\"\n\n assert (\n metadata_reproj[\"x1\"] == metadata_dst[\"x1\"]\n ), \"x-value lower left corner is not equal to radar composite\"\n assert (\n metadata_reproj[\"x2\"] == metadata_dst[\"x2\"]\n ), \"x-value upper right corner is not equal to radar composite\"\n assert (\n metadata_reproj[\"y1\"] == metadata_dst[\"y1\"]\n ), \"y-value lower left corner is not equal to radar composite\"\n assert (\n metadata_reproj[\"y2\"] == metadata_dst[\"y2\"]\n ), \"y-value upper right corner is not equal to radar composite\"\n\n assert (\n metadata_reproj[\"projection\"] == metadata_dst[\"projection\"]\n ), \"projection is different than destination projection\"\n" }, { "path": "pysteps/tests/test_utils_spectral.py", "content": "import numpy as np\nimport pytest\nfrom pysteps.utils import spectral\n\n_rapsd_input_fields = [\n np.random.uniform(size=(255, 255)),\n np.random.uniform(size=(256, 256)),\n np.random.uniform(size=(255, 256)),\n np.random.uniform(size=(256, 255)),\n]\n\n\n@pytest.mark.parametrize(\"field\", _rapsd_input_fields)\ndef test_rapsd(field):\n rapsd, freq = spectral.rapsd(field, return_freq=True)\n\n m, n = field.shape\n l = max(m, n)\n\n if l % 2 == 0:\n assert len(rapsd) == int(l / 2)\n else:\n assert len(rapsd) == int(l / 2 + 1)\n assert len(rapsd) == len(freq)\n assert np.all(freq >= 0.0)\n" }, { "path": "pysteps/tests/test_utils_transformation.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.utils import transformation\n\n# boxcox_transform\ntest_data = [\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n None,\n None,\n None,\n False,\n np.array([0]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"BoxCox\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n None,\n None,\n None,\n True,\n np.array([np.exp(1)]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n 1.0,\n None,\n None,\n False,\n np.array([0]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"BoxCox\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n 1.0,\n None,\n None,\n True,\n np.array([2.0]),\n ),\n]\n\n\n@pytest.mark.parametrize(\n \"R, metadata, Lambda, threshold, zerovalue, inverse, expected\", test_data\n)\ndef test_boxcox_transform(R, metadata, Lambda, threshold, zerovalue, inverse, expected):\n \"\"\"Test the boxcox_transform.\"\"\"\n assert_array_almost_equal(\n transformation.boxcox_transform(\n R, metadata, Lambda, threshold, zerovalue, inverse\n )[0],\n expected,\n )\n\n\n# dB_transform\ntest_data = [\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n None,\n None,\n False,\n np.array([0]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"dB\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n None,\n None,\n True,\n np.array([1.25892541]),\n ),\n]\n\n\n@pytest.mark.parametrize(\n \"R, metadata, threshold, zerovalue, inverse, expected\", test_data\n)\ndef test_dB_transform(R, metadata, threshold, zerovalue, inverse, expected):\n \"\"\"Test the dB_transform.\"\"\"\n assert_array_almost_equal(\n transformation.dB_transform(R, metadata, threshold, zerovalue, inverse)[0],\n expected,\n )\n\n\n# NQ_transform\ntest_data = [\n (\n np.array([1, 2]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n False,\n np.array([-0.4307273, 0.4307273]),\n )\n]\n\n\n@pytest.mark.parametrize(\"R, metadata, inverse, expected\", test_data)\ndef test_NQ_transform(R, metadata, inverse, expected):\n \"\"\"Test the NQ_transform.\"\"\"\n assert_array_almost_equal(\n transformation.NQ_transform(R, metadata, inverse)[0], expected\n )\n\n\n# sqrt_transform\ntest_data = [\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": None,\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n False,\n np.array([1]),\n ),\n (\n np.array([1]),\n {\n \"accutime\": 5,\n \"transform\": \"sqrt\",\n \"unit\": \"mm/h\",\n \"threshold\": 0,\n \"zerovalue\": 0,\n },\n True,\n np.array([1]),\n ),\n]\n\n\n@pytest.mark.parametrize(\"R, metadata, inverse, expected\", test_data)\ndef test_sqrt_transform(R, metadata, inverse, expected):\n \"\"\"Test the sqrt_transform.\"\"\"\n assert_array_almost_equal(\n transformation.sqrt_transform(R, metadata, inverse)[0], expected\n )\n" }, { "path": "pysteps/tests/test_verification_detcatscores.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.verification import det_cat_fct\n\n# CREATE A LARGE DATASET TO MATCH\n# EXAMPLES IN\n# http://www.cawcr.gov.au/projects/verification/\n\nfct_hits = 1.0 * np.ones(82)\nobs_hits = 1.0 * np.ones(82)\nfct_fa = 1.0 * np.ones(38)\nobs_fa = 1.0 * np.zeros(38)\nfct_misses = 1.0 * np.zeros(23)\nobs_misses = 1.0 * np.ones(23)\nfct_cr = 1.0 * np.zeros(222)\nobs_cr = 1.0 * np.zeros(222)\nobs_data = np.concatenate([obs_hits, obs_fa, obs_misses, obs_cr])\nfct_data = np.concatenate([fct_hits, fct_fa, fct_misses, fct_cr])\n\ntest_data = [\n ([0.0], [0.0], 0.0, None, []),\n ([1.0, 3.0], [2.0, 5.0], 0.0, None, []),\n ([1.0, 3.0], [2.0, 5.0], 0.0, \"CSI\", [1.0]),\n ([1.0, 3.0], [2.0, 5.0], 0.0, (\"CSI\", \"FAR\"), [1.0, 0.0]),\n ([1.0, 3.0], [2.0, 5.0], 0.0, (\"lolo\",), []),\n ([1.0, 3.0], [2.0, 5.0], 0.0, (\"CSI\", None, \"FAR\"), [1.0, 0.0]),\n ([1.0, 3.0], [2.0, 5.0], 1.0, (\"CSI\", None, \"FAR\"), [0.5, 0.0]),\n ([1.0, 3.0], [2.0, 5.0], 1.0, (\"lolo\"), []), # test unknown score\n (fct_data, obs_data, 0.0, (\"ACC\"), [0.83287671]), # ACCURACY score\n (fct_data, obs_data, 0.0, (\"BIAS\"), [1.1428571]), # BIAS score\n (fct_data, obs_data, 0.0, (\"POD\"), [0.7809524]), # POD score\n (fct_data, obs_data, 0.0, (\"FAR\"), [0.316667]), # FAR score\n # Probability of false detection (false alarm rate)\n (fct_data, obs_data, 0.0, (\"FA\"), [0.146154]),\n # CSI score\n (fct_data, obs_data, 0.0, (\"CSI\"), [0.573426]),\n # Heidke Skill Score\n (fct_data, obs_data, 0.0, (\"HSS\"), [0.608871]),\n # Hanssen-Kuipers Discriminant\n (fct_data, obs_data, 0.0, (\"HK\"), [0.6348]),\n # Gilbert Skill Score\n (fct_data, obs_data, 0.0, (\"GSS\"), [0.437682]),\n # Gilbert Skill Score\n (fct_data, obs_data, 0.0, (\"ETS\"), [0.437682]),\n # Symmetric extremal dependence index\n (fct_data, obs_data, 0.0, (\"SEDI\"), [0.789308]),\n # Matthews correlation coefficient\n (fct_data, obs_data, 0.0, (\"MCC\"), [0.611707]),\n # F1-score\n (fct_data, obs_data, 0.0, (\"F1\"), [0.728889]),\n]\n\n\n@pytest.mark.parametrize(\"pred, obs, thr, scores, expected\", test_data)\ndef test_det_cat_fct(pred, obs, thr, scores, expected):\n \"\"\"Test the det_cat_fct.\"\"\"\n assert_array_almost_equal(\n list(det_cat_fct(pred, obs, thr, scores).values()), expected\n )\n" }, { "path": "pysteps/tests/test_verification_detcontscores.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.verification import det_cont_fct\n\n# CREATE A DATASET TO MATCH\n# EXAMPLES IN\n# http://www.cawcr.gov.au/projects/verification/\n\nobs_data = np.asarray(\n [7, 10, 9, 15, 22, 13, 17, 17, 19, 23.0, 0, 10, 0, 15, 0, 13, 0, 17, 0, 0.0]\n)\nfct_data = np.asarray(\n [1, 8, 12, 13, 18, 10, 16, 19, 23, 24.0, 0, 0, 12, 0, 0, 0, 16, 0, 0, 0.0]\n)\n\ntest_data = [\n # test None as score\n ([0.0], [0.0], None, None, None, []),\n # test unknown score\n ([1.0, 3.0], [2.0, 5.0], (\"lolo\"), None, None, []),\n # test unknown score and None\n ([1.0, 3.0], [2.0, 5.0], (\"lolo\", None), None, None, []),\n # Mean Error as string\n (fct_data, obs_data, \"ME\", None, None, [-1.75]),\n # Mean Error\n (fct_data, obs_data, (\"ME\"), None, None, [-1.75]),\n # Mean Error single conditional\n (fct_data, obs_data, (\"ME\"), None, \"single\", [-2.1875]),\n # Mean Error double conditional\n (fct_data, obs_data, (\"ME\"), None, \"double\", [-0.8]),\n # Mean Absolute Error\n (fct_data, obs_data, (\"MAE\"), None, None, [5.55]),\n # Mean Square Error\n (fct_data, obs_data, (\"MSE\"), None, None, [64.15]),\n # Normalized Mean Square Error\n (fct_data, obs_data, (\"NMSE\"), None, None, [0.113711]),\n # Root Mean Square Error\n (fct_data, obs_data, (\"RMSE\"), None, None, [8.009370]),\n # Beta1\n (fct_data, obs_data, (\"beta1\"), None, None, [0.498200]),\n # Beta2\n (fct_data, obs_data, (\"beta2\"), None, None, [0.591673]),\n # reduction of variance\n (fct_data, obs_data, (\"RV\"), None, None, [-0.054622]),\n # debiased RMSE\n (fct_data, obs_data, (\"DRMSE\"), None, None, [7.815849]),\n # Pearson correlation\n (fct_data, obs_data, (\"corr_p\"), None, None, [0.542929]),\n # Spearman correlation\n (fct_data, obs_data, (\"corr_s\"), None, None, [0.565251]),\n # Spearman correlation single conditional\n (fct_data, obs_data, (\"corr_s\"), None, \"single\", [0.467913]),\n # Spearman correlation double conditional\n (fct_data, obs_data, (\"corr_s\"), None, \"double\", [0.917937]),\n # scatter\n (fct_data, obs_data, (\"scatter\"), None, None, [0.808023]),\n # Mean Error along axis 0 as tuple\n (\n np.tile(fct_data, (2, 1)).T,\n np.tile(obs_data, (2, 1)).T,\n \"ME\",\n (0,),\n None,\n [[-1.75, -1.75]],\n ),\n # Mean Error along axis 0\n (\n np.tile(fct_data, (2, 1)).T,\n np.tile(obs_data, (2, 1)).T,\n \"ME\",\n 0,\n None,\n [[-1.75, -1.75]],\n ),\n # Mean Error along axis 1\n (\n np.tile(fct_data, (2, 1)).T,\n np.tile(obs_data, (2, 1)).T,\n \"ME\",\n 1,\n None,\n [[-6, -2, 3, -2, -4, -3, -1, 2, 4, 1, 0, -10, 12, -15, 0, -13, 16, -17, 0, 0]],\n ),\n # Mean Error along axis (1,2)\n (\n np.tile(fct_data, (2, 1)).T,\n np.tile(obs_data, (2, 1)).T,\n \"ME\",\n (0, 1),\n None,\n [-1.75],\n ),\n # Mean Error along axis (2,1)\n (\n np.tile(fct_data, (2, 1)).T,\n np.tile(obs_data, (2, 1)).T,\n \"ME\",\n (1, 0),\n None,\n [-1.75],\n ),\n # scatter along axis 0 as tuple\n (\n np.tile(fct_data, (2, 1)).T,\n np.tile(obs_data, (2, 1)).T,\n \"scatter\",\n (0,),\n None,\n [[0.808023, 0.808023]],\n ),\n # scatter along axis 0\n (\n np.tile(fct_data, (2, 1)).T,\n np.tile(obs_data, (2, 1)).T,\n \"scatter\",\n 0,\n None,\n [[0.808023, 0.808023]],\n ),\n # scatter along axis (1,2)\n (\n np.tile(fct_data, (2, 1)).T,\n np.tile(obs_data, (2, 1)).T,\n \"scatter\",\n (0, 1),\n None,\n [0.804806],\n ),\n # scatter along axis (2,1)\n (\n np.tile(fct_data, (2, 1)).T,\n np.tile(obs_data, (2, 1)).T,\n \"scatter\",\n (1, 0),\n None,\n [0.804806],\n ),\n]\n\n\n@pytest.mark.parametrize(\"pred, obs, scores, axis, conditioning, expected\", test_data)\ndef test_det_cont_fct(pred, obs, scores, axis, conditioning, expected):\n \"\"\"Test the det_cont_fct.\"\"\"\n assert_array_almost_equal(\n list(det_cont_fct(pred, obs, scores, axis, conditioning).values()), expected\n )\n" }, { "path": "pysteps/tests/test_verification_probscores.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.postprocessing.ensemblestats import excprob\nfrom pysteps.tests.helpers import get_precipitation_fields\nfrom pysteps.verification import probscores\n\nprecip = get_precipitation_fields(num_next_files=10, return_raw=True)\n\n# CRPS\ntest_data = [(precip[:10], precip[-1], 0.01470871)]\n\n\n@pytest.mark.parametrize(\"X_f, X_o, expected\", test_data)\ndef test_CRPS(X_f, X_o, expected):\n \"\"\"Test the CRPS.\"\"\"\n assert_array_almost_equal(probscores.CRPS(X_f, X_o), expected)\n\n\n# reldiag\ntest_data = [(precip[:10], precip[-1], 1.0, 10, 10, 3.38751492)]\n\n\n@pytest.mark.parametrize(\"X_f, X_o, X_min, n_bins, min_count, expected\", test_data)\ndef test_reldiag_sum(X_f, X_o, X_min, n_bins, min_count, expected):\n \"\"\"Test the reldiag.\"\"\"\n P_f = excprob(X_f, X_min, ignore_nan=False)\n assert_array_almost_equal(\n np.sum(probscores.reldiag(P_f, X_o, X_min, n_bins, min_count)[1]), expected\n )\n\n\n# ROC_curve\ntest_data = [(precip[:10], precip[-1], 1.0, 10, True, 0.79557329)]\n\n\n@pytest.mark.parametrize(\n \"X_f, X_o, X_min, n_prob_thrs, compute_area, expected\", test_data\n)\ndef test_ROC_curve_area(X_f, X_o, X_min, n_prob_thrs, compute_area, expected):\n \"\"\"Test the ROC_curve.\"\"\"\n P_f = excprob(X_f, X_min, ignore_nan=False)\n assert_array_almost_equal(\n probscores.ROC_curve(P_f, X_o, X_min, n_prob_thrs, compute_area)[2], expected\n )\n" }, { "path": "pysteps/tests/test_verification_salscores.py", "content": "# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pytest\n\nfrom pysteps.tests.helpers import get_precipitation_fields\nfrom pysteps.verification.salscores import sal\nfrom pysteps.utils import to_rainrate, to_reflectivity\n\ntest_data = [\n (to_rainrate, 1 / 15),\n (to_reflectivity, None),\n]\n\n\n@pytest.mark.parametrize(\"converter, thr_factor\", test_data)\nclass TestSAL:\n pytest.importorskip(\"pandas\")\n pytest.importorskip(\"skimage\")\n\n def test_sal_zeros(self, converter, thr_factor):\n \"\"\"Test the SAL verification method.\"\"\"\n precip, metadata = get_precipitation_fields(\n num_prev_files=0, log_transform=False, metadata=True\n )\n precip, metadata = converter(precip.filled(np.nan), metadata)\n result = sal(precip * 0, precip * 0, thr_factor)\n assert np.isnan(result).all()\n result = sal(precip * 0, precip, thr_factor)\n assert result[:2] == (-2, -2)\n assert np.isnan(result[2])\n result = sal(precip, precip * 0, thr_factor)\n assert result[:2] == (2, 2)\n assert np.isnan(result[2])\n\n def test_sal_same_image(self, converter, thr_factor):\n \"\"\"Test the SAL verification method.\"\"\"\n precip, metadata = get_precipitation_fields(\n num_prev_files=0, log_transform=False, metadata=True\n )\n precip, metadata = converter(precip.filled(np.nan), metadata)\n result = sal(precip, precip, thr_factor)\n assert isinstance(result, tuple)\n assert len(result) == 3\n assert np.allclose(result, [0, 0, 0])\n\n def test_sal_translation(self, converter, thr_factor):\n precip, metadata = get_precipitation_fields(\n num_prev_files=0, log_transform=False, metadata=True\n )\n precip, metadata = converter(precip.filled(np.nan), metadata)\n precip_translated = np.roll(precip, 10, axis=0)\n result = sal(precip, precip_translated, thr_factor)\n assert np.allclose(result[0], 0)\n assert np.allclose(result[1], 0)\n assert not np.allclose(result[2], 0)\n" }, { "path": "pysteps/tests/test_verification_spatialscores.py", "content": "# -*- coding: utf-8 -*-\n\nimport pytest\nfrom numpy.testing import assert_array_almost_equal\n\nfrom pysteps.tests.helpers import get_precipitation_fields\nfrom pysteps.verification import spatialscores\n\nR = get_precipitation_fields(num_prev_files=1, return_raw=True)\ntest_data = [\n (R[0], R[1], \"FSS\", [1], [10], None, 0.85161531),\n (R[0], R[1], \"BMSE\", [1], None, \"Haar\", 0.99989651),\n]\n\n\n@pytest.mark.parametrize(\"X_f, X_o, name, thrs, scales, wavelet, expected\", test_data)\ndef test_intensity_scale(X_f, X_o, name, thrs, scales, wavelet, expected):\n \"\"\"Test the intensity_scale.\"\"\"\n if name == \"BMSE\":\n pytest.importorskip(\"pywt\")\n\n assert_array_almost_equal(\n spatialscores.intensity_scale(X_f, X_o, name, thrs, scales, wavelet)[0][0],\n expected,\n )\n\n\nR = get_precipitation_fields(num_prev_files=3, return_raw=True)\ntest_data = [\n (R[:2], R[2:], \"FSS\", [1], [10], None),\n (R[:2], R[2:], \"BMSE\", [1], None, \"Haar\"),\n]\n\n\n@pytest.mark.parametrize(\"R1, R2, name, thrs, scales, wavelet\", test_data)\ndef test_intensity_scale_methods(R1, R2, name, thrs, scales, wavelet):\n \"\"\"\n Test the intensity_scale merge.\"\"\"\n if name == \"BMSE\":\n pytest.importorskip(\"pywt\")\n\n # expected reult\n int = spatialscores.intensity_scale_init(name, thrs, scales, wavelet)\n spatialscores.intensity_scale_accum(int, R1[0], R1[1])\n spatialscores.intensity_scale_accum(int, R2[0], R2[1])\n expected = spatialscores.intensity_scale_compute(int)[0][0]\n\n # init\n int_1 = spatialscores.intensity_scale_init(name, thrs, scales, wavelet)\n int_2 = spatialscores.intensity_scale_init(name, thrs, scales, wavelet)\n\n # accum\n spatialscores.intensity_scale_accum(int_1, R1[0], R1[1])\n spatialscores.intensity_scale_accum(int_2, R2[0], R2[1])\n\n # merge\n int = spatialscores.intensity_scale_merge(int_1, int_2)\n\n # compute\n score = spatialscores.intensity_scale_compute(int)[0][0]\n\n assert_array_almost_equal(score, expected)\n" }, { "path": "pysteps/timeseries/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Methods and models for time series analysis.\"\"\"\n" }, { "path": "pysteps/timeseries/autoregression.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.timeseries.autoregression\n=================================\n\nMethods related to autoregressive AR(p) models.\n\n.. autosummary::\n :toctree: ../generated/\n\n adjust_lag2_corrcoef1\n adjust_lag2_corrcoef2\n ar_acf\n estimate_ar_params_ols\n estimate_ar_params_ols_localized\n estimate_ar_params_yw\n estimate_ar_params_yw_localized\n estimate_var_params_ols\n estimate_var_params_ols_localized\n estimate_var_params_yw\n iterate_ar_model\n iterate_var_model\n\"\"\"\n\nimport numpy as np\nfrom scipy.special import binom\nfrom scipy import linalg as la\nfrom scipy import ndimage\n\n\ndef adjust_lag2_corrcoef1(gamma_1, gamma_2):\n \"\"\"\n A simple adjustment of lag-2 temporal autocorrelation coefficient to\n ensure that the resulting AR(2) process is stationary when the parameters\n are estimated from the Yule-Walker equations.\n\n Parameters\n ----------\n gamma_1: float\n Lag-1 temporal autocorrelation coeffient.\n gamma_2: float\n Lag-2 temporal autocorrelation coeffient.\n\n Returns\n -------\n out: float\n The adjusted lag-2 correlation coefficient.\n \"\"\"\n gamma_2 = np.maximum(gamma_2, 2 * gamma_1 * gamma_1 - 1 + 1e-10)\n gamma_2 = np.minimum(gamma_2, 1 - 1e-10)\n\n return gamma_2\n\n\ndef adjust_lag2_corrcoef2(gamma_1, gamma_2):\n \"\"\"\n A more advanced adjustment of lag-2 temporal autocorrelation coefficient\n to ensure that the resulting AR(2) process is stationary when\n the parameters are estimated from the Yule-Walker equations.\n\n Parameters\n ----------\n gamma_1: float\n Lag-1 temporal autocorrelation coeffient.\n gamma_2: float\n Lag-2 temporal autocorrelation coeffient.\n\n Returns\n -------\n out: float\n The adjusted lag-2 correlation coefficient.\n \"\"\"\n gamma_2 = np.maximum(gamma_2, 2 * gamma_1 * gamma_2 - 1)\n gamma_2 = np.maximum(\n gamma_2, (3 * gamma_1**2 - 2 + 2 * (1 - gamma_1**2) ** 1.5) / gamma_1**2\n )\n\n return gamma_2\n\n\ndef ar_acf(gamma, n=None):\n \"\"\"\n Compute theoretical autocorrelation function (ACF) from the AR(p) model\n with lag-l, l=1,2,...,p temporal autocorrelation coefficients.\n\n Parameters\n ----------\n gamma: array-like\n Array of length p containing the lag-l, l=1,2,...p, temporal\n autocorrelation coefficients.\n The correlation coefficients are assumed to be in ascending\n order with respect to time lag.\n n: int\n Desired length of ACF array. Must be greater than len(gamma).\n\n Returns\n -------\n out: array-like\n Array containing the ACF values.\n \"\"\"\n ar_order = len(gamma)\n if n == ar_order or n is None:\n return gamma\n elif n < ar_order:\n raise ValueError(\n \"n=%i, but must be larger than the order of the AR process %i\"\n % (n, ar_order)\n )\n\n phi = estimate_ar_params_yw(gamma)[:-1]\n\n acf = gamma.copy()\n for t in range(0, n - ar_order):\n # Retrieve gammas (in reverse order)\n gammas = acf[t : t + ar_order][::-1]\n # Compute next gamma\n gamma_ = np.sum(gammas * phi)\n acf.append(gamma_)\n\n return acf\n\n\ndef estimate_ar_params_ols(\n x, p, d=0, check_stationarity=True, include_constant_term=False, h=0, lam=0.0\n):\n r\"\"\"\n Estimate the parameters of an autoregressive AR(p) model\n\n :math:`x_{k+1}=c+\\phi_1 x_k+\\phi_2 x_{k-1}+\\dots+\\phi_p x_{k-p}+\\phi_{p+1}\\epsilon`\n\n by using ordinary least squares (OLS). If :math:`d\\geq 1`, the parameters\n are estimated for a d times differenced time series that is integrated back\n to the original one by summation of the differences.\n\n Parameters\n ----------\n x: array_like\n Array of shape (n,...) containing a time series of length n=p+d+h+1.\n The remaining dimensions are flattened. The rows and columns of x\n represent time steps and samples, respectively.\n p: int\n The order of the model.\n d: {0,1}\n The order of differencing to apply to the time series.\n check_stationarity: bool\n Check the stationarity of the estimated model.\n include_constant_term: bool\n Include the constant term :math:`c` to the model.\n h: int\n If h>0, the fitting is done by using a history of length h in addition\n to the minimal required number of time steps n=p+d+1.\n lam: float\n If lam>0, the regression is regularized by adding a penalty term\n (i.e. ridge regression).\n\n Returns\n -------\n out: list\n The estimated parameter matrices :math:`\\mathbf{\\Phi}_1,\\mathbf{\\Phi}_2,\n \\dots,\\mathbf{\\Phi}_{p+1}`. If include_constant_term is True, the\n constant term :math:`c` is added to the beginning of the list.\n\n Notes\n -----\n Estimation of the innovation term parameter :math:`\\phi_{p+1}` is currently\n implemented for p<=2. If p > 2, :math:`\\phi_{p+1}` is set to zero.\n \"\"\"\n n = x.shape[0]\n\n if n != p + d + h + 1:\n raise ValueError(\n \"n = %d, p = %d, d = %d, h = %d, but n = p+d+h+1 = %d required\"\n % (n, p, d, h, p + d + h + 1)\n )\n\n if len(x.shape) > 1:\n x = x.reshape((n, np.prod(x.shape[1:])))\n\n if d not in [0, 1]:\n raise ValueError(\"d = %d, but 0 or 1 required\" % d)\n\n if d == 1:\n x = np.diff(x, axis=0)\n n -= d\n\n x_lhs = x[p:, :]\n\n Z = []\n for i in range(x.shape[1]):\n for j in range(p - 1, n - 1 - h):\n z_ = np.hstack([x[j - k, i] for k in range(p)])\n if include_constant_term:\n z_ = np.hstack([[1], z_])\n Z.append(z_)\n Z = np.column_stack(Z)\n\n b = np.dot(\n np.dot(x_lhs, Z.T), np.linalg.inv(np.dot(Z, Z.T) + lam * np.eye(Z.shape[0]))\n )\n b = b.flatten()\n\n if include_constant_term:\n c = b[0]\n phi = list(b[1:])\n else:\n phi = list(b)\n\n if p == 1:\n phi_pert = np.sqrt(1.0 - phi[0] * phi[0])\n elif p == 2:\n phi_pert = np.sqrt(\n (1.0 + phi[1]) * ((1.0 - phi[1]) ** 2.0 - phi[0] ** 2.0) / (1.0 - phi[1])\n )\n else:\n phi_pert = 0.0\n\n if check_stationarity:\n if not test_ar_stationarity(phi):\n raise RuntimeError(\n \"Error in estimate_ar_params_yw: \" \"nonstationary AR(p) process\"\n )\n\n if d == 1:\n phi_out = _compute_differenced_model_params(phi, p, 1, 1)\n else:\n phi_out = phi\n\n phi_out.append(phi_pert)\n if include_constant_term:\n phi_out.insert(0, c)\n\n return phi_out\n\n\ndef estimate_ar_params_ols_localized(\n x,\n p,\n window_radius,\n d=0,\n include_constant_term=False,\n h=0,\n lam=0.0,\n window=\"gaussian\",\n):\n r\"\"\"\n Estimate the parameters of a localized AR(p) model\n\n :math:`x_{k+1,i}=c_i+\\phi_{1,i}x_{k,i}+\\phi_{2,i}x_{k-1,i}+\\dots+\\phi_{p,i}x_{k-p,i}+\\phi_{p+1,i}\\epsilon`\n\n by using ordinary least squares (OLS), where :math:`i` denote spatial\n coordinates with arbitrary dimension. If :math:`d\\geq 1`, the parameters\n are estimated for a d times differenced time series that is integrated back\n to the original one by summation of the differences.\n\n Parameters\n ----------\n x: array_like\n Array of shape (n,...) containing a time series of length n=p+d+h+1.\n The remaining dimensions are flattened. The rows and columns of x\n represent time steps and samples, respectively.\n p: int\n The order of the model.\n window_radius: float\n Radius of the moving window. If window is 'gaussian', window_radius is\n the standard deviation of the Gaussian filter. If window is 'uniform',\n the size of the window is 2*window_radius+1.\n d: {0,1}\n The order of differencing to apply to the time series.\n include_constant_term: bool\n Include the constant term :math:`c_i` to the model.\n h: int\n If h>0, the fitting is done by using a history of length h in addition\n to the minimal required number of time steps n=p+d+1.\n lam: float\n If lam>0, the regression is regularized by adding a penalty term\n (i.e. ridge regression).\n window: {\"gaussian\", \"uniform\"}\n The weight function to use for the moving window. Applicable if\n window_radius < np.inf. Defaults to 'gaussian'.\n\n Returns\n -------\n out: list\n List of length p+1 containing the AR(p) parameter fields for for the\n lag-p terms and the innovation term. The parameter fields have the same\n shape as the elements of gamma. Nan values are assigned, where the\n sample size for estimating the parameters is too small. If\n include_constant_term is True, the constant term :math:`c_i` is added\n to the beginning of the list.\n\n Notes\n -----\n Estimation of the innovation term parameter :math:`\\phi_{p+1}` is currently\n implemented for p<=2. If p > 2, :math:`\\phi_{p+1}` is set to a zero array.\n \"\"\"\n n = x.shape[0]\n\n if n != p + d + h + 1:\n raise ValueError(\n \"n = %d, p = %d, d = %d, h = %d, but n = p+d+h+1 = %d required\"\n % (n, p, d, h, p + d + h + 1)\n )\n\n if d == 1:\n x = np.diff(x, axis=0)\n n -= d\n\n if window == \"gaussian\":\n convol_filter = ndimage.gaussian_filter\n else:\n convol_filter = ndimage.uniform_filter\n\n if window == \"uniform\":\n window_size = 2 * window_radius + 1\n else:\n window_size = window_radius\n\n XZ = np.zeros(np.hstack([[p], x.shape[1:]]))\n for i in range(p):\n for j in range(h + 1):\n tmp = convol_filter(\n x[p + j, :] * x[p - 1 - i + j, :], window_size, mode=\"constant\"\n )\n XZ[i, :] += tmp\n\n if include_constant_term:\n v = 0.0\n for i in range(h + 1):\n v += convol_filter(x[p + i, :], window_size, mode=\"constant\")\n XZ = np.vstack([v[np.newaxis, :], XZ])\n\n if not include_constant_term:\n Z2 = np.zeros(np.hstack([[p, p], x.shape[1:]]))\n for i in range(p):\n for j in range(p):\n for k in range(h + 1):\n tmp = convol_filter(\n x[p - 1 - i + k, :] * x[p - 1 - j + k, :],\n window_size,\n mode=\"constant\",\n )\n Z2[i, j, :] += tmp\n else:\n Z2 = np.zeros(np.hstack([[p + 1, p + 1], x.shape[1:]]))\n Z2[0, 0, :] = convol_filter(np.ones(x.shape[1:]), window_size, mode=\"constant\")\n for i in range(p):\n for j in range(h + 1):\n tmp = convol_filter(x[p - 1 - i + j, :], window_size, mode=\"constant\")\n Z2[0, i + 1, :] += tmp\n Z2[i + 1, 0, :] += tmp\n for i in range(p):\n for j in range(p):\n for k in range(h + 1):\n tmp = convol_filter(\n x[p - 1 - i + k, :] * x[p - 1 - j + k, :],\n window_size,\n mode=\"constant\",\n )\n Z2[i + 1, j + 1, :] += tmp\n\n m = np.prod(x.shape[1:])\n phi = np.empty(np.hstack([[p], m]))\n if include_constant_term:\n c = np.empty(m)\n XZ = XZ.reshape(np.hstack([[XZ.shape[0]], m]))\n Z2 = Z2.reshape(np.hstack([[Z2.shape[0], Z2.shape[1]], m]))\n\n for i in range(m):\n try:\n b = np.dot(XZ[:, i], np.linalg.inv(Z2[:, :, i] + lam * np.eye(Z2.shape[0])))\n if not include_constant_term:\n phi[:, i] = b\n else:\n phi[:, i] = b[1:]\n c[i] = b[0]\n except np.linalg.LinAlgError:\n phi[:, i] = np.nan\n if include_constant_term:\n c[i] = np.nan\n\n if p == 1:\n phi_pert = np.sqrt(1.0 - phi[0, :] * phi[0, :])\n elif p == 2:\n phi_pert = np.sqrt(\n (1.0 + phi[1, :])\n * ((1.0 - phi[1, :]) ** 2.0 - phi[0, :] ** 2.0)\n / (1.0 - phi[1, :])\n )\n else:\n phi_pert = np.zeros(m)\n\n phi = list(phi.reshape(np.hstack([[phi.shape[0]], x.shape[1:]])))\n if d == 1:\n phi = _compute_differenced_model_params(phi, p, 1, 1)\n phi.append(phi_pert.reshape(x.shape[1:]))\n if include_constant_term:\n phi.insert(0, c.reshape(x.shape[1:]))\n\n return phi\n\n\ndef estimate_ar_params_yw(gamma, d=0, check_stationarity=True):\n r\"\"\"\n Estimate the parameters of an AR(p) model\n\n :math:`x_{k+1}=\\phi_1 x_k+\\phi_2 x_{k-1}+\\dots+\\phi_p x_{k-p}+\\phi_{p+1}\\epsilon`\n\n from the Yule-Walker equations using the given set of autocorrelation\n coefficients.\n\n Parameters\n ----------\n gamma: array_like\n Array of length p containing the lag-l temporal autocorrelation\n coefficients for l=1,2,...p. The correlation coefficients are assumed\n to be in ascending order with respect to time lag.\n d: {0,1}\n The order of differencing. If d=1, the correlation coefficients gamma\n are assumed to be computed from the differenced time series, which is\n also done for the resulting parameter estimates.\n check_stationarity: bool\n If True, the stationarity of the resulting VAR(p) process is tested. An\n exception is thrown if the process is not stationary.\n\n Returns\n -------\n out: ndarray\n Array of length p+1 containing the AR(p) parameters for for the\n lag-p terms and the innovation term.\n\n Notes\n -----\n To estimate the parameters of an integrated ARI(p,d) model, compute the\n correlation coefficients gamma by calling\n :py:func:`pysteps.timeseries.correlation.temporal_autocorrelation` with d>0.\n \"\"\"\n if d not in [0, 1]:\n raise ValueError(\"d = %d, but 0 or 1 required\" % d)\n\n p = len(gamma)\n\n g = np.hstack([[1.0], gamma])\n G = []\n for j in range(p):\n G.append(np.roll(g[:-1], j))\n G = np.array(G)\n phi = np.linalg.solve(G, g[1:].flatten())\n\n # Check that the absolute values of the roots of the characteristic\n # polynomial are less than one.\n # Otherwise the AR(p) model is not stationary.\n if check_stationarity:\n if not test_ar_stationarity(phi):\n raise RuntimeError(\n \"Error in estimate_ar_params_yw: \" \"nonstationary AR(p) process\"\n )\n\n c = 1.0\n for j in range(p):\n c -= gamma[j] * phi[j]\n phi_pert = np.sqrt(c)\n\n # If the expression inside the square root is negative, phi_pert cannot\n # be computed and it is set to zero instead.\n if not np.isfinite(phi_pert):\n phi_pert = 0.0\n\n if d == 1:\n phi = _compute_differenced_model_params(phi, p, 1, 1)\n\n phi_out = np.empty(len(phi) + 1)\n phi_out[: len(phi)] = phi\n phi_out[-1] = phi_pert\n\n return phi_out\n\n\ndef estimate_ar_params_yw_localized(gamma, d=0):\n r\"\"\"\n Estimate the parameters of a localized AR(p) model\n\n :math:`x_{k+1,i}=\\phi_{1,i}x_{k,i}+\\phi_{2,i}x_{k-1,i}+\\dots+\\phi_{p,i}x_{k-p,i}+\\phi_{p+1}\\epsilon`\n\n from the Yule-Walker equations using the given set of autocorrelation\n coefficients :math`\\gamma_{l,i}`, where :math`l` denotes time lag and\n :math:`i` denote spatial coordinates with arbitrary dimension.\n\n Parameters\n ----------\n gamma: array_like\n A list containing the lag-l temporal autocorrelation coefficient fields\n for l=1,2,...p. The correlation coefficients are assumed to be in\n ascending order with respect to time lag.\n d: {0,1}\n The order of differencing. If d=1, the correlation coefficients gamma\n are assumed to be computed from the differenced time series, which is\n also done for the resulting parameter estimates.\n\n Returns\n -------\n out: list\n List of length p+1 containing the AR(p) parameter fields for for the\n lag-p terms and the innovation term. The parameter fields have the same\n shape as the elements of gamma.\n\n Notes\n -----\n To estimate the parameters of an integrated ARI(p,d) model, compute the\n correlation coefficients gamma by calling\n :py:func:`pysteps.timeseries.correlation.temporal_autocorrelation` with d>0\n and window_radius0, the fitting is done by using a history of length h in addition\n to the minimal required number of time steps n=p+d+1.\n lam: float\n If lam>0, the regression is regularized by adding a penalty term\n (i.e. ridge regression).\n\n Returns\n -------\n out: list\n The estimated parameter matrices :math:`\\mathbf{\\Phi}_1,\\mathbf{\\Phi}_2,\n \\dots,\\mathbf{\\Phi}_{p+1}`. If include_constant_term is True, the\n constant term :math:`\\mathbf{c}` is added to the beginning of the list.\n\n Notes\n -----\n Estimation of the innovation parameter :math:`\\mathbf{\\Phi}_{p+1}` is not\n currently implemented, and it is set to a zero matrix.\n \"\"\"\n q = x.shape[1]\n n = x.shape[0]\n\n if n != p + d + h + 1:\n raise ValueError(\n \"n = %d, p = %d, d = %d, h = %d, but n = p+d+h+1 = %d required\"\n % (n, p, d, h, p + d + h + 1)\n )\n\n if d not in [0, 1]:\n raise ValueError(\"d = %d, but 0 or 1 required\" % d)\n\n if d == 1:\n x = np.diff(x, axis=0)\n n -= d\n\n x = x.reshape((n, q, np.prod(x.shape[2:])))\n\n X = []\n for i in range(x.shape[2]):\n for j in range(p + h, n):\n x_ = x[j, :, i]\n X.append(x_.reshape((q, 1)))\n X = np.hstack(X)\n\n Z = []\n for i in range(x.shape[2]):\n for j in range(p - 1, n - 1 - h):\n z_ = np.vstack([x[j - k, :, i].reshape((q, 1)) for k in range(p)])\n if include_constant_term:\n z_ = np.vstack([[1], z_])\n Z.append(z_)\n Z = np.column_stack(Z)\n\n B = np.dot(np.dot(X, Z.T), np.linalg.inv(np.dot(Z, Z.T) + lam * np.eye(Z.shape[0])))\n\n phi = []\n if include_constant_term:\n c = B[:, 0]\n for i in range(p):\n phi.append(B[:, i * q + 1 : (i + 1) * q + 1])\n else:\n for i in range(p):\n phi.append(B[:, i * q : (i + 1) * q])\n\n if check_stationarity:\n M = np.zeros((p * q, p * q))\n\n for i in range(p):\n M[0:q, i * q : (i + 1) * q] = phi[i]\n for i in range(1, p):\n M[i * q : (i + 1) * q, (i - 1) * q : i * q] = np.eye(q, q)\n r, v = np.linalg.eig(M)\n\n if np.any(np.abs(r) > 0.999):\n raise RuntimeError(\n \"Error in estimate_var_params_ols: \" \"nonstationary VAR(p) process\"\n )\n\n if d == 1:\n phi = _compute_differenced_model_params(phi, p, q, 1)\n\n if include_constant_term:\n phi.insert(0, c)\n phi.append(np.zeros((q, q)))\n\n return phi\n\n\ndef estimate_var_params_ols_localized(\n x,\n p,\n window_radius,\n d=0,\n include_constant_term=False,\n h=0,\n lam=0.0,\n window=\"gaussian\",\n):\n r\"\"\"\n Estimate the parameters of a vector autoregressive VAR(p) model\n\n :math:`\\mathbf{x}_{k+1,i}=\\mathbf{c}_i+\\mathbf{\\Phi}_{1,i}\\mathbf{x}_{k,i}+\n \\mathbf{\\Phi}_{2,i}\\mathbf{x}_{k-1,i}+\\dots+\\mathbf{\\Phi}_{p,i}\n \\mathbf{x}_{k-p,i}+\\mathbf{\\Phi}_{p+1,i}\\mathbf{\\epsilon}`\n\n by using ordinary least squares (OLS), where :math:`i` denote spatial\n coordinates with arbitrary dimension. If :math:`d\\geq 1`, the parameters\n are estimated for a d times differenced time series that is integrated back\n to the original one by summation of the differences.\n\n Parameters\n ----------\n x: array_like\n Array of shape (n, q, :) containing a time series of length n=p+d+h+1\n with q-dimensional variables. The remaining dimensions are flattened.\n The remaining dimensions starting from the third one represent the\n samples.\n p: int\n The order of the model.\n window_radius: float\n Radius of the moving window. If window is 'gaussian', window_radius is\n the standard deviation of the Gaussian filter. If window is 'uniform',\n the size of the window is 2*window_radius+1.\n d: {0,1}\n The order of differencing to apply to the time series.\n include_constant_term: bool\n Include the constant term :math:`\\mathbf{c}` to the model.\n h: int\n If h>0, the fitting is done by using a history of length h in addition\n to the minimal required number of time steps n=p+d+1.\n lam: float\n If lam>0, the regression is regularized by adding a penalty term\n (i.e. ridge regression).\n window: {\"gaussian\", \"uniform\"}\n The weight function to use for the moving window. Applicable if\n window_radius < np.inf. Defaults to 'gaussian'.\n\n Returns\n -------\n out: list\n The estimated parameter matrices :math:`\\mathbf{\\Phi}_{1,i},\n \\mathbf{\\Phi}_{2,i},\\dots,\\mathbf{\\Phi}_{p+1,i}`. If\n include_constant_term is True, the constant term :math:`\\mathbf{c}_i` is\n added to the beginning of the list. Each element of the list is a matrix\n of shape (x.shape[2:], q, q).\n\n Notes\n -----\n Estimation of the innovation parameter :math:`\\mathbf{\\Phi}_{p+1}` is not\n currently implemented, and it is set to a zero matrix.\n \"\"\"\n q = x.shape[1]\n n = x.shape[0]\n\n if n != p + d + h + 1:\n raise ValueError(\n \"n = %d, p = %d, d = %d, h = %d, but n = p+d+h+1 = %d required\"\n % (n, p, d, h, p + d + h + 1)\n )\n\n if d == 1:\n x = np.diff(x, axis=0)\n n -= d\n\n if window == \"gaussian\":\n convol_filter = ndimage.gaussian_filter\n else:\n convol_filter = ndimage.uniform_filter\n\n if window == \"uniform\":\n window_size = 2 * window_radius + 1\n else:\n window_size = window_radius\n\n XZ = np.zeros(np.hstack([[q, p * q], x.shape[2:]]))\n for i in range(q):\n for k in range(p):\n for j in range(q):\n for l in range(h + 1):\n tmp = convol_filter(\n x[p + l, i, :] * x[p - 1 - k + l, j, :],\n window_size,\n mode=\"constant\",\n )\n XZ[i, k * q + j, :] += tmp\n\n if include_constant_term:\n v = np.zeros(np.hstack([[q], x.shape[2:]]))\n for i in range(q):\n for j in range(h + 1):\n v[i, :] += convol_filter(x[p + j, i, :], window_size, mode=\"constant\")\n XZ = np.hstack([v[:, np.newaxis, :], XZ])\n\n if not include_constant_term:\n Z2 = np.zeros(np.hstack([[p * q, p * q], x.shape[2:]]))\n for i in range(p):\n for j in range(q):\n for k in range(p):\n for l in range(q):\n for m in range(h + 1):\n tmp = convol_filter(\n x[p - 1 - i + m, j, :] * x[p - 1 - k + m, l, :],\n window_size,\n mode=\"constant\",\n )\n Z2[i * q + j, k * q + l, :] += tmp\n else:\n Z2 = np.zeros(np.hstack([[p * q + 1, p * q + 1], x.shape[2:]]))\n Z2[0, 0, :] = convol_filter(np.ones(x.shape[2:]), window_size, mode=\"constant\")\n for i in range(p):\n for j in range(q):\n for k in range(h + 1):\n tmp = convol_filter(\n x[p - 1 - i + k, j, :], window_size, mode=\"constant\"\n )\n Z2[0, i * q + j + 1, :] += tmp\n Z2[i * q + j + 1, 0, :] += tmp\n for i in range(p):\n for j in range(q):\n for k in range(p):\n for l in range(q):\n for m in range(h + 1):\n tmp = convol_filter(\n x[p - 1 - i + m, j, :] * x[p - 1 - k + m, l, :],\n window_size,\n mode=\"constant\",\n )\n Z2[i * q + j + 1, k * q + l + 1, :] += tmp\n\n m = np.prod(x.shape[2:])\n if include_constant_term:\n c = np.empty((m, q))\n XZ = XZ.reshape((XZ.shape[0], XZ.shape[1], m))\n Z2 = Z2.reshape((Z2.shape[0], Z2.shape[1], m))\n\n phi = np.empty((p, m, q, q))\n for i in range(m):\n try:\n B = np.dot(\n XZ[:, :, i], np.linalg.inv(Z2[:, :, i] + lam * np.eye(Z2.shape[0]))\n )\n for k in range(p):\n if not include_constant_term:\n phi[k, i, :, :] = B[:, k * q : (k + 1) * q]\n else:\n phi[k, i, :, :] = B[:, k * q + 1 : (k + 1) * q + 1]\n if include_constant_term:\n c[i, :] = B[:, 0]\n except np.linalg.LinAlgError:\n phi[:, i, :, :] = np.nan\n if include_constant_term:\n c[i, :] = np.nan\n\n phi_out = [\n phi[i].reshape(np.hstack([x.shape[2:], [q, q]])) for i in range(len(phi))\n ]\n if d == 1:\n phi_out = _compute_differenced_model_params(phi_out, p, q, 1)\n\n phi_out.append(np.zeros(phi_out[0].shape))\n if include_constant_term:\n phi_out.insert(0, c.reshape(np.hstack([x.shape[2:], [q]])))\n\n return phi_out\n\n\ndef estimate_var_params_yw(gamma, d=0, check_stationarity=True):\n r\"\"\"\n Estimate the parameters of a VAR(p) model\n\n :math:`\\mathbf{x}_{k+1}=\\mathbf{\\Phi}_1\\mathbf{x}_k+\n \\mathbf{\\Phi}_2\\mathbf{x}_{k-1}+\\dots+\\mathbf{\\Phi}_p\\mathbf{x}_{k-p}+\n \\mathbf{\\Phi}_{p+1}\\mathbf{\\epsilon}`\n\n from the Yule-Walker equations using the given correlation matrices\n :math:`\\mathbf{\\Gamma}_0,\\mathbf{\\Gamma}_1,\\dots,\\mathbf{\\Gamma}_n`, where\n n=p.\n\n Parameters\n ----------\n gamma: list\n List of correlation matrices\n :math:`\\mathbf{\\Gamma}_0,\\mathbf{\\Gamma}_1,\\dots,\\mathbf{\\Gamma}_n`.\n To obtain these matrices, use\n :py:func:`pysteps.timeseries.correlation.temporal_autocorrelation_multivariate`\n with window_radius=np.inf.\n d: {0,1}\n The order of differencing. If d=1, the correlation coefficients gamma\n are assumed to be computed from the differenced time series, which is\n also done for the resulting parameter estimates.\n check_stationarity: bool\n If True, the stationarity of the resulting VAR(p) process is tested. An\n exception is thrown if the process is not stationary.\n\n Returns\n -------\n out: list\n List of VAR(p) coefficient matrices :math:`\\mathbf{\\Phi}_1,\n \\mathbf{\\Phi}_2,\\dots\\mathbf{\\Phi}_{p+1}`, where the last matrix\n corresponds to the innovation term.\n\n Notes\n -----\n To estimate the parameters of an integrated VARI(p,d) model, compute the\n correlation coefficients gamma by calling\n :py:func:`pysteps.timeseries.correlation.temporal_autocorrelation_multivariate`\n with d>0. Estimation of the innovation parameter :math:`\\mathbf{\\Phi}_{p+1}`\n is not currently implemented, and it is set to a zero matrix.\n \"\"\"\n p = len(gamma) - 1\n q = gamma[0].shape[0]\n\n for i in range(len(gamma)):\n if gamma[i].shape[0] != q or gamma[i].shape[1] != q:\n raise ValueError(\n \"dimension mismatch: gamma[%d].shape=%s, but (%d,%d) expected\"\n % (i, str(gamma[i].shape), q, q)\n )\n\n if d not in [0, 1]:\n raise ValueError(\"d = %d, but 0 or 1 required\" % d)\n\n a = np.empty((p * q, p * q))\n for i in range(p):\n for j in range(p):\n a_tmp = gamma[abs(i - j)]\n if i > j:\n a_tmp = a_tmp.T\n a[i * q : (i + 1) * q, j * q : (j + 1) * q] = a_tmp\n\n b = np.vstack([gamma[i].T for i in range(1, p + 1)])\n x = np.linalg.solve(a, b)\n\n phi = []\n for i in range(p):\n phi.append(x[i * q : (i + 1) * q, :])\n\n if check_stationarity:\n if not test_var_stationarity(phi):\n raise RuntimeError(\n \"Error in estimate_var_params_yw: \" \"nonstationary VAR(p) process\"\n )\n\n if d == 1:\n phi = _compute_differenced_model_params(phi, p, q, 1)\n\n phi.append(np.zeros(phi[0].shape))\n\n return phi\n\n\ndef estimate_var_params_yw_localized(gamma, d=0):\n r\"\"\"\n Estimate the parameters of a vector autoregressive VAR(p) model\n\n :math:`\\mathbf{x}_{k+1,i}=\\mathbf{\\Phi}_{1,i}\\mathbf{x}_{k,i}+\n \\mathbf{\\Phi}_{2,i}\\mathbf{x}_{k-1,i}+\\dots+\\mathbf{\\Phi}_{p,i}\n \\mathbf{x}_{k-p,i}+\\mathbf{\\Phi}_{p+1,i}\\mathbf{\\epsilon}`\n\n from the Yule-Walker equations by using the given correlation matrices,\n where :math:`i` denote spatial coordinates with arbitrary dimension.\n\n Parameters\n ----------\n gamma: list\n List of correlation matrices\n :math:`\\mathbf{\\Gamma}_0,\\mathbf{\\Gamma}_1,\\dots,\\mathbf{\\Gamma}_n`.\n To obtain these matrices, use\n :py:func:`pysteps.timeseries.correlation.temporal_autocorrelation_multivariate`\n with window_radius0 and window_radius j:\n a_tmp = a_tmp.T\n a[i * q : (i + 1) * q, j * q : (j + 1) * q] = a_tmp\n\n b = np.vstack([gamma_1d[i][k, :].T for i in range(1, p + 1)])\n x = np.linalg.solve(a, b)\n\n for i in range(p):\n phi_out[i][k, :, :] = x[i * q : (i + 1) * q, :]\n\n for i in range(len(phi_out)):\n phi_out[i] = phi_out[i].reshape(np.hstack([gamma[0].shape[:-2], [q, q]]))\n if d == 1:\n phi_out = _compute_differenced_model_params(phi_out, p, 1, 1)\n phi_out.append(np.zeros(gamma[0].shape))\n\n return phi_out\n\n\ndef iterate_ar_model(x, phi, eps=None):\n r\"\"\"Apply an AR(p) model\n\n :math:`x_{k+1}=\\phi_1 x_k+\\phi_2 x_{k-1}+\\dots+\\phi_p x_{k-p}+\\phi_{p+1}\\epsilon`\n\n to a time series :math:`x_k`.\n\n Parameters\n ----------\n x: array_like\n Array of shape (n,...), n>=p, containing a time series of a input variable\n x. The elements of x along the first dimension are assumed to be in\n ascending order by time, and the time intervals are assumed to be regular.\n phi: list\n List or array of length p+1 specifying the parameters of the AR(p) model.\n The parameters are in ascending order by increasing time lag, and the\n last element is the parameter corresponding to the innovation term eps.\n eps: array_like\n Optional innovation term for the AR(p) process. The shape of eps is\n expected to be a scalar or x.shape[1:] if len(x.shape)>1. If eps is\n None, the innovation term is not added.\n \"\"\"\n if x.shape[0] < len(phi) - 1:\n raise ValueError(\n \"dimension mismatch between x and phi: x.shape[0]=%d, len(phi)=%d\"\n % (x.shape[0], len(phi))\n )\n\n if len(x.shape) == 1:\n x_simple_shape = True\n x = x[:, np.newaxis]\n else:\n x_simple_shape = False\n\n if eps is not None and eps.shape != x.shape[1:]:\n raise ValueError(\n \"dimension mismatch between x and eps: x[1:].shape=%s, eps.shape=%s\"\n % (str(x[1:].shape), str(eps.shape))\n )\n\n x_new = 0.0\n\n p = len(phi) - 1\n\n for i in range(p):\n x_new += phi[i] * x[-(i + 1), :]\n\n if eps is not None:\n x_new += phi[-1] * eps\n\n if x_simple_shape:\n return np.hstack([x[1:], [x_new]])\n else:\n return np.concatenate([x[1:, :], x_new[np.newaxis, :]])\n\n\ndef iterate_var_model(x, phi, eps=None):\n r\"\"\"Apply a VAR(p) model\n\n :math:`\\mathbf{x}_{k+1}=\\mathbf{\\Phi}_1\\mathbf{x}_k+\\mathbf{\\Phi}_2\n \\mathbf{x}_{k-1}+\\dots+\\mathbf{\\Phi}_p\\mathbf{x}_{k-p}+\n \\mathbf{\\Phi}_{p+1}\\mathbf{\\epsilon}`\n\n to a q-variate time series :math:`\\mathbf{x}_k`.\n\n Parameters\n ----------\n x: array_like\n Array of shape (n,q,...), n>=p, containing a q-variate time series of a\n input variable x. The elements of x along the first dimension are\n assumed to be in ascending order by time, and the time intervals are\n assumed to be regular.\n phi: list\n List of parameter matrices :math:`\\mathbf{\\Phi}_1,\\mathbf{\\Phi}_2,\\dots,\n \\mathbf{\\Phi}_{p+1}`.\n eps: array_like\n Optional innovation term for the AR(p) process. The shape of eps is\n expected to be (x.shape[1],) or (x.shape[1],x.shape[2:]) if\n len(x.shape)>2. If eps is None, the innovation term is not added.\n \"\"\"\n if x.shape[0] < len(phi) - 1:\n raise ValueError(\n \"dimension mismatch between x and phi: x.shape[0]=%d, len(phi)=%d\"\n % (x.shape[1], len(phi))\n )\n\n phi_shape = phi[0].shape\n if phi_shape[-1] != phi_shape[-2]:\n raise ValueError(\n \"phi[0].shape = %s, but the last two dimensions are expected to be equal\"\n % str(phi_shape)\n )\n for i in range(1, len(phi)):\n if phi[i].shape != phi_shape:\n raise ValueError(\"dimension mismatch between parameter matrices phi\")\n\n if len(x.shape) == 2:\n x_simple_shape = True\n x = x[:, :, np.newaxis]\n else:\n x_simple_shape = False\n\n x_new = np.zeros(x.shape[1:])\n p = len(phi) - 1\n\n for l in range(p):\n x_new += np.einsum(\"...ij,j...->i...\", phi[l], x[-(l + 1), :])\n\n if eps is not None:\n x_new += np.dot(np.dot(phi[-1], phi[-1]), eps)\n\n if x_simple_shape:\n return np.vstack([x[1:, :, 0], x_new[:, 0]])\n else:\n x_new = x_new.reshape(x.shape[1:])\n return np.concatenate([x[1:, :], x_new[np.newaxis, :, :]], axis=0)\n\n\ndef test_ar_stationarity(phi):\n r\"\"\"\n Test stationarity of an AR(p) process. That is, test that the roots of\n the equation :math:`x^p-\\phi_1*x^{p-1}-\\dots-\\phi_p` lie inside the unit\n circle.\n\n Parameters\n ----------\n phi: list\n List of AR(p) parameters :math:`\\phi_1,\\phi_2,\\dots,\\phi_p`.\n\n Returns\n -------\n out: bool\n True/False if the process is/is not stationary.\n \"\"\"\n r = np.array(\n [\n np.abs(r_)\n for r_ in np.roots([1.0 if i == 0 else -phi[i] for i in range(len(phi))])\n ]\n )\n\n return False if np.any(r >= 1) else True\n\n\ndef test_var_stationarity(phi):\n r\"\"\"\n Test stationarity of an AR(p) process. That is, test that the moduli of\n the eigenvalues of the companion matrix lie inside the unit circle.\n\n Parameters\n ----------\n phi: list\n List of VAR(p) parameter matrices :math:`\\mathbf{\\Phi}_1,\\mathbf{\\Phi}_2,\n \\dots,\\mathbf{\\Phi}_p`.\n\n Returns\n -------\n out: bool\n True/False if the process is/is not stationary.\n \"\"\"\n q = phi[0].shape\n for i in range(1, len(phi)):\n if phi[i].shape != q:\n raise ValueError(\"dimension mismatch between parameter matrices phi\")\n\n p = len(phi)\n q = phi[0].shape[0]\n\n M = np.zeros((p * q, p * q))\n\n for i in range(p):\n M[0:q, i * q : (i + 1) * q] = phi[i]\n for i in range(1, p):\n M[i * q : (i + 1) * q, (i - 1) * q : i * q] = np.eye(q, q)\n r = np.linalg.eig(M)[0]\n\n return False if np.any(np.abs(r) >= 1) else True\n\n\ndef _compute_differenced_model_params(phi, p, q, d):\n phi_out = []\n for i in range(p + d):\n if q > 1:\n if len(phi[0].shape) == 2:\n phi_out.append(np.zeros((q, q)))\n else:\n phi_out.append(np.zeros(phi[0].shape))\n else:\n phi_out.append(0.0)\n\n for i in range(1, d + 1):\n if q > 1:\n phi_out[i - 1] -= binom(d, i) * (-1) ** i * np.eye(q)\n else:\n phi_out[i - 1] -= binom(d, i) * (-1) ** i\n for i in range(1, p + 1):\n phi_out[i - 1] += phi[i - 1]\n for i in range(1, p + 1):\n for j in range(1, d + 1):\n phi_out[i + j - 1] += phi[i - 1] * binom(d, j) * (-1) ** j\n\n return phi_out\n" }, { "path": "pysteps/timeseries/correlation.py", "content": "# -*- coding: utf-8 -*-\nr\"\"\"\npysteps.timeseries.correlation\n==============================\n\nMethods for computing spatial and temporal correlation of time series of\ntwo-dimensional fields.\n\n.. autosummary::\n :toctree: ../generated/\n\n temporal_autocorrelation\n temporal_autocorrelation_multivariate\n\"\"\"\n\nimport numpy as np\nfrom scipy import ndimage\nfrom pysteps.utils import spectral\n\n\ndef temporal_autocorrelation(\n x,\n d=0,\n domain=\"spatial\",\n x_shape=None,\n mask=None,\n use_full_fft=False,\n window=\"gaussian\",\n window_radius=np.inf,\n):\n r\"\"\"\n Compute lag-l temporal autocorrelation coefficients\n :math:`\\gamma_l=\\mbox{corr}(x(t),x(t-l))`, :math:`l=1,2,\\dots,n-1`,\n from a time series :math:`x_1,x_2,\\dots,x_n`. If a multivariate time series\n is given, each element of :math:`x_i` is treated as one sample from the\n process generating the time series. Use\n :py:func:`temporal_autocorrelation_multivariate` if cross-correlations\n between different elements of the time series are desired.\n\n Parameters\n ----------\n x: array_like\n Array of shape (n, ...), where each row contains one sample from the\n time series :math:`x_i`. The inputs are assumed to be in increasing\n order with respect to time, and the time step is assumed to be regular.\n All inputs are required to have finite values. The remaining dimensions\n after the first one are flattened before computing the correlation\n coefficients.\n d: {0,1}\n The order of differencing. If d=1, the input time series is differenced\n before computing the correlation coefficients. In this case, a time\n series of length n+1 is needed for computing the n-1 coefficients.\n domain: {\"spatial\", \"spectral\"}\n The domain of the time series x. If domain is \"spectral\", the elements\n of x are assumed to represent the FFTs of the original elements.\n x_shape: tuple\n The shape of the original arrays in the spatial domain before applying\n the FFT. Required if domain is \"spectral\".\n mask: array_like\n Optional mask to use for computing the correlation coefficients. Input\n elements with mask==False are excluded from the computations. The shape\n of the mask is expected to be x.shape[1:]. Applicable if domain is\n \"spatial\".\n use_full_fft: bool\n If True, x represents the full FFTs of the original arrays. Otherwise,\n the elements of x are assumed to contain only the symmetric part, i.e.\n in the format returned by numpy.fft.rfft2. Applicable if domain is\n 'spectral'. Defaults to False.\n window: {\"gaussian\", \"uniform\"}\n The weight function to use for the moving window. Applicable if\n window_radius < np.inf. Defaults to 'gaussian'.\n window_radius: float\n If window_radius < np.inf, the correlation coefficients are computed in\n a moving window. Defaults to np.inf (i.e. the coefficients are computed\n over the whole domain). If window is 'gaussian', window_radius is the\n standard deviation of the Gaussian filter. If window is 'uniform', the\n size of the window is 2*window_radius+1.\n\n Returns\n -------\n out: list\n List of length n-1 containing the temporal autocorrelation coefficients\n :math:`\\gamma_i` for time lags :math:`l=1,2,...,n-1`. If\n window_radius= 2\")\n if len(x.shape) != 3 and domain == \"spectral\":\n raise NotImplementedError(\n \"len(x.shape[1:]) = %d, but with domain == 'spectral', this function has only been implemented for two-dimensional fields\"\n % len(x.shape[1:])\n )\n if mask is not None and mask.shape != x.shape[1:]:\n raise ValueError(\n \"dimension mismatch between x and mask: x.shape[1:]=%s, mask.shape=%s\"\n % (str(x.shape[1:]), str(mask.shape))\n )\n if np.any(~np.isfinite(x)):\n raise ValueError(\"x contains non-finite values\")\n\n if d == 1:\n x = np.diff(x, axis=0)\n\n if domain == \"spatial\" and mask is None:\n mask = np.ones(x.shape[1:], dtype=bool)\n\n gamma = []\n for k in range(x.shape[0] - 1):\n if domain == \"spatial\":\n if window_radius == np.inf:\n cc = np.corrcoef(x[-1, :][mask], x[-(k + 2), :][mask])[0, 1]\n else:\n cc = _moving_window_corrcoef(\n x[-1, :], x[-(k + 2), :], window_radius, mask=mask\n )\n else:\n cc = spectral.corrcoef(\n x[-1, :, :], x[-(k + 2), :], x_shape, use_full_fft=use_full_fft\n )\n gamma.append(cc)\n\n return gamma\n\n\ndef temporal_autocorrelation_multivariate(\n x, d=0, mask=None, window=\"gaussian\", window_radius=np.inf\n):\n r\"\"\"\n For a :math:`q`-variate time series\n :math:`\\mathbf{x}_1,\\mathbf{x}_2,\\dots,\\mathbf{x}_n`, compute the lag-l\n correlation matrices :math:`\\mathbf{\\Gamma}_l`, where\n :math:`\\Gamma_{l,i,j}=\\gamma_{l,i,j}` and\n :math:`\\gamma_{l,i,j}=\\mbox{corr}(x_i(t),x_j(t-l))` for\n :math:`i,j=1,2,\\dots,q` and :math:`l=0,1,\\dots,n-1`.\n\n Parameters\n ----------\n x: array_like\n Array of shape (n, q, ...) containing the time series :math:`\\mathbf{x}_i`.\n The inputs are assumed to be in increasing order with respect to time,\n and the time step is assumed to be regular. All inputs are required to\n have finite values. The remaining dimensions after the second one are\n flattened before computing the correlation coefficients.\n d: {0,1}\n The order of differencing. If d=1, the input time series is differenced\n before computing the correlation coefficients. In this case, a time\n series of length n+1 is needed for computing the n-1 coefficients.\n mask: array_like\n Optional mask to use for computing the correlation coefficients. Input\n elements with mask==False are excluded from the computations. The shape\n of the mask is expected to be x.shape[2:].\n window: {\"gaussian\", \"uniform\"}\n The weight function to use for the moving window. Applicable if\n window_radius < np.inf. Defaults to 'gaussian'.\n window_radius: float\n If window_radius < np.inf, the correlation coefficients are computed in\n a moving window. Defaults to np.inf (i.e. the correlations are computed\n over the whole domain). If window is 'gaussian', window_radius is the\n standard deviation of the Gaussian filter. If window is 'uniform', the\n size of the window is 2*window_radius+1.\n\n Returns\n -------\n out: list\n List of correlation matrices :math:`\\Gamma_0,\\Gamma_1,\\dots,\\Gamma_{n-1}`\n of shape (q,q). If window_radius= 3\")\n if mask is not None and mask.shape != x.shape[2:]:\n raise ValueError(\n \"dimension mismatch between x and mask: x.shape[2:]=%s, mask.shape=%s\"\n % (str(x.shape[2:]), str(mask.shape))\n )\n if np.any(~np.isfinite(x)):\n raise ValueError(\"x contains non-finite values\")\n\n if d == 1:\n x = np.diff(x, axis=0)\n\n p = x.shape[0] - 1\n q = x.shape[1]\n\n gamma = []\n for k in range(p + 1):\n if window_radius == np.inf:\n gamma_k = np.empty((q, q))\n else:\n gamma_k = np.empty(np.hstack([x.shape[2:], [q, q]]))\n for i in range(q):\n x_i = x[-1, i, :]\n for j in range(q):\n x_j = x[-(k + 1), j, :]\n if window_radius == np.inf:\n gamma_k[i, j] = np.corrcoef(x_i.flatten(), x_j.flatten())[0, 1]\n else:\n gamma_k[..., i, j] = _moving_window_corrcoef(\n x_i, x_j, window_radius, window=window, mask=mask\n )\n\n gamma.append(gamma_k)\n\n return gamma\n\n\ndef _moving_window_corrcoef(x, y, window_radius, window=\"gaussian\", mask=None):\n if window not in [\"gaussian\", \"uniform\"]:\n raise ValueError(\n \"unknown window type %s, the available options are 'gaussian' and 'uniform'\"\n % window\n )\n\n if mask is None:\n mask = np.ones(x.shape)\n else:\n x = x.copy()\n x[~mask] = 0.0\n y = y.copy()\n y[~mask] = 0.0\n mask = mask.astype(float)\n\n if window == \"gaussian\":\n convol_filter = ndimage.gaussian_filter\n else:\n convol_filter = ndimage.uniform_filter\n\n if window == \"uniform\":\n window_size = 2 * window_radius + 1\n else:\n window_size = window_radius\n\n n = convol_filter(mask, window_size, mode=\"constant\") * window_size**2\n\n sx = convol_filter(x, window_size, mode=\"constant\") * window_size**2\n sy = convol_filter(y, window_size, mode=\"constant\") * window_size**2\n\n ssx = convol_filter(x**2, window_size, mode=\"constant\") * window_size**2\n ssy = convol_filter(y**2, window_size, mode=\"constant\") * window_size**2\n sxy = convol_filter(x * y, window_size, mode=\"constant\") * window_size**2\n\n mux = sx / n\n muy = sy / n\n\n stdx = np.sqrt(ssx - 2 * mux * sx + n * mux**2)\n stdy = np.sqrt(ssy - 2 * muy * sy + n * muy**2)\n cov = sxy - muy * sx - mux * sy + n * mux * muy\n\n mask = np.logical_and(stdx > 1e-8, stdy > 1e-8)\n mask = np.logical_and(mask, stdx * stdy > 1e-8)\n mask = np.logical_and(mask, n >= 3)\n corr = np.empty(x.shape)\n corr[mask] = cov[mask] / (stdx[mask] * stdy[mask])\n corr[~mask] = np.nan\n\n return corr\n" }, { "path": "pysteps/tracking/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Implementations of feature tracking methods.\"\"\"\n\nfrom pysteps.tracking.interface import get_method\n" }, { "path": "pysteps/tracking/interface.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.tracking.interface\n===========================\n\nInterface to the tracking module. It returns a callable function for tracking\nfeatures.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nfrom pysteps.tracking import lucaskanade\nfrom pysteps.tracking import tdating\n\n_tracking_methods = dict()\n_tracking_methods[\"lucaskanade\"] = lucaskanade.track_features\n_tracking_methods[\"tdating\"] = tdating.dating\n\n\ndef get_method(name):\n \"\"\"\n Return a callable function for tracking features.\n\n Description:\n Return a callable function for tracking features on input images .\n\n Implemented methods:\n\n +-----------------+--------------------------------------------------------+\n | Name | Description |\n +=================+========================================================+\n | lucaskanade | Wrapper to the OpenCV implementation of the |\n | | Lucas-Kanade tracking algorithm |\n +-----------------+--------------------------------------------------------+\n | tdating | Thunderstorm Detection and Tracking (DATing) module |\n +-----------------+--------------------------------------------------------+\n \"\"\"\n if isinstance(name, str):\n name = name.lower()\n else:\n raise TypeError(\n \"Only strings supported for the method's names.\\n\"\n + \"Available names:\"\n + str(list(_tracking_methods.keys()))\n ) from None\n\n try:\n return _tracking_methods[name]\n except KeyError:\n raise ValueError(\n \"Unknown tracking method {}\\n\".format(name)\n + \"The available methods are:\"\n + str(list(_tracking_methods.keys()))\n ) from None\n" }, { "path": "pysteps/tracking/lucaskanade.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.tracking.lucaskanade\n============================\n\nThe Lucas-Kanade (LK) feature tracking module.\n\nThis module implements the interface to the local `Lucas-Kanade`_ routine\navailable in OpenCV_.\n\n.. _OpenCV: https://opencv.org/\n\n.. _`Lucas-Kanade`:\\\n https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323\n\n.. autosummary::\n :toctree: ../generated/\n\n track_features\n\"\"\"\n\nimport numpy as np\nfrom numpy.ma.core import MaskedArray\n\nfrom pysteps.exceptions import MissingOptionalDependency\n\ntry:\n import cv2\n\n CV2_IMPORTED = True\nexcept ImportError:\n CV2_IMPORTED = False\n\n\ndef track_features(\n prvs_image,\n next_image,\n points,\n winsize=(50, 50),\n nr_levels=3,\n criteria=(3, 10, 0),\n flags=0,\n min_eig_thr=1e-4,\n verbose=False,\n):\n \"\"\"\n Interface to the OpenCV `Lucas-Kanade`_ feature tracking algorithm\n (cv.calcOpticalFlowPyrLK).\n\n .. _`Lucas-Kanade`:\\\n https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323\n\n .. _calcOpticalFlowPyrLK:\\\n https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323\n\n\n .. _MaskedArray:\\\n https://docs.scipy.org/doc/numpy/reference/maskedarray.baseclass.html#numpy.ma.MaskedArray\n\n .. _ndarray:\\\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\n\n Parameters\n ----------\n prvs_image: ndarray_ or MaskedArray_\n Array of shape (m, n) containing the first image.\n Invalid values (Nans or infs) are replaced with the min value.\n next_image: ndarray_ or MaskedArray_\n Array of shape (m, n) containing the successive image.\n Invalid values (Nans or infs) are replaced with the min value.\n points: array_like\n Array of shape (p, 2) indicating the pixel coordinates of the\n tracking points (corners).\n winsize: tuple of int, optional\n The **winSize** parameter in calcOpticalFlowPyrLK_.\n It represents the size of the search window that it is used at each\n pyramid level. The default is (50, 50).\n nr_levels: int, optional\n The **maxLevel** parameter in calcOpticalFlowPyrLK_.\n It represents the 0-based maximal pyramid level number.\n The default is 3.\n criteria: tuple of int, optional\n The **TermCriteria** parameter in calcOpticalFlowPyrLK_ ,\n which specifies the termination criteria of the iterative search\n algorithm. The default is (3, 10, 0).\n flags: int, optional\n Operation flags, see documentation calcOpticalFlowPyrLK_. The\n default is 0.\n min_eig_thr: float, optional\n The **minEigThreshold** parameter in calcOpticalFlowPyrLK_. The\n default is 1e-4.\n verbose: bool, optional\n Print the number of vectors that have been found. The default\n is False.\n\n Returns\n -------\n xy: ndarray_\n Array of shape (d, 2) with the x- and y-coordinates of *d* <= *p*\n detected sparse motion vectors.\n uv: ndarray_\n Array of shape (d, 2) with the u- and v-components of *d* <= *p*\n detected sparse motion vectors.\n\n Notes\n -----\n The tracking points can be obtained with the\n :py:func:`pysteps.utils.images.ShiTomasi_detection` routine.\n\n See also\n --------\n pysteps.motion.lucaskanade.dense_lucaskanade\n\n References\n ----------\n Bouguet, J.-Y.: Pyramidal implementation of the affine Lucas Kanade\n feature tracker description of the algorithm, Intel Corp., 5, 4, 2001\n\n Lucas, B. D. and Kanade, T.: An iterative image registration technique with\n an application to stereo vision, in: Proceedings of the 1981 DARPA Imaging\n Understanding Workshop, pp. 121–130, 1981.\n \"\"\"\n\n if not CV2_IMPORTED:\n raise MissingOptionalDependency(\n \"opencv package is required for the calcOpticalFlowPyrLK() \"\n \"routine but it is not installed\"\n )\n\n prvs_img = prvs_image.copy()\n next_img = next_image.copy()\n p0 = np.copy(points)\n\n # Check if a MaskedArray is used. If not, mask the ndarray\n if not isinstance(prvs_img, MaskedArray):\n prvs_img = np.ma.masked_invalid(prvs_img)\n np.ma.set_fill_value(prvs_img, prvs_img.min())\n\n if not isinstance(next_img, MaskedArray):\n next_img = np.ma.masked_invalid(next_img)\n np.ma.set_fill_value(next_img, next_img.min())\n\n # scale between 0 and 255\n im_min = prvs_img.min()\n im_max = prvs_img.max()\n if (im_max - im_min) > 1e-8:\n prvs_img = (prvs_img.filled() - im_min) / (im_max - im_min) * 255\n else:\n prvs_img = prvs_img.filled() - im_min\n\n im_min = next_img.min()\n im_max = next_img.max()\n if (im_max - im_min) > 1e-8:\n next_img = (next_img.filled() - im_min) / (im_max - im_min) * 255\n else:\n next_img = next_img.filled() - im_min\n\n # convert to 8-bit\n prvs_img = np.ndarray.astype(prvs_img, \"uint8\")\n next_img = np.ndarray.astype(next_img, \"uint8\")\n\n # Lucas-Kanade\n # TODO: use the error returned by the OpenCV routine\n params = dict(\n winSize=winsize,\n maxLevel=nr_levels,\n criteria=criteria,\n flags=flags,\n minEigThreshold=min_eig_thr,\n )\n p1, st, __ = cv2.calcOpticalFlowPyrLK(prvs_img, next_img, p0, None, **params)\n\n # keep only features that have been found\n st = np.atleast_1d(st.squeeze()) == 1\n if np.any(st):\n p1 = p1[st, :]\n p0 = p0[st, :]\n\n # extract vectors\n xy = p0\n uv = p1 - p0\n\n else:\n xy = uv = np.empty(shape=(0, 2))\n\n if verbose:\n print(f\"--- {xy.shape[0]} sparse vectors found ---\")\n\n return xy, uv\n" }, { "path": "pysteps/tracking/tdating.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.tracking.tdating\n========================\n\nThunderstorm Detection and Tracking (DATing) module\nThis module was implemented following the procedures used in the TRT Thunderstorms\nRadar Tracking algorithm (:cite:`TRT2004`) used operationally at MeteoSwiss.\nFull documentation is published in :cite:`Feldmann2021`.\nModifications include advecting the identified thunderstorms with the optical flow\nobtained from pysteps, as well as additional options in the thresholding.\n\nReferences\n...............\n:cite:`TRT2004`\n:cite:`Feldmann2021`\n\n@author: mfeldman\n\n.. autosummary::\n :toctree: ../generated/\n\n dating\n tracking\n advect\n match\n couple_track\n\"\"\"\n\nimport numpy as np\n\nimport pysteps.feature.tstorm as tstorm_detect\nfrom pysteps import motion\nfrom pysteps.exceptions import MissingOptionalDependency\n\ntry:\n import skimage\n\n SKIMAGE_IMPORTED = True\nexcept ImportError:\n SKIMAGE_IMPORTED = False\nif SKIMAGE_IMPORTED:\n import skimage.measure as skime\ntry:\n import pandas as pd\n\n PANDAS_IMPORTED = True\nexcept ImportError:\n PANDAS_IMPORTED = False\n\n\ndef dating(\n input_video,\n timelist,\n mintrack=3,\n cell_list=None,\n label_list=None,\n start=0,\n minref=35,\n maxref=48,\n mindiff=6,\n minsize=50,\n minmax=41,\n mindis=10,\n dyn_thresh=False,\n match_frac=0.4,\n split_frac=0.1,\n merge_frac=0.1,\n output_splits_merges=False,\n):\n \"\"\"\n This function performs the thunderstorm detection and tracking DATing.\n It requires a 3-D input array that contains the temporal succession of the 2-D data\n array of each timestep. On each timestep the detection is performed, the identified\n objects are advected with a flow prediction and the advected objects are matched to\n the newly identified objects of the next timestep.\n The last portion re-arranges the data into tracks sorted by ID-number.\n\n Parameters\n ----------\n input_video: array-like\n Array of shape (t,m,n) containing input image, with t being the temporal\n dimension and m,n the spatial dimensions. Thresholds are tuned to maximum\n reflectivity in dBZ with a spatial resolution of 1 km and a temporal resolution\n of 5 min. Nan values are ignored.\n timelist: list\n List of length t containing string of time and date of each (m,n) field.\n mintrack: int, optional\n minimum duration of cell-track to be counted. The default is 3 time steps.\n cell_list: list or None, optional\n If you wish to expand an existing list of cells, insert previous cell-list here.\n The default is None.\n If not None, requires that label_list has the same length.\n label_list: list or None, optional\n If you wish to expand an existing list of cells, insert previous label-list here.\n The default is None.\n If not None, requires that cell_list has the same length.\n start: int, optional\n If you wish to expand an existing list of cells, the input video must contain 2\n timesteps prior to the merging. The start can then be set to 2, allowing the\n motion vectors to be formed from the first three grids and continuing the cell\n tracking from there. The default is 0, which initiates a new tracking sequence.\n minref: float, optional\n Lower threshold for object detection. Lower values will be set to NaN.\n The default is 35 dBZ.\n maxref: float, optional\n Upper threshold for object detection. Higher values will be set to this value.\n The default is 48 dBZ.\n mindiff: float, optional\n Minimal difference between two identified maxima within same area to split area\n into two objects. The default is 6 dBZ.\n minsize: float, optional\n Minimal area for possible detected object. The default is 50 pixels.\n minmax: float, optional\n Minimum value of maximum in identified objects. Objects with a maximum lower\n than this will be discarded. The default is 41 dBZ.\n mindis: float, optional\n Minimum distance between two maxima of identified objects. Objects with a\n smaller distance will be merged. The default is 10 km.\n match_frac: float, optional\n Minimum overlap fraction between two objects to be considered the same object.\n Default is 0.4.\n split_frac: float, optional\n Minimum overlap fraction between two objects for the object at second timestep\n to be considered possibly split from the object at the first timestep.\n Default is 0.1.\n merge_frac: float, optional\n Minimum overlap fraction between two objects for the object at second timestep\n to be considered possibly merged from the object at the first timestep.\n Default is 0.1.\n output_splits_merges: bool, optional\n If True, the output will contain information about splits and merges.\n The provided columns are:\n\n .. tabularcolumns:: |p{2cm}|L|\n\n +-------------------+--------------------------------------------------------------+\n | Attribute | Description |\n +===================+==============================================================+\n | splitted | Indicates if the cell is considered split into multiple cells|\n +-------------------+--------------------------------------------------------------+\n | split_IDs | List of IDs at the next timestep that the cell split into |\n +-------------------+--------------------------------------------------------------+\n | merged | Indicates if the cell is considered a merge of multiple cells|\n +-------------------+--------------------------------------------------------------+\n | merged_IDs | List of IDs from the previous timestep that merged into this |\n | | cell |\n +-------------------+--------------------------------------------------------------+\n | results_from_split| True if the cell is a result of a split (i.e., the ID of the |\n | | cell is present in the split_IDs of some cell at the previous|\n | | timestep) |\n +-------------------+--------------------------------------------------------------+\n | will_merge | True if the cell will merge at the next timestep (i.e., the |\n | | ID of the cell is present in the merge_IDs of some cell at |\n | | the next timestep; empty if the next timestep is not tracked)|\n +-------------------+--------------------------------------------------------------+\n\n Returns\n -------\n track_list: list of dataframes\n Each dataframe contains the track and properties belonging to one cell ID.\n Columns of dataframes: ID - cell ID, time - time stamp, x - array of all\n x-coordinates of cell, y - array of all y-coordinates of cell, cen_x -\n x-coordinate of cell centroid, cen_y - y-coordinate of cell centroid, max_ref -\n maximum (reflectivity) value of cell, cont - cell contours\n cell_list: list of dataframes\n Each dataframe contains the detected cells and properties belonging to one\n timestep. The IDs are already matched to provide a track.\n Columns of dataframes: ID - cell ID, time - time stamp, x - array of all\n x-coordinates of cell, y - array of all y-coordinates of cell, cen_x -\n x-coordinate of cell centroid, cen_y - y-coordinate of cell centroid, max_ref -\n maximum (reflectivity) value of cell, cont - cell contours\n label_list: list of arrays\n Each (n,m) array contains the gridded IDs of the cells identified in the\n corresponding timestep. The IDs are already matched to provide a track.\n\n \"\"\"\n if not SKIMAGE_IMPORTED:\n raise MissingOptionalDependency(\n \"skimage is required for thunderstorm DATing \" \"but it is not installed\"\n )\n if not PANDAS_IMPORTED:\n raise MissingOptionalDependency(\n \"pandas is required for thunderstorm DATing \" \"but it is not installed\"\n )\n\n # Check arguments\n if cell_list is None or label_list is None:\n cell_list = []\n label_list = []\n else:\n if not len(cell_list) == len(label_list):\n raise ValueError(\"len(cell_list) != len(label_list)\")\n if start > len(timelist):\n raise ValueError(\"start > len(timelist)\")\n\n oflow_method = motion.get_method(\"LK\")\n if len(label_list) == 0:\n max_ID = 0\n else:\n max_ID = np.nanmax([np.nanmax(np.unique(label_list)), 0])\n for t in range(start, len(timelist)):\n cells_id, labels = tstorm_detect.detection(\n input_video[t, :, :],\n minref=minref,\n maxref=maxref,\n mindiff=mindiff,\n minsize=minsize,\n minmax=minmax,\n mindis=mindis,\n time=timelist[t],\n output_splits_merges=output_splits_merges,\n )\n if len(cell_list) < 2:\n cell_list.append(cells_id)\n label_list.append(labels)\n cid = np.unique(labels)\n max_ID = np.nanmax([np.nanmax(cid), max_ID]) + 1\n continue\n if t >= 2:\n flowfield = oflow_method(input_video[t - 2 : t + 1, :, :])\n cells_id, max_ID, newlabels, splitted_cells = tracking(\n cells_id,\n cell_list[-1],\n labels,\n flowfield,\n max_ID,\n match_frac=match_frac,\n split_frac=split_frac,\n merge_frac=merge_frac,\n output_splits_merges=output_splits_merges,\n )\n\n if output_splits_merges:\n # Assign splitted parameters for the previous timestep\n for _, split_cell in splitted_cells.iterrows():\n prev_list_id = cell_list[-1][\n cell_list[-1].ID == split_cell.ID\n ].index.item()\n\n split_ids = split_cell.split_IDs\n split_ids_updated = []\n for sid in split_ids:\n split_ids_updated.append(newlabels[labels == sid][0])\n\n cell_list[-1].at[prev_list_id, \"splitted\"] = True\n cell_list[-1].at[prev_list_id, \"split_IDs\"] = split_ids_updated\n\n for sid in split_ids_updated:\n cur_list_id = cells_id[cells_id.ID == sid].index.item()\n cells_id.at[cur_list_id, \"results_from_split\"] = True\n\n merged_cells = cells_id[cells_id.merged == True]\n for _, cell in merged_cells.iterrows():\n for merged_id in cell.merged_IDs:\n prev_list_id = cell_list[-1][\n cell_list[-1].ID == merged_id\n ].index.item()\n cell_list[-1].at[prev_list_id, \"will_merge\"] = True\n\n cid = np.unique(newlabels)\n # max_ID = np.nanmax([np.nanmax(cid), max_ID])\n cell_list.append(cells_id)\n label_list.append(newlabels)\n\n track_list = couple_track(cell_list[2:], int(max_ID), mintrack)\n\n return track_list, cell_list, label_list\n\n\ndef tracking(\n cells_id,\n cells_id_prev,\n labels,\n V1,\n max_ID,\n match_frac=0.4,\n merge_frac=0.1,\n split_frac=0.1,\n output_splits_merges=False,\n):\n \"\"\"\n This function performs the actual tracking procedure. First the cells are advected,\n then overlapped and finally their IDs are matched. If no match is found, a new ID\n is assigned.\n \"\"\"\n cells_id_new = cells_id.copy()\n cells_ad = advect(\n cells_id_prev, labels, V1, output_splits_merges=output_splits_merges\n )\n cells_ov, labels, possible_merge_ids = match(\n cells_ad,\n labels,\n output_splits_merges=output_splits_merges,\n split_frac=split_frac,\n match_frac=match_frac,\n )\n\n splitted_cells = None\n if output_splits_merges:\n splitted_cells = cells_ov[cells_ov.splitted == True]\n\n newlabels = np.zeros(labels.shape)\n possible_merge_ids_new = {}\n for index, cell in cells_id_new.iterrows():\n if cell.ID == 0 or np.isnan(cell.ID):\n continue\n new_ID = cells_ov[cells_ov.t_ID == cell.ID].ID.values\n if len(new_ID) > 0:\n xx = cells_ov[cells_ov.t_ID == cell.ID].x\n size = []\n for x in xx:\n size.append(len(x))\n biggest = np.argmax(size)\n new_ID = new_ID[biggest]\n cells_id_new.loc[index, \"ID\"] = new_ID\n else:\n max_ID += 1\n new_ID = max_ID\n cells_id_new.loc[index, \"ID\"] = new_ID\n newlabels[labels == index + 1] = new_ID\n possible_merge_ids_new[new_ID] = possible_merge_ids[cell.ID]\n del new_ID\n\n if output_splits_merges:\n # Process possible merges\n for target_id, possible_IDs in possible_merge_ids_new.items():\n merge_ids = []\n for p_id in possible_IDs:\n cell_a = cells_ad[cells_ad.ID == p_id]\n\n ID_vec = newlabels[cell_a.y.item(), cell_a.x.item()]\n overlap = np.sum(ID_vec == target_id) / len(ID_vec)\n if overlap > merge_frac:\n merge_ids.append(p_id)\n\n if len(merge_ids) > 1:\n cell_id = cells_id_new[cells_id_new.ID == target_id].index.item()\n # Merge cells\n cells_id_new.at[cell_id, \"merged\"] = True\n cells_id_new.at[cell_id, \"merged_IDs\"] = merge_ids\n\n return cells_id_new, max_ID, newlabels, splitted_cells\n\n\ndef advect(cells_id, labels, V1, output_splits_merges=False):\n \"\"\"\n This function advects all identified cells with the estimated flow.\n \"\"\"\n columns = [\n \"ID\",\n \"x\",\n \"y\",\n \"cen_x\",\n \"cen_y\",\n \"max_ref\",\n \"cont\",\n \"t_ID\",\n \"frac\",\n \"flowx\",\n \"flowy\",\n ]\n if output_splits_merges:\n columns.extend([\"splitted\", \"split_IDs\", \"split_fracs\"])\n cells_ad = pd.DataFrame(\n data=None,\n index=range(len(cells_id)),\n columns=columns,\n )\n for ID, cell in cells_id.iterrows():\n if cell.ID == 0 or np.isnan(cell.ID):\n continue\n ad_x = np.round(np.nanmean(V1[0, cell.y, cell.x])).astype(int)\n ad_y = np.round(np.nanmean(V1[1, cell.y, cell.x])).astype(int)\n new_x = cell.x + ad_x\n new_y = cell.y + ad_y\n new_x[new_x > labels.shape[1] - 1] = labels.shape[1] - 1\n new_y[new_y > labels.shape[0] - 1] = labels.shape[0] - 1\n new_x[new_x < 0] = 0\n new_y[new_y < 0] = 0\n new_cen_x = cell.cen_x + ad_x\n new_cen_y = cell.cen_y + ad_y\n\n # Use scalar cell assignment (.at) to store array-like payloads in object columns.\n cells_ad.at[ID, \"x\"] = new_x\n cells_ad.at[ID, \"y\"] = new_y\n cells_ad.at[ID, \"flowx\"] = ad_x\n cells_ad.at[ID, \"flowy\"] = ad_y\n cells_ad.at[ID, \"cen_x\"] = new_cen_x\n cells_ad.at[ID, \"cen_y\"] = new_cen_y\n cells_ad.at[ID, \"ID\"] = cell.ID\n\n cell_unique = np.zeros(labels.shape)\n cell_unique[new_y, new_x] = 1\n cells_ad.at[ID, \"cont\"] = skime.find_contours(cell_unique, 0.8)\n\n return cells_ad\n\n\ndef match(cells_ad, labels, match_frac=0.4, split_frac=0.1, output_splits_merges=False):\n \"\"\"\n This function matches the advected cells of the previous timestep to the newly\n identified ones. A minimal overlap of 40% is required. In case of split of merge,\n the larger cell supersedes the smaller one in naming.\n \"\"\"\n cells_ov = cells_ad.copy()\n possible_merge_ids = {i: [] for i in np.unique(labels)}\n for ID_a, cell_a in cells_ov.iterrows():\n if cell_a.ID == 0 or np.isnan(cell_a.ID):\n continue\n ID_vec = labels[cell_a.y, cell_a.x]\n IDs = np.unique(ID_vec)\n n_IDs = len(IDs)\n if n_IDs == 1 and IDs[0] == 0:\n cells_ov.at[ID_a, \"t_ID\"] = 0\n continue\n IDs = IDs[IDs != 0]\n n_IDs = len(IDs)\n\n for i in IDs:\n possible_merge_ids[i].append(cell_a.ID)\n\n N = np.zeros(n_IDs)\n for n in range(n_IDs):\n N[n] = len(np.where(ID_vec == IDs[n])[0])\n\n if output_splits_merges:\n # Only consider possible split if overlap is large enough\n valid_split_ids = (N / len(ID_vec)) > split_frac\n # splits here\n if sum(valid_split_ids) > 1:\n # Save split information\n cells_ov.at[ID_a, \"splitted\"] = True\n cells_ov.at[ID_a, \"split_IDs\"] = IDs[valid_split_ids].tolist()\n cells_ov.at[ID_a, \"split_fracs\"] = (N / len(ID_vec)).tolist()\n\n m = np.argmax(N)\n ID_match = IDs[m]\n ID_coverage = N[m] / len(ID_vec)\n if ID_coverage >= match_frac:\n cells_ov.at[ID_a, \"t_ID\"] = ID_match\n else:\n cells_ov.at[ID_a, \"t_ID\"] = 0\n cells_ov.at[ID_a, \"frac\"] = ID_coverage\n return cells_ov, labels, possible_merge_ids\n\n\ndef couple_track(cell_list, max_ID, mintrack):\n \"\"\"\n The coupled cell tracks are re-arranged from the list of cells sorted by time, to\n a list of tracks sorted by ID. Tracks shorter than mintrack are rejected.\n \"\"\"\n track_list = []\n for n in range(1, max_ID):\n cell_track = pd.DataFrame(\n data=None,\n index=None,\n columns=[\"ID\", \"time\", \"x\", \"y\", \"cen_x\", \"cen_y\", \"max_ref\", \"cont\"],\n )\n cell_track = []\n for t in range(len(cell_list)):\n mytime = cell_list[t]\n cell_track.append(mytime[mytime.ID == n])\n cell_track = pd.concat(cell_track, axis=0)\n\n if len(cell_track) < mintrack:\n continue\n track_list.append(cell_track)\n return track_list\n" }, { "path": "pysteps/utils/__init__.py", "content": "\"\"\"Miscellaneous utility functions.\"\"\"\n\nfrom .arrays import *\nfrom .cleansing import *\nfrom .conversion import *\nfrom .dimension import *\nfrom .images import *\nfrom .interface import get_method\nfrom .interpolate import *\nfrom .fft import *\nfrom .pca import *\nfrom .reprojection import *\nfrom .spectral import *\nfrom .tapering import *\nfrom .transformation import *\n" }, { "path": "pysteps/utils/arrays.py", "content": "\"\"\"\npysteps.utils.arrays\n====================\n\nUtility methods for creating and processing arrays.\n\n.. autosummary::\n :toctree: ../generated/\n\n compute_centred_coord_array\n\"\"\"\n\nimport numpy as np\n\n\ndef compute_centred_coord_array(M, N):\n \"\"\"\n Compute a 2D coordinate array, where the origin is at the center.\n\n Parameters\n ----------\n M : int\n The height of the array.\n N : int\n The width of the array.\n\n Returns\n -------\n out : ndarray\n The coordinate array.\n\n Examples\n --------\n >>> compute_centred_coord_array(2, 2)\n\n (array([[-2],\\n\n [-1],\\n\n [ 0],\\n\n [ 1],\\n\n [ 2]]), array([[-2, -1, 0, 1, 2]]))\n\n \"\"\"\n\n if M % 2 == 1:\n s1 = np.s_[-int(M / 2) : int(M / 2) + 1]\n else:\n s1 = np.s_[-int(M / 2) : int(M / 2)]\n\n if N % 2 == 1:\n s2 = np.s_[-int(N / 2) : int(N / 2) + 1]\n else:\n s2 = np.s_[-int(N / 2) : int(N / 2)]\n\n YC, XC = np.ogrid[s1, s2]\n\n return YC, XC\n" }, { "path": "pysteps/utils/check_norain.py", "content": "import numpy as np\n\nfrom pysteps import utils\n\n\ndef check_norain(precip_arr, precip_thr=None, norain_thr=0.0, win_fun=None):\n \"\"\"\n\n Parameters\n ----------\n precip_arr: array-like\n An at least 2 dimensional array containing the input precipitation field\n precip_thr: float, optional\n Specifies the threshold value for minimum observable precipitation intensity. If None, the\n minimum value over the domain is taken.\n norain_thr: float, optional\n Specifies the threshold value for the fraction of rainy pixels in precip_arr below which we consider there to be\n no rain. Standard set to 0.0\n win_fun: {'hann', 'tukey', None}\n Optional tapering function to be applied to the input field, generated with\n :py:func:`pysteps.utils.tapering.compute_window_function`\n (default None).\n This parameter needs to match the window function you use in later noise generation,\n or else this method will say that there is rain, while after the tapering function is\n applied there is no rain left, so you will run into a ValueError.\n Returns\n -------\n norain: bool\n Returns whether the fraction of rainy pixels is below the norain_thr threshold.\n\n \"\"\"\n\n if win_fun is not None:\n tapering = utils.tapering.compute_window_function(\n precip_arr.shape[-2], precip_arr.shape[-1], win_fun\n )\n else:\n tapering = np.ones((precip_arr.shape[-2], precip_arr.shape[-1]))\n\n tapering_mask = tapering == 0.0\n masked_precip = precip_arr.copy()\n masked_precip[..., tapering_mask] = np.nanmin(precip_arr)\n\n if precip_thr is None:\n precip_thr = np.nanmin(masked_precip)\n rain_pixels = masked_precip[masked_precip > precip_thr]\n norain = rain_pixels.size / masked_precip.size <= norain_thr\n print(\n f\"Rain fraction is: {str(rain_pixels.size / masked_precip.size)}, while minimum fraction is {str(norain_thr)}\"\n )\n return norain\n" }, { "path": "pysteps/utils/cleansing.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\npysteps.utils.cleansing\r\n=======================\r\n\r\nData cleansing routines for pysteps.\r\n\r\n.. autosummary::\r\n :toctree: ../generated/\r\n\r\n decluster\r\n detect_outliers\r\n\"\"\"\r\n\r\nimport warnings\r\n\r\nimport numpy as np\r\nimport scipy.spatial\r\n\r\n\r\ndef decluster(coord, input_array, scale, min_samples=1, verbose=False):\r\n \"\"\"\r\n Decluster a set of sparse data points by aggregating, that is, taking\r\n the median value of all values lying within a certain distance (i.e., a\r\n cluster).\r\n\r\n Parameters\r\n ----------\r\n coord: array_like\r\n Array of shape (n, d) containing the coordinates of the input data into\r\n a space of *d* dimensions.\r\n input_array: array_like\r\n Array of shape (n) or (n, m), where *n* is the number of samples and\r\n *m* the number of variables.\r\n All values in ``input_array`` are required to have finite values.\r\n scale: float or array_like\r\n The ``scale`` parameter in the same units of ``coord``.\r\n It can be a scalar or an array_like of shape (d).\r\n Data points within the declustering ``scale`` are aggregated.\r\n min_samples: int, optional\r\n The minimum number of samples for computing the median within a given\r\n cluster.\r\n verbose: bool, optional\r\n Print out information.\r\n\r\n Returns\r\n -------\r\n out: tuple of ndarrays\r\n A two-element tuple (``out_coord``, ``output_array``) containing the\r\n declustered coordinates (l, d) and input array (l, m), where *l* is\r\n the new number of samples with *l* <= *n*.\r\n \"\"\"\r\n\r\n coord = np.copy(coord)\r\n input_array = np.copy(input_array)\r\n\r\n # check inputs\r\n if np.any(~np.isfinite(input_array)):\r\n raise ValueError(\"input_array contains non-finite values\")\r\n\r\n if input_array.ndim == 1:\r\n nvar = 1\r\n input_array = input_array[:, None]\r\n elif input_array.ndim == 2:\r\n nvar = input_array.shape[1]\r\n else:\r\n raise ValueError(\r\n \"input_array must have 1 (n) or 2 dimensions (n, m), but it has %i\"\r\n % input_array.ndim\r\n )\r\n\r\n if coord.ndim != 2:\r\n raise ValueError(\r\n \"coord must have 2 dimensions (n, d), but it has %i\" % coord.ndim\r\n )\r\n if coord.shape[0] != input_array.shape[0]:\r\n raise ValueError(\r\n \"the number of samples in the input_array does not match the \"\r\n + \"number of coordinates %i!=%i\" % (input_array.shape[0], coord.shape[0])\r\n )\r\n\r\n if np.isscalar(scale):\r\n scale = float(scale)\r\n else:\r\n scale = np.copy(scale)\r\n if scale.ndim != 1:\r\n raise ValueError(\r\n \"scale must have 1 dimension (d), but it has %i\" % scale.ndim\r\n )\r\n if scale.shape[0] != coord.shape[1]:\r\n raise ValueError(\r\n \"scale must have %i elements, but it has %i\"\r\n % (coord.shape[1], scale.shape[0])\r\n )\r\n scale = scale[None, :]\r\n\r\n # reduce original coordinates\r\n coord_ = np.floor(coord / scale)\r\n\r\n # keep only unique pairs of the reduced coordinates\r\n ucoord_ = np.unique(coord_, axis=0)\r\n\r\n # loop through these unique values and average data points which belong to\r\n # the same cluster\r\n dinput = np.empty(shape=(0, nvar))\r\n dcoord = np.empty(shape=(0, coord.shape[1]))\r\n for i in range(ucoord_.shape[0]):\r\n idx = np.all(coord_ == ucoord_[i, :], axis=1)\r\n npoints = np.sum(idx)\r\n if npoints >= min_samples:\r\n dinput = np.append(\r\n dinput, np.median(input_array[idx, :], axis=0)[None, :], axis=0\r\n )\r\n dcoord = np.append(\r\n dcoord, np.median(coord[idx, :], axis=0)[None, :], axis=0\r\n )\r\n\r\n if verbose:\r\n print(\"--- %i samples left after declustering ---\" % dinput.shape[0])\r\n\r\n return dcoord, dinput\r\n\r\n\r\ndef detect_outliers(input_array, thr, coord=None, k=None, verbose=False):\r\n \"\"\"\r\n Detect outliers in a (multivariate and georeferenced) dataset.\r\n\r\n Assume a (multivariate) Gaussian distribution and detect outliers based on\r\n the number of standard deviations from the mean.\r\n\r\n If spatial information is provided through coordinates, the outlier\r\n detection can be localized by considering only the k-nearest neighbours\r\n when computing the local mean and standard deviation.\r\n\r\n Parameters\r\n ----------\r\n input_array: array_like\r\n Array of shape (n) or (n, m), where *n* is the number of samples and\r\n *m* the number of variables. If *m* > 1, the Mahalanobis distance\r\n is used.\r\n All values in ``input_array`` are required to have finite values.\r\n thr: float\r\n The number of standard deviations from the mean used to define an outlier.\r\n coord: array_like or None, optional\r\n Array of shape (n, d) containing the coordinates of the input data into\r\n a space of *d* dimensions.\r\n Passing ``coord`` requires that ``k`` is not None.\r\n k: int or None, optional\r\n The number of nearest neighbours used to localize the outlier\r\n detection. If set to None (the default), it employs all the data points (global\r\n detection). Setting ``k`` requires that ``coord`` is not None.\r\n verbose: bool, optional\r\n Print out information.\r\n\r\n Returns\r\n -------\r\n out: array_like\r\n A 1-D boolean array of shape (n) with True values indicating the outliers\r\n detected in ``input_array``.\r\n \"\"\"\r\n\r\n input_array = np.copy(input_array)\r\n\r\n if np.any(~np.isfinite(input_array)):\r\n raise ValueError(\"input_array contains non-finite values\")\r\n\r\n if input_array.ndim == 1:\r\n nsamples = input_array.size\r\n nvar = 1\r\n elif input_array.ndim == 2:\r\n nsamples = input_array.shape[0]\r\n nvar = input_array.shape[1]\r\n else:\r\n raise ValueError(\r\n f\"input_array must have 1 (n) or 2 dimensions (n, m), \"\r\n f\"but it has {input_array.ndim}\"\r\n )\r\n\r\n if nsamples < 2:\r\n return np.zeros(nsamples, dtype=bool)\r\n\r\n if coord is not None and k is not None:\r\n coord = np.copy(coord)\r\n if coord.ndim == 1:\r\n coord = coord[:, None]\r\n\r\n elif coord.ndim > 2:\r\n raise ValueError(\r\n \"coord must have 2 dimensions (n, d),\" f\"but it has {coord.ndim}\"\r\n )\r\n\r\n if coord.shape[0] != nsamples:\r\n raise ValueError(\r\n \"the number of samples in input_array does not match the \"\r\n f\"number of coordinates {nsamples}!={coord.shape[0]}\"\r\n )\r\n\r\n k = np.min((nsamples, k + 1))\r\n\r\n # global\r\n\r\n if k is None or coord is None:\r\n if nvar == 1:\r\n # univariate\r\n zdata = np.abs(input_array - np.mean(input_array)) / np.std(input_array)\r\n outliers = zdata > thr\r\n else:\r\n # multivariate (mahalanobis distance)\r\n zdata = input_array - np.mean(input_array, axis=0)\r\n V = np.cov(zdata.T)\r\n try:\r\n VI = np.linalg.inv(V)\r\n MD = np.sqrt(np.dot(np.dot(zdata, VI), zdata.T).diagonal())\r\n except np.linalg.LinAlgError as err:\r\n warnings.warn(f\"{err} during outlier detection\")\r\n MD = np.zeros(nsamples)\r\n outliers = MD > thr\r\n\r\n # local\r\n else:\r\n tree = scipy.spatial.cKDTree(coord)\r\n __, inds = tree.query(coord, k=k)\r\n outliers = np.empty(shape=0, dtype=bool)\r\n for i in range(inds.shape[0]):\r\n if nvar == 1:\r\n # univariate\r\n thisdata = input_array[i]\r\n neighbours = input_array[inds[i, 1:]]\r\n thiszdata = np.abs(thisdata - np.mean(neighbours)) / np.std(neighbours)\r\n outliers = np.append(outliers, thiszdata > thr)\r\n else:\r\n # multivariate (mahalanobis distance)\r\n thisdata = input_array[i, :]\r\n neighbours = input_array[inds[i, 1:], :].copy()\r\n thiszdata = thisdata - np.mean(neighbours, axis=0)\r\n neighbours = neighbours - np.mean(neighbours, axis=0)\r\n V = np.cov(neighbours.T)\r\n try:\r\n VI = np.linalg.inv(V)\r\n MD = np.sqrt(np.dot(np.dot(thiszdata, VI), thiszdata.T))\r\n except np.linalg.LinAlgError as err:\r\n warnings.warn(f\"{err} during outlier detection\")\r\n MD = 0\r\n outliers = np.append(outliers, MD > thr)\r\n\r\n if verbose:\r\n print(f\"--- {np.sum(outliers)} outliers detected ---\")\r\n\r\n return outliers\r\n" }, { "path": "pysteps/utils/conversion.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\npysteps.utils.conversion\r\n========================\r\n\r\nMethods for converting physical units.\r\n\r\n.. autosummary::\r\n :toctree: ../generated/\r\n\r\n to_rainrate\r\n to_raindepth\r\n to_reflectivity\r\n\"\"\"\r\n\r\nimport warnings\r\nfrom . import transformation\r\n\r\n# TODO: This should not be done. Instead fix the code so that it doesn't\r\n# produce the warnings.\r\n# to deactivate warnings for comparison operators with NaNs\r\nwarnings.filterwarnings(\"ignore\", category=RuntimeWarning)\r\n\r\n\r\ndef to_rainrate(R, metadata, zr_a=None, zr_b=None):\r\n \"\"\"\r\n Convert to rain rate [mm/h].\r\n\r\n Parameters\r\n ----------\r\n R: array-like\r\n Array of any shape to be (back-)transformed.\r\n metadata: dict\r\n Metadata dictionary containing the accutime, transform, unit, threshold\r\n and zerovalue attributes as described in the documentation of\r\n :py:mod:`pysteps.io.importers`.\r\n\r\n Additionally, in case of conversion to/from reflectivity units, the\r\n zr_a and zr_b attributes are also required,\r\n but only if zr_a = zr_b = None.\r\n If missing, it defaults to Marshall–Palmer relation,\r\n that is, zr_a = 200.0 and zr_b = 1.6.\r\n zr_a, zr_b: float, optional\r\n The a and b coefficients of the Z-R relationship (Z = a*R^b).\r\n\r\n Returns\r\n -------\r\n R: array-like\r\n Array of any shape containing the converted units.\r\n metadata: dict\r\n The metadata with updated attributes.\r\n \"\"\"\r\n\r\n R = R.copy()\r\n metadata = metadata.copy()\r\n\r\n if metadata[\"transform\"] is not None:\r\n if metadata[\"transform\"] == \"dB\":\r\n R, metadata = transformation.dB_transform(R, metadata, inverse=True)\r\n\r\n elif metadata[\"transform\"] in [\"BoxCox\", \"log\"]:\r\n R, metadata = transformation.boxcox_transform(R, metadata, inverse=True)\r\n\r\n elif metadata[\"transform\"] == \"NQT\":\r\n R, metadata = transformation.NQ_transform(R, metadata, inverse=True)\r\n\r\n elif metadata[\"transform\"] == \"sqrt\":\r\n R, metadata = transformation.sqrt_transform(R, metadata, inverse=True)\r\n\r\n else:\r\n raise ValueError(\"Unknown transformation %s\" % metadata[\"transform\"])\r\n\r\n if metadata[\"unit\"] == \"mm/h\":\r\n pass\r\n\r\n elif metadata[\"unit\"] == \"mm\":\r\n threshold = metadata[\"threshold\"] # convert the threshold, too\r\n zerovalue = metadata[\"zerovalue\"] # convert the zerovalue, too\r\n\r\n R = R / float(metadata[\"accutime\"]) * 60.0\r\n threshold = threshold / float(metadata[\"accutime\"]) * 60.0\r\n zerovalue = zerovalue / float(metadata[\"accutime\"]) * 60.0\r\n\r\n metadata[\"threshold\"] = threshold\r\n metadata[\"zerovalue\"] = zerovalue\r\n\r\n elif metadata[\"unit\"] == \"dBZ\":\r\n threshold = metadata[\"threshold\"] # convert the threshold, too\r\n zerovalue = metadata[\"zerovalue\"] # convert the zerovalue, too\r\n\r\n # Z to R\r\n if zr_a is None:\r\n zr_a = metadata.get(\"zr_a\", 200.0) # default to Marshall–Palmer\r\n if zr_b is None:\r\n zr_b = metadata.get(\"zr_b\", 1.6) # default to Marshall–Palmer\r\n R = (R / zr_a) ** (1.0 / zr_b)\r\n threshold = (threshold / zr_a) ** (1.0 / zr_b)\r\n zerovalue = (zerovalue / zr_a) ** (1.0 / zr_b)\r\n\r\n metadata[\"zr_a\"] = zr_a\r\n metadata[\"zr_b\"] = zr_b\r\n metadata[\"threshold\"] = threshold\r\n metadata[\"zerovalue\"] = zerovalue\r\n\r\n else:\r\n raise ValueError(\r\n \"Cannot convert unit %s and transform %s to mm/h\"\r\n % (metadata[\"unit\"], metadata[\"transform\"])\r\n )\r\n\r\n metadata[\"unit\"] = \"mm/h\"\r\n\r\n return R, metadata\r\n\r\n\r\ndef to_raindepth(R, metadata, zr_a=None, zr_b=None):\r\n \"\"\"\r\n Convert to rain depth [mm].\r\n\r\n Parameters\r\n ----------\r\n R: array-like\r\n Array of any shape to be (back-)transformed.\r\n metadata: dict\r\n Metadata dictionary containing the accutime, transform, unit, threshold\r\n and zerovalue attributes as described in the documentation of\r\n :py:mod:`pysteps.io.importers`.\r\n\r\n Additionally, in case of conversion to/from reflectivity units, the\r\n zr_a and zr_b attributes are also required,\r\n but only if zr_a = zr_b = None.\r\n If missing, it defaults to Marshall–Palmer relation, that is,\r\n zr_a = 200.0 and zr_b = 1.6.\r\n zr_a, zr_b: float, optional\r\n The a and b coefficients of the Z-R relationship (Z = a*R^b).\r\n\r\n Returns\r\n -------\r\n R: array-like\r\n Array of any shape containing the converted units.\r\n metadata: dict\r\n The metadata with updated attributes.\r\n \"\"\"\r\n\r\n R = R.copy()\r\n metadata = metadata.copy()\r\n\r\n if metadata[\"transform\"] is not None:\r\n if metadata[\"transform\"] == \"dB\":\r\n R, metadata = transformation.dB_transform(R, metadata, inverse=True)\r\n\r\n elif metadata[\"transform\"] in [\"BoxCox\", \"log\"]:\r\n R, metadata = transformation.boxcox_transform(R, metadata, inverse=True)\r\n\r\n elif metadata[\"transform\"] == \"NQT\":\r\n R, metadata = transformation.NQ_transform(R, metadata, inverse=True)\r\n\r\n elif metadata[\"transform\"] == \"sqrt\":\r\n R, metadata = transformation.sqrt_transform(R, metadata, inverse=True)\r\n\r\n else:\r\n raise ValueError(\"Unknown transformation %s\" % metadata[\"transform\"])\r\n\r\n if metadata[\"unit\"] == \"mm\" and metadata[\"transform\"] is None:\r\n pass\r\n\r\n elif metadata[\"unit\"] == \"mm/h\":\r\n threshold = metadata[\"threshold\"] # convert the threshold, too\r\n zerovalue = metadata[\"zerovalue\"] # convert the zerovalue, too\r\n\r\n R = R / 60.0 * metadata[\"accutime\"]\r\n threshold = threshold / 60.0 * metadata[\"accutime\"]\r\n zerovalue = zerovalue / 60.0 * metadata[\"accutime\"]\r\n\r\n metadata[\"threshold\"] = threshold\r\n metadata[\"zerovalue\"] = zerovalue\r\n\r\n elif metadata[\"unit\"] == \"dBZ\":\r\n threshold = metadata[\"threshold\"] # convert the threshold, too\r\n zerovalue = metadata[\"zerovalue\"] # convert the zerovalue, too\r\n\r\n # Z to R\r\n if zr_a is None:\r\n zr_a = metadata.get(\"zr_a\", 200.0) # Default to Marshall–Palmer\r\n if zr_b is None:\r\n zr_b = metadata.get(\"zr_b\", 1.6) # Default to Marshall–Palmer\r\n R = (R / zr_a) ** (1.0 / zr_b) / 60.0 * metadata[\"accutime\"]\r\n threshold = (threshold / zr_a) ** (1.0 / zr_b) / 60.0 * metadata[\"accutime\"]\r\n zerovalue = (zerovalue / zr_a) ** (1.0 / zr_b) / 60.0 * metadata[\"accutime\"]\r\n\r\n metadata[\"zr_a\"] = zr_a\r\n metadata[\"zr_b\"] = zr_b\r\n metadata[\"threshold\"] = threshold\r\n metadata[\"zerovalue\"] = zerovalue\r\n\r\n else:\r\n raise ValueError(\r\n \"Cannot convert unit %s and transform %s to mm\"\r\n % (metadata[\"unit\"], metadata[\"transform\"])\r\n )\r\n\r\n metadata[\"unit\"] = \"mm\"\r\n\r\n return R, metadata\r\n\r\n\r\ndef to_reflectivity(R, metadata, zr_a=None, zr_b=None):\r\n \"\"\"\r\n Convert to reflectivity [dBZ].\r\n\r\n Parameters\r\n ----------\r\n R: array-like\r\n Array of any shape to be (back-)transformed.\r\n metadata: dict\r\n Metadata dictionary containing the accutime, transform, unit, threshold\r\n and zerovalue attributes as described in the documentation of\r\n :py:mod:`pysteps.io.importers`.\r\n\r\n Additionally, in case of conversion to/from reflectivity units, the\r\n zr_a and zr_b attributes are also required,\r\n but only if zr_a = zr_b = None.\r\n If missing, it defaults to Marshall–Palmer relation, that is,\r\n zr_a = 200.0 and zr_b = 1.6.\r\n zr_a, zr_b: float, optional\r\n The a and b coefficients of the Z-R relationship (Z = a*R^b).\r\n\r\n Returns\r\n -------\r\n R: array-like\r\n Array of any shape containing the converted units.\r\n metadata: dict\r\n The metadata with updated attributes.\r\n \"\"\"\r\n\r\n R = R.copy()\r\n metadata = metadata.copy()\r\n\r\n if metadata[\"transform\"] is not None:\r\n if metadata[\"transform\"] == \"dB\":\r\n R, metadata = transformation.dB_transform(R, metadata, inverse=True)\r\n\r\n elif metadata[\"transform\"] in [\"BoxCox\", \"log\"]:\r\n R, metadata = transformation.boxcox_transform(R, metadata, inverse=True)\r\n\r\n elif metadata[\"transform\"] == \"NQT\":\r\n R, metadata = transformation.NQ_transform(R, metadata, inverse=True)\r\n\r\n elif metadata[\"transform\"] == \"sqrt\":\r\n R, metadata = transformation.sqrt_transform(R, metadata, inverse=True)\r\n\r\n else:\r\n raise ValueError(\"Unknown transformation %s\" % metadata[\"transform\"])\r\n\r\n if metadata[\"unit\"] == \"mm/h\":\r\n # Z to R\r\n if zr_a is None:\r\n zr_a = metadata.get(\"zr_a\", 200.0) # Default to Marshall–Palmer\r\n if zr_b is None:\r\n zr_b = metadata.get(\"zr_b\", 1.6) # Default to Marshall–Palmer\r\n\r\n R = zr_a * R**zr_b\r\n metadata[\"threshold\"] = zr_a * metadata[\"threshold\"] ** zr_b\r\n metadata[\"zerovalue\"] = zr_a * metadata[\"zerovalue\"] ** zr_b\r\n metadata[\"zr_a\"] = zr_a\r\n metadata[\"zr_b\"] = zr_b\r\n\r\n # Z to dBZ\r\n R, metadata = transformation.dB_transform(R, metadata)\r\n\r\n elif metadata[\"unit\"] == \"mm\":\r\n # depth to rate\r\n R, metadata = to_rainrate(R, metadata)\r\n\r\n # Z to R\r\n if zr_a is None:\r\n zr_a = metadata.get(\"zr_a\", 200.0) # Default to Marshall-Palmer\r\n if zr_b is None:\r\n zr_b = metadata.get(\"zr_b\", 1.6) # Default to Marshall-Palmer\r\n R = zr_a * R**zr_b\r\n metadata[\"threshold\"] = zr_a * metadata[\"threshold\"] ** zr_b\r\n metadata[\"zerovalue\"] = zr_a * metadata[\"zerovalue\"] ** zr_b\r\n metadata[\"zr_a\"] = zr_a\r\n metadata[\"zr_b\"] = zr_b\r\n\r\n # Z to dBZ\r\n R, metadata = transformation.dB_transform(R, metadata)\r\n\r\n elif metadata[\"unit\"] == \"dBZ\":\r\n # Z to dBZ\r\n R, metadata = transformation.dB_transform(R, metadata)\r\n\r\n else:\r\n raise ValueError(\r\n \"Cannot convert unit %s and transform %s to mm/h\"\r\n % (metadata[\"unit\"], metadata[\"transform\"])\r\n )\r\n metadata[\"unit\"] = \"dBZ\"\r\n return R, metadata\r\n" }, { "path": "pysteps/utils/dimension.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.utils.dimension\n=======================\n\nFunctions to manipulate array dimensions.\n\n.. autosummary::\n :toctree: ../generated/\n\n aggregate_fields\n aggregate_fields_time\n aggregate_fields_space\n clip_domain\n square_domain\n\"\"\"\n\nimport numpy as np\n\n_aggregation_methods = dict(\n sum=np.sum, mean=np.mean, nanmean=np.nanmean, nansum=np.nansum\n)\n\n\ndef aggregate_fields_time(R, metadata, time_window_min, ignore_nan=False):\n \"\"\"Aggregate fields in time.\n\n Parameters\n ----------\n R: array-like\n Array of shape (t,m,n) or (l,t,m,n) containing\n a time series of (ensemble) input fields.\n They must be evenly spaced in time.\n metadata: dict\n Metadata dictionary containing the timestamps and unit attributes as\n described in the documentation of :py:mod:`pysteps.io.importers`.\n time_window_min: float or None\n The length in minutes of the time window that is used to\n aggregate the fields.\n The time spanned by the t dimension of R must be a multiple of\n time_window_min.\n If set to None, it returns a copy of the original R and metadata.\n ignore_nan: bool, optional\n If True, ignore nan values.\n\n Returns\n -------\n outputarray: array-like\n The new array of aggregated fields of shape (k,m,n) or (l,k,m,n), where\n k = t*delta/time_window_min and delta is the time interval between two\n successive timestamps.\n metadata: dict\n The metadata with updated attributes.\n\n See also\n --------\n pysteps.utils.dimension.aggregate_fields_space,\n pysteps.utils.dimension.aggregate_fields\n \"\"\"\n\n R = R.copy()\n metadata = metadata.copy()\n\n if time_window_min is None:\n return R, metadata\n\n unit = metadata[\"unit\"]\n timestamps = metadata[\"timestamps\"]\n if \"leadtimes\" in metadata:\n leadtimes = metadata[\"leadtimes\"]\n\n if len(R.shape) < 3:\n raise ValueError(\"The number of dimension must be > 2\")\n if len(R.shape) == 3:\n axis = 0\n if len(R.shape) == 4:\n axis = 1\n if len(R.shape) > 4:\n raise ValueError(\"The number of dimension must be <= 4\")\n\n if R.shape[axis] != len(timestamps):\n raise ValueError(\n \"The list of timestamps has length %i, \" % len(timestamps)\n + \"but R contains %i frames\" % R.shape[axis]\n )\n\n # assumes that frames are evenly spaced\n delta = (timestamps[1] - timestamps[0]).seconds / 60\n if delta == time_window_min:\n return R, metadata\n if (R.shape[axis] * delta) % time_window_min:\n raise ValueError(\"time_window_size does not equally split R\")\n\n nframes = int(time_window_min / delta)\n\n # specify the operator to be used to aggregate\n # the values within the time window\n if unit == \"mm/h\":\n method = \"mean\"\n elif unit == \"mm\":\n method = \"sum\"\n else:\n raise ValueError(\n \"can only aggregate units of 'mm/h' or 'mm'\" + \" not %s\" % unit\n )\n\n if ignore_nan:\n method = \"\".join((\"nan\", method))\n\n R = aggregate_fields(R, nframes, axis=axis, method=method)\n\n metadata[\"accutime\"] = time_window_min\n metadata[\"timestamps\"] = timestamps[nframes - 1 :: nframes]\n if \"leadtimes\" in metadata:\n metadata[\"leadtimes\"] = leadtimes[nframes - 1 :: nframes]\n\n return R, metadata\n\n\ndef aggregate_fields_space(R, metadata, space_window, ignore_nan=False):\n \"\"\"\n Upscale fields in space.\n\n Parameters\n ----------\n R: array-like\n Array of shape (m,n), (t,m,n) or (l,t,m,n) containing a single field or\n a time series of (ensemble) input fields.\n metadata: dict\n Metadata dictionary containing the xpixelsize, ypixelsize and unit\n attributes as described in the documentation of\n :py:mod:`pysteps.io.importers`.\n space_window: float, tuple or None\n The length of the space window that is used to upscale the fields.\n If a float is given, the same window size is used for the x- and\n y-directions. Separate window sizes are used for x- and y-directions if\n a two-element tuple is given. The space_window unit is the same used in\n the geographical projection of R and hence the same as for the xpixelsize\n and ypixelsize attributes. The space spanned by the n- and m-dimensions\n of R must be a multiple of space_window. If set to None, the function\n returns a copy of the original R and metadata.\n ignore_nan: bool, optional\n If True, ignore nan values.\n\n Returns\n -------\n outputarray: array-like\n The new array of aggregated fields of shape (k,j), (t,k,j) or (l,t,k,j),\n where k = m*ypixelsize/space_window[1] and j = n*xpixelsize/space_window[0].\n metadata: dict\n The metadata with updated attributes.\n\n See also\n --------\n pysteps.utils.dimension.aggregate_fields_time,\n pysteps.utils.dimension.aggregate_fields\n \"\"\"\n\n R = R.copy()\n metadata = metadata.copy()\n\n if space_window is None:\n return R, metadata\n\n unit = metadata[\"unit\"]\n ypixelsize = metadata[\"ypixelsize\"]\n xpixelsize = metadata[\"xpixelsize\"]\n\n if len(R.shape) < 2:\n raise ValueError(\"The number of dimensions must be >= 2\")\n if len(R.shape) == 2:\n axes = [0, 1]\n if len(R.shape) == 3:\n axes = [1, 2]\n if len(R.shape) == 4:\n axes = [2, 3]\n if len(R.shape) > 4:\n raise ValueError(\"The number of dimensions must be <= 4\")\n\n if np.isscalar(space_window):\n space_window = (space_window, space_window)\n\n # assumes that frames are evenly spaced\n if ypixelsize == space_window[1] and xpixelsize == space_window[0]:\n return R, metadata\n\n ysize = R.shape[axes[0]] * ypixelsize\n xsize = R.shape[axes[1]] * xpixelsize\n\n if (\n abs(ysize / space_window[1] - round(ysize / space_window[1])) > 1e-10\n or abs(xsize / space_window[0] - round(xsize / space_window[0])) > 1e-10\n ):\n raise ValueError(\"space_window does not equally split R\")\n\n nframes = [int(space_window[1] / ypixelsize), int(space_window[0] / xpixelsize)]\n\n # specify the operator to be used to aggregate the values\n # within the space window\n if unit == \"mm/h\" or unit == \"mm\":\n method = \"mean\"\n else:\n raise ValueError(\n \"can only aggregate units of 'mm/h' or 'mm' \" + \"not %s\" % unit\n )\n\n if ignore_nan:\n method = \"\".join((\"nan\", method))\n\n R = aggregate_fields(R, nframes[0], axis=axes[0], method=method)\n R = aggregate_fields(R, nframes[1], axis=axes[1], method=method)\n\n metadata[\"ypixelsize\"] = space_window[1]\n metadata[\"xpixelsize\"] = space_window[0]\n\n return R, metadata\n\n\ndef aggregate_fields(data, window_size, axis=0, method=\"mean\", trim=False):\n \"\"\"Aggregate fields along a given direction.\n\n It attempts to aggregate the given R axis in an integer number of sections\n of length = ``window_size``.\n If such a aggregation is not possible, an error is raised unless ``trim``\n set to True, in which case the axis is trimmed (from the end)\n to make it perfectly divisible\".\n\n Parameters\n ----------\n data: array-like\n Array of any shape containing the input fields.\n window_size: int or tuple of ints\n The length of the window that is used to aggregate the fields.\n If a single integer value is given, the same window is used for\n all the selected axis.\n\n If ``window_size`` is a 1D array-like,\n each element indicates the length of the window that is used\n to aggregate the fields along each axis. In this case,\n the number of elements of 'window_size' must be the same as the elements\n in the ``axis`` argument.\n axis: int or array-like of ints\n Axis or axes where to perform the aggregation.\n If this is a tuple of ints, the aggregation is performed over multiple\n axes, instead of a single axis\n method: string, optional\n Optional argument that specifies the operation to use\n to aggregate the values within the window.\n Default to mean operator.\n trim: bool\n In case that the ``data`` is not perfectly divisible by\n ``window_size`` along the selected axis:\n\n - trim=True: the data will be trimmed (from the end) along that\n axis to make it perfectly divisible.\n - trim=False: a ValueError exception is raised.\n\n Returns\n -------\n new_array: array-like\n The new aggregated array with shape[axis] = k,\n where k = R.shape[axis] / window_size.\n\n See also\n --------\n pysteps.utils.dimension.aggregate_fields_time,\n pysteps.utils.dimension.aggregate_fields_space\n \"\"\"\n\n if np.ndim(axis) > 1:\n raise TypeError(\n \"Only integers or integer 1D arrays can be used for the \" \"'axis' argument.\"\n )\n\n if np.ndim(axis) == 1:\n axis = np.asarray(axis)\n if np.ndim(window_size) == 0:\n window_size = (window_size,) * axis.size\n\n window_size = np.asarray(window_size, dtype=\"int\")\n\n if window_size.shape != axis.shape:\n raise ValueError(\n \"The 'window_size' and 'axis' shapes are incompatible.\"\n f\"window_size.shape: {str(window_size.shape)}, \"\n f\"axis.shape: {str(axis.shape)}, \"\n )\n\n new_data = data.copy()\n for i in range(axis.size):\n # Recursively call the aggregate_fields function\n new_data = aggregate_fields(\n new_data, window_size[i], axis=axis[i], method=method, trim=trim\n )\n\n return new_data\n\n if np.ndim(window_size) != 0:\n raise TypeError(\n \"A single axis was selected for the aggregation but several\"\n f\"of window_sizes were given: {str(window_size)}.\"\n )\n\n data = np.asarray(data).copy()\n orig_shape = data.shape\n\n if method not in _aggregation_methods:\n raise ValueError(\n \"Aggregation method not recognized. \"\n f\"Available methods: {str(list(_aggregation_methods.keys()))}\"\n )\n\n if window_size <= 0:\n raise ValueError(\"'window_size' must be strictly positive\")\n\n if (orig_shape[axis] % window_size) and (not trim):\n raise ValueError(\n f\"Since 'trim' argument was set to False,\"\n f\"the 'window_size' {window_size} must exactly divide\"\n f\"the dimension along the selected axis:\"\n f\"data.shape[axis]={orig_shape[axis]}\"\n )\n\n new_data = data.swapaxes(axis, 0)\n if trim:\n trim_size = data.shape[axis] % window_size\n if trim_size > 0:\n new_data = new_data[:-trim_size]\n\n new_data_shape = list(new_data.shape)\n new_data_shape[0] //= window_size # Final shape\n\n new_data = new_data.reshape(new_data_shape[0], window_size, -1)\n\n new_data = _aggregation_methods[method](new_data, axis=1)\n\n new_data = new_data.reshape(new_data_shape).swapaxes(axis, 0)\n\n return new_data\n\n\ndef clip_domain(R, metadata, extent=None):\n \"\"\"\n Clip the field domain by geographical coordinates.\n\n Parameters\n ----------\n R: array-like\n Array of shape (m,n) or (t,m,n) containing the input fields.\n metadata: dict\n Metadata dictionary containing the x1, x2, y1, y2,\n xpixelsize, ypixelsize,\n zerovalue and yorigin attributes as described in the documentation of\n :py:mod:`pysteps.io.importers`.\n extent: scalars (left, right, bottom, top), optional\n The extent of the bounding box in data coordinates to be used to clip\n the data.\n Note that the direction of the vertical axis and thus the default\n values for top and bottom depend on origin. We follow the same\n convention as in the imshow method of matplotlib:\n https://matplotlib.org/tutorials/intermediate/imshow_extent.html\n\n Returns\n -------\n R: array-like\n the clipped array\n metadata: dict\n the metadata with updated attributes.\n \"\"\"\n\n R = R.copy()\n R_shape = np.array(R.shape)\n metadata = metadata.copy()\n\n if extent is None:\n return R, metadata\n\n if len(R.shape) < 2:\n raise ValueError(\"The number of dimension must be > 1\")\n if len(R.shape) == 2:\n R = R[None, None, :, :]\n if len(R.shape) == 3:\n R = R[None, :, :, :]\n if len(R.shape) > 4:\n raise ValueError(\"The number of dimension must be <= 4\")\n\n # extract original domain coordinates\n left = metadata[\"x1\"]\n right = metadata[\"x2\"]\n bottom = metadata[\"y1\"]\n top = metadata[\"y2\"]\n\n # extract bounding box coordinates\n left_ = extent[0]\n right_ = extent[1]\n bottom_ = extent[2]\n top_ = extent[3]\n\n # compute its extent in pixels\n dim_x_ = int((right_ - left_) / metadata[\"xpixelsize\"])\n dim_y_ = int((top_ - bottom_) / metadata[\"ypixelsize\"])\n R_ = np.ones((R.shape[0], R.shape[1], dim_y_, dim_x_)) * metadata[\"zerovalue\"]\n\n # build set of coordinates for the original domain\n y_coord = (\n np.linspace(bottom, top - metadata[\"ypixelsize\"], R.shape[2])\n + metadata[\"ypixelsize\"] / 2.0\n )\n x_coord = (\n np.linspace(left, right - metadata[\"xpixelsize\"], R.shape[3])\n + metadata[\"xpixelsize\"] / 2.0\n )\n\n # build set of coordinates for the new domain\n y_coord_ = (\n np.linspace(bottom_, top_ - metadata[\"ypixelsize\"], R_.shape[2])\n + metadata[\"ypixelsize\"] / 2.0\n )\n x_coord_ = (\n np.linspace(left_, right_ - metadata[\"xpixelsize\"], R_.shape[3])\n + metadata[\"xpixelsize\"] / 2.0\n )\n\n # origin='upper' reverses the vertical axes direction\n if metadata[\"yorigin\"] == \"upper\":\n y_coord = y_coord[::-1]\n y_coord_ = y_coord_[::-1]\n\n # extract original domain\n idx_y = np.where(np.logical_and(y_coord < top_, y_coord > bottom_))[0]\n idx_x = np.where(np.logical_and(x_coord < right_, x_coord > left_))[0]\n\n # extract new domain\n idx_y_ = np.where(np.logical_and(y_coord_ < top, y_coord_ > bottom))[0]\n idx_x_ = np.where(np.logical_and(x_coord_ < right, x_coord_ > left))[0]\n\n # compose the new array\n R_[:, :, idx_y_[0] : (idx_y_[-1] + 1), idx_x_[0] : (idx_x_[-1] + 1)] = R[\n :, :, idx_y[0] : (idx_y[-1] + 1), idx_x[0] : (idx_x[-1] + 1)\n ]\n\n # update coordinates\n metadata[\"y1\"] = bottom_\n metadata[\"y2\"] = top_\n metadata[\"x1\"] = left_\n metadata[\"x2\"] = right_\n\n R_shape[-2] = R_.shape[-2]\n R_shape[-1] = R_.shape[-1]\n\n return R_.reshape(R_shape), metadata\n\n\ndef square_domain(R, metadata, method=\"pad\", inverse=False):\n \"\"\"\n Either pad or crop a field to obtain a square domain.\n\n Parameters\n ----------\n R: array-like\n Array of shape (m,n) or (t,m,n) containing the input fields.\n metadata: dict\n Metadata dictionary containing the x1, x2, y1, y2,\n xpixelsize, ypixelsize,\n attributes as described in the documentation of\n :py:mod:`pysteps.io.importers`.\n method: {'pad', 'crop'}, optional\n Either pad or crop.\n If pad, an equal number of zeros is added to both ends of its shortest\n side in order to produce a square domain.\n If crop, an equal number of pixels is removed\n to both ends of its longest side in order to produce a square domain.\n Note that the crop method involves an irreversible loss of data.\n inverse: bool, optional\n Perform the inverse method to recover the original domain shape.\n After a crop, the inverse is performed by padding the field with zeros.\n\n Returns\n -------\n R: array-like\n the reshape dataset\n metadata: dict\n the metadata with updated attributes.\n \"\"\"\n\n R = R.copy()\n R_shape = np.array(R.shape)\n metadata = metadata.copy()\n\n if not inverse:\n if len(R.shape) < 2:\n raise ValueError(\"The number of dimension must be > 1\")\n if len(R.shape) == 2:\n R = R[None, None, :]\n if len(R.shape) == 3:\n R = R[None, :]\n if len(R.shape) > 4:\n raise ValueError(\"The number of dimension must be <= 4\")\n\n if R.shape[2] == R.shape[3]:\n return R.squeeze()\n\n orig_dim = R.shape\n orig_dim_n = orig_dim[0]\n orig_dim_t = orig_dim[1]\n orig_dim_y = orig_dim[2]\n orig_dim_x = orig_dim[3]\n\n if method == \"pad\":\n new_dim = np.max(orig_dim[2:])\n R_ = np.ones((orig_dim_n, orig_dim_t, new_dim, new_dim)) * R.min()\n\n if orig_dim_x < new_dim:\n idx_buffer = int((new_dim - orig_dim_x) / 2.0)\n R_[:, :, :, idx_buffer : (idx_buffer + orig_dim_x)] = R\n metadata[\"x1\"] -= idx_buffer * metadata[\"xpixelsize\"]\n metadata[\"x2\"] += idx_buffer * metadata[\"xpixelsize\"]\n\n elif orig_dim_y < new_dim:\n idx_buffer = int((new_dim - orig_dim_y) / 2.0)\n R_[:, :, idx_buffer : (idx_buffer + orig_dim_y), :] = R\n metadata[\"y1\"] -= idx_buffer * metadata[\"ypixelsize\"]\n metadata[\"y2\"] += idx_buffer * metadata[\"ypixelsize\"]\n\n elif method == \"crop\":\n new_dim = np.min(orig_dim[2:])\n R_ = np.zeros((orig_dim_n, orig_dim_t, new_dim, new_dim))\n\n if orig_dim_x > new_dim:\n idx_buffer = int((orig_dim_x - new_dim) / 2.0)\n R_ = R[:, :, :, idx_buffer : (idx_buffer + new_dim)]\n metadata[\"x1\"] += idx_buffer * metadata[\"xpixelsize\"]\n metadata[\"x2\"] -= idx_buffer * metadata[\"xpixelsize\"]\n\n elif orig_dim_y > new_dim:\n idx_buffer = int((orig_dim_y - new_dim) / 2.0)\n R_ = R[:, :, idx_buffer : (idx_buffer + new_dim), :]\n metadata[\"y1\"] += idx_buffer * metadata[\"ypixelsize\"]\n metadata[\"y2\"] -= idx_buffer * metadata[\"ypixelsize\"]\n\n else:\n raise ValueError(\"Unknown type\")\n\n metadata[\"orig_domain\"] = (orig_dim_y, orig_dim_x)\n metadata[\"square_method\"] = method\n\n R_shape[-2] = R_.shape[-2]\n R_shape[-1] = R_.shape[-1]\n\n return R_.reshape(R_shape), metadata\n\n elif inverse:\n if len(R.shape) < 2:\n raise ValueError(\"The number of dimension must be > 2\")\n if len(R.shape) == 2:\n R = R[None, None, :]\n if len(R.shape) == 3:\n R = R[None, :]\n if len(R.shape) > 4:\n raise ValueError(\"The number of dimension must be <= 4\")\n\n method = metadata.pop(\"square_method\")\n shape = metadata.pop(\"orig_domain\")\n\n if R.shape[2] == shape[0] and R.shape[3] == shape[1]:\n return R.squeeze(), metadata\n\n R_ = np.zeros((R.shape[0], R.shape[1], shape[0], shape[1]))\n\n if method == \"pad\":\n if R.shape[2] == shape[0]:\n idx_buffer = int((R.shape[3] - shape[1]) / 2.0)\n R_ = R[:, :, :, idx_buffer : (idx_buffer + shape[1])]\n metadata[\"x1\"] += idx_buffer * metadata[\"xpixelsize\"]\n metadata[\"x2\"] -= idx_buffer * metadata[\"xpixelsize\"]\n\n elif R.shape[3] == shape[1]:\n idx_buffer = int((R.shape[2] - shape[0]) / 2.0)\n R_ = R[:, :, idx_buffer : (idx_buffer + shape[0]), :]\n metadata[\"y1\"] += idx_buffer * metadata[\"ypixelsize\"]\n metadata[\"y2\"] -= idx_buffer * metadata[\"ypixelsize\"]\n\n elif method == \"crop\":\n if R.shape[2] == shape[0]:\n idx_buffer = int((shape[1] - R.shape[3]) / 2.0)\n R_[:, :, :, idx_buffer : (idx_buffer + R.shape[3])] = R\n metadata[\"x1\"] -= idx_buffer * metadata[\"xpixelsize\"]\n metadata[\"x2\"] += idx_buffer * metadata[\"xpixelsize\"]\n\n elif R.shape[3] == shape[1]:\n idx_buffer = int((shape[0] - R.shape[2]) / 2.0)\n R_[:, :, idx_buffer : (idx_buffer + R.shape[2]), :] = R\n metadata[\"y1\"] -= idx_buffer * metadata[\"ypixelsize\"]\n metadata[\"y2\"] += idx_buffer * metadata[\"ypixelsize\"]\n\n R_shape[-2] = R_.shape[-2]\n R_shape[-1] = R_.shape[-1]\n\n return R_.reshape(R_shape), metadata\n" }, { "path": "pysteps/utils/fft.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.utils.fft\n=================\n\nInterface module for different FFT methods.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_numpy\n get_scipy\n get_pyfftw\n\"\"\"\n\nfrom pysteps.exceptions import MissingOptionalDependency\nfrom types import SimpleNamespace\n\n\ndef get_numpy(shape, fftn_shape=None, **kwargs):\n import numpy.fft as numpy_fft\n\n f = {\n \"fft2\": numpy_fft.fft2,\n \"ifft2\": numpy_fft.ifft2,\n \"rfft2\": numpy_fft.rfft2,\n \"irfft2\": lambda X: numpy_fft.irfft2(X, s=shape),\n \"fftshift\": numpy_fft.fftshift,\n \"ifftshift\": numpy_fft.ifftshift,\n \"fftfreq\": numpy_fft.fftfreq,\n }\n if fftn_shape is not None:\n f[\"fftn\"] = numpy_fft.fftn\n fft = SimpleNamespace(**f)\n\n return fft\n\n\ndef get_scipy(shape, fftn_shape=None, **kwargs):\n import numpy.fft as numpy_fft\n import scipy.fftpack as scipy_fft\n\n # use numpy implementation of rfft2/irfft2 because they have not been\n # implemented in scipy.fftpack\n f = {\n \"fft2\": scipy_fft.fft2,\n \"ifft2\": scipy_fft.ifft2,\n \"rfft2\": numpy_fft.rfft2,\n \"irfft2\": lambda X: numpy_fft.irfft2(X, s=shape),\n \"fftshift\": scipy_fft.fftshift,\n \"ifftshift\": scipy_fft.ifftshift,\n \"fftfreq\": scipy_fft.fftfreq,\n }\n if fftn_shape is not None:\n f[\"fftn\"] = scipy_fft.fftn\n fft = SimpleNamespace(**f)\n\n return fft\n\n\ndef get_pyfftw(shape, fftn_shape=None, n_threads=1, **kwargs):\n try:\n import pyfftw.interfaces.numpy_fft as pyfftw_fft\n import pyfftw\n\n pyfftw.interfaces.cache.enable()\n except ImportError:\n raise MissingOptionalDependency(\"pyfftw is required but not installed\")\n\n X = pyfftw.empty_aligned(shape, dtype=\"complex128\")\n F = pyfftw.empty_aligned(shape, dtype=\"complex128\")\n\n fft_obj = pyfftw.FFTW(\n X,\n F,\n flags=[\"FFTW_ESTIMATE\"],\n direction=\"FFTW_FORWARD\",\n axes=(0, 1),\n threads=n_threads,\n )\n ifft_obj = pyfftw.FFTW(\n F,\n X,\n flags=[\"FFTW_ESTIMATE\"],\n direction=\"FFTW_BACKWARD\",\n axes=(0, 1),\n threads=n_threads,\n )\n\n if fftn_shape is not None:\n X = pyfftw.empty_aligned(fftn_shape, dtype=\"complex128\")\n F = pyfftw.empty_aligned(fftn_shape, dtype=\"complex128\")\n\n fftn_obj = pyfftw.FFTW(\n X,\n F,\n flags=[\"FFTW_ESTIMATE\"],\n direction=\"FFTW_FORWARD\",\n axes=list(range(len(fftn_shape))),\n threads=n_threads,\n )\n\n X = pyfftw.empty_aligned(shape, dtype=\"float64\")\n output_shape = list(shape[:-1])\n output_shape.append(int(shape[-1] / 2) + 1)\n output_shape = tuple(output_shape)\n F = pyfftw.empty_aligned(output_shape, dtype=\"complex128\")\n\n rfft_obj = pyfftw.FFTW(\n X,\n F,\n flags=[\"FFTW_ESTIMATE\"],\n direction=\"FFTW_FORWARD\",\n axes=(0, 1),\n threads=n_threads,\n )\n irfft_obj = pyfftw.FFTW(\n F,\n X,\n flags=[\"FFTW_ESTIMATE\"],\n direction=\"FFTW_BACKWARD\",\n axes=(0, 1),\n threads=n_threads,\n )\n\n f = {\n \"fft2\": lambda X: fft_obj(input_array=X.copy()).copy(),\n \"ifft2\": lambda X: ifft_obj(input_array=X.copy()).copy(),\n \"rfft2\": lambda X: rfft_obj(input_array=X.copy()).copy(),\n \"irfft2\": lambda X: irfft_obj(input_array=X.copy()).copy(),\n \"fftshift\": pyfftw_fft.fftshift,\n \"ifftshift\": pyfftw_fft.ifftshift,\n \"fftfreq\": pyfftw_fft.fftfreq,\n }\n\n if fftn_shape is not None:\n f[\"fftn\"] = lambda X: fftn_obj(input_array=X).copy()\n fft = SimpleNamespace(**f)\n\n return fft\n" }, { "path": "pysteps/utils/images.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\npysteps.utils.images\r\n====================\r\n\r\nImage processing routines for pysteps.\r\n\r\n.. autosummary::\r\n :toctree: ../generated/\r\n\r\n morph_opening\r\n\"\"\"\r\n\r\nimport numpy as np\r\nfrom numpy.ma.core import MaskedArray\r\n\r\nfrom pysteps.exceptions import MissingOptionalDependency\r\n\r\ntry:\r\n import cv2\r\n\r\n CV2_IMPORTED = True\r\nexcept ImportError:\r\n CV2_IMPORTED = False\r\n\r\n\r\ndef morph_opening(input_image, thr, n):\r\n \"\"\"\r\n Filter out small scale noise on the image by applying a binary\r\n morphological opening, that is, erosion followed by dilation.\r\n\r\n .. _MaskedArray:\\\r\n https://docs.scipy.org/doc/numpy/reference/maskedarray.baseclass.html#numpy.ma.MaskedArray\r\n\r\n .. _ndarray:\\\r\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\r\n\r\n Parameters\r\n ----------\r\n input_image: ndarray_ or MaskedArray_\r\n Array of shape (m, n) containing the input image.\r\n thr: float\r\n The threshold used to convert the image into a binary image.\r\n n: int\r\n The structuring element size [pixels].\r\n\r\n Returns\r\n -------\r\n input_image: ndarray_ or MaskedArray_\r\n Array of shape (m,n) containing the filtered image.\r\n \"\"\"\r\n if not CV2_IMPORTED:\r\n raise MissingOptionalDependency(\r\n \"opencv package is required for the morphologyEx \"\r\n \"routine but it is not installed\"\r\n )\r\n\r\n input_image = input_image.copy()\r\n\r\n # Check if a MaskedArray is used. If not, mask the ndarray\r\n to_ndarray = False\r\n if not isinstance(input_image, MaskedArray):\r\n to_ndarray = True\r\n input_image = np.ma.masked_invalid(input_image)\r\n\r\n np.ma.set_fill_value(input_image, input_image.min())\r\n\r\n # Convert to binary image\r\n field_bin = np.ndarray.astype(input_image.filled() > thr, \"uint8\")\r\n\r\n # Build a structuring element of size n\r\n kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (n, n))\r\n\r\n # Apply morphological opening (i.e. erosion then dilation)\r\n field_bin_out = cv2.morphologyEx(field_bin, cv2.MORPH_OPEN, kernel)\r\n\r\n # Build mask to be applied on the original image\r\n mask = (field_bin - field_bin_out) > 0\r\n\r\n # Filter out small isolated pixels based on mask\r\n input_image[mask] = np.nanmin(input_image)\r\n\r\n if to_ndarray:\r\n input_image = np.array(input_image)\r\n\r\n return input_image\r\n" }, { "path": "pysteps/utils/interface.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.utils.interface\n=======================\n\nInterface for the utils module.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\nfrom . import arrays\nfrom . import cleansing\nfrom . import conversion\nfrom . import dimension\nfrom . import fft\nfrom . import images\nfrom . import interpolate\nfrom . import pca\nfrom . import reprojection\nfrom . import spectral\nfrom . import tapering\nfrom . import transformation\n\n\ndef get_method(name, **kwargs):\n \"\"\"\n Return a callable function for the utility method corresponding to the\n given name.\\n\\\n\n Arrays methods:\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | centred_coord | compute a 2D coordinate array |\n +-------------------+-----------------------------------------------------+\n\n Cleansing methods:\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | decluster | decluster a set of sparse data points |\n +-------------------+-----------------------------------------------------+\n | detect_outliers | detect outliers in a dataset |\n +-------------------+-----------------------------------------------------+\n\n Conversion methods:\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | mm/h or rainrate | convert to rain rate [mm/h] |\n +-------------------+-----------------------------------------------------+\n | mm or raindepth | convert to rain depth [mm] |\n +-------------------+-----------------------------------------------------+\n | dbz or | convert to reflectivity [dBZ] |\n | reflectivity | |\n +-------------------+-----------------------------------------------------+\n\n Dimension methods:\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | accumulate | aggregate fields in time |\n +-------------------+-----------------------------------------------------+\n | clip | resize the field domain by geographical coordinates |\n +-------------------+-----------------------------------------------------+\n | square | either pad or crop the data to get a square domain |\n +-------------------+-----------------------------------------------------+\n | upscale | upscale the field |\n +-------------------+-----------------------------------------------------+\n\n FFT methods (wrappers to different implementations):\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | numpy | numpy.fft |\n +-------------------+-----------------------------------------------------+\n | scipy | scipy.fftpack |\n +-------------------+-----------------------------------------------------+\n | pyfftw | pyfftw.interfaces.numpy_fft |\n +-------------------+-----------------------------------------------------+\n\n Image processing methods:\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | morph_opening | filter small scale noise |\n +-------------------+-----------------------------------------------------+\n\n Interpolation methods:\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | rbfinterp2d | Radial Basis Function (RBF) interpolation of a |\n | | (multivariate) array over a 2D grid. |\n +-------------------+-----------------------------------------------------+\n | idwinterp2d | Inverse distance weighting (IDW) interpolation of a |\n | | (multivariate) array over a 2D grid. |\n +-------------------+-----------------------------------------------------+\n\n Additional keyword arguments are passed to the initializer of the FFT\n methods, see utils.fft.\n\n Principal component analysis methods:\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | pca_transform | Transform a two-dimensional array into principal |\n | | component analysis |\n +-------------------+-----------------------------------------------------+\n | pca_backtransform | Transform a given principal component trans- |\n | | formation back into physical space |\n +-------------------+-----------------------------------------------------+ \n\n Reprojection methods:\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | reproject_grids | Reproject grids to a destination grid. |\n +-------------------+-----------------------------------------------------+\n\n Spectral methods:\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | rapsd | Compute radially averaged power spectral density |\n +-------------------+-----------------------------------------------------+\n | rm_rdisc | remove the rain / no-rain discontinuity |\n +-------------------+-----------------------------------------------------+\n\n Tapering methods:\n\n +-------------------------------+-----------------------------------------+\n | Name | Description |\n +===============================+=========================================+\n | compute_mask_window_function | Compute window function for a |\n | | two-dimensional area defined by a |\n | | non-rectangular mask. |\n +-------------------------------+-----------------------------------------+\n | compute_window_function | Compute window function for a |\n | | two-dimensional rectangular area. |\n +-------------------------------+-----------------------------------------+\n\n Transformation methods:\n\n +-------------------+-----------------------------------------------------+\n | Name | Description |\n +===================+=====================================================+\n | boxcox or box-cox | one-parameter Box-Cox transform |\n +-------------------+-----------------------------------------------------+\n | db or decibel | transform to units of decibel |\n +-------------------+-----------------------------------------------------+\n | log | log transform |\n +-------------------+-----------------------------------------------------+\n | nqt | Normal Quantile Transform |\n +-------------------+-----------------------------------------------------+\n | sqrt | square-root transform |\n +-------------------+-----------------------------------------------------+\n\n \"\"\"\n\n if name is None:\n name = \"none\"\n\n name = name.lower()\n\n def donothing(R, metadata=None, *args, **kwargs):\n return R.copy(), {} if metadata is None else metadata.copy()\n\n methods_objects = dict()\n methods_objects[\"none\"] = donothing\n\n # arrays methods\n methods_objects[\"centred_coord\"] = arrays.compute_centred_coord_array\n\n # cleansing methods\n methods_objects[\"decluster\"] = cleansing.decluster\n methods_objects[\"detect_outliers\"] = cleansing.detect_outliers\n\n # conversion methods\n methods_objects[\"mm/h\"] = conversion.to_rainrate\n methods_objects[\"rainrate\"] = conversion.to_rainrate\n methods_objects[\"mm\"] = conversion.to_raindepth\n methods_objects[\"raindepth\"] = conversion.to_raindepth\n methods_objects[\"dbz\"] = conversion.to_reflectivity\n methods_objects[\"reflectivity\"] = conversion.to_reflectivity\n\n # dimension methods\n methods_objects[\"accumulate\"] = dimension.aggregate_fields_time\n methods_objects[\"clip\"] = dimension.clip_domain\n methods_objects[\"square\"] = dimension.square_domain\n methods_objects[\"upscale\"] = dimension.aggregate_fields_space\n\n # image processing methods\n methods_objects[\"morph_opening\"] = images.morph_opening\n\n # interpolation methods\n methods_objects[\"rbfinterp2d\"] = interpolate.rbfinterp2d\n methods_objects[\"idwinterp2d\"] = interpolate.idwinterp2d\n\n # pca methods\n methods_objects[\"pca_transform\"] = pca.pca_transform\n methods_objects[\"pca_backtransform\"] = pca.pca_backtransform\n\n # reprojection methods\n methods_objects[\"reproject_grids\"] = reprojection.reproject_grids\n\n # spectral methods\n methods_objects[\"rapsd\"] = spectral.rapsd\n methods_objects[\"rm_rdisc\"] = spectral.remove_rain_norain_discontinuity\n\n # tapering methods\n methods_objects[\"compute_mask_window_function\"] = (\n tapering.compute_mask_window_function\n )\n methods_objects[\"compute_window_function\"] = tapering.compute_window_function\n\n # transformation methods\n methods_objects[\"boxcox\"] = transformation.boxcox_transform\n methods_objects[\"box-cox\"] = transformation.boxcox_transform\n methods_objects[\"db\"] = transformation.dB_transform\n methods_objects[\"decibel\"] = transformation.dB_transform\n methods_objects[\"log\"] = transformation.boxcox_transform\n methods_objects[\"nqt\"] = transformation.NQ_transform\n methods_objects[\"sqrt\"] = transformation.sqrt_transform\n\n # FFT methods\n if name in [\"numpy\", \"pyfftw\", \"scipy\"]:\n if \"shape\" not in kwargs.keys():\n raise KeyError(\"mandatory keyword argument shape not given\")\n return _get_fft_method(name, **kwargs)\n else:\n try:\n return methods_objects[name]\n except KeyError as e:\n raise ValueError(\n \"Unknown method %s\\n\" % e\n + \"Supported methods:%s\" % str(methods_objects.keys())\n )\n\n\ndef _get_fft_method(name, **kwargs):\n kwargs = kwargs.copy()\n shape = kwargs[\"shape\"]\n kwargs.pop(\"shape\")\n\n if name == \"numpy\":\n return fft.get_numpy(shape, **kwargs)\n elif name == \"scipy\":\n return fft.get_scipy(shape, **kwargs)\n elif name == \"pyfftw\":\n return fft.get_pyfftw(shape, **kwargs)\n else:\n raise ValueError(\n \"Unknown method {}\\n\".format(name)\n + \"The available methods are:\"\n + str([\"numpy\", \"pyfftw\", \"scipy\"])\n ) from None\n" }, { "path": "pysteps/utils/interpolate.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\npysteps.utils.interpolate\r\n=========================\r\n\r\nInterpolation routines for pysteps.\r\n\r\n.. autosummary::\r\n :toctree: ../generated/\r\n\r\n idwinterp2d\r\n rbfinterp2d\r\n\"\"\"\r\n\r\nimport warnings\r\n\r\nimport numpy as np\r\nfrom scipy.interpolate import Rbf\r\nfrom scipy.spatial import cKDTree\r\nfrom scipy.spatial.distance import cdist\r\n\r\n\r\nfrom pysteps.decorators import memoize, prepare_interpolator\r\n\r\n\r\n@prepare_interpolator()\r\ndef idwinterp2d(\r\n xy_coord, values, xgrid, ygrid, power=0.5, k=20, dist_offset=0.5, **kwargs\r\n):\r\n \"\"\"\r\n Inverse distance weighting interpolation of a sparse (multivariate) array.\r\n\r\n .. _ndarray:\\\r\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\r\n\r\n Parameters\r\n ----------\r\n xy_coord: ndarray_\r\n Array of shape (n, 2) containing the coordinates of the data points in\r\n a 2-dimensional space.\r\n values: ndarray_\r\n Array of shape (n) or (n, m) containing the values of the data points,\r\n where *n* is the number of data points and *m* the number of co-located\r\n variables. All elements in ``values`` are required to be finite.\r\n xgrid, ygrid: ndarray_\r\n 1-D arrays representing the coordinates of the 2-D output grid.\r\n power: positive float, optional\r\n The power parameter used to compute the distance weights as\r\n ``weight = distance ** (-power)``.\r\n k: positive int or None, optional\r\n The number of nearest neighbours used for each target location.\r\n If set to None, it interpolates using all the data points at once.\r\n dist_offset: float, optional\r\n A small, positive constant that is added to distances to avoid zero\r\n values. It has units of pixels.\r\n\r\n Other Parameters\r\n ----------------\r\n {extra_kwargs_doc}\r\n\r\n Returns\r\n -------\r\n output_array: ndarray_\r\n The interpolated field(s) having shape (``ygrid.size``, ``xgrid.size``)\r\n or (*m*, ``ygrid.size``, ``xgrid.size``).\r\n \"\"\"\r\n if values.ndim == 1:\r\n nvar = 1\r\n values = values[:, None]\r\n\r\n elif values.ndim == 2:\r\n nvar = values.shape[1]\r\n\r\n npoints = values.shape[0]\r\n\r\n # generate the target grid\r\n xgridv, ygridv = np.meshgrid(xgrid, ygrid)\r\n gridv = np.column_stack((xgridv.ravel(), ygridv.ravel()))\r\n\r\n if k is not None:\r\n k = int(np.min((k, npoints)))\r\n tree = _cKDTree_cached(xy_coord, hkey=kwargs.get(\"hkey\", None))\r\n dist, inds = tree.query(gridv, k=k)\r\n if dist.ndim == 1:\r\n dist = dist[..., None]\r\n inds = inds[..., None]\r\n else:\r\n # use all points\r\n dist = cdist(xy_coord, gridv, \"euclidean\").transpose()\r\n inds = np.arange(npoints)[None, :] * np.ones((gridv.shape[0], npoints)).astype(\r\n int\r\n )\r\n\r\n # convert geographical distances to number of pixels\r\n x_res = np.gradient(xgrid)\r\n y_res = np.gradient(ygrid)\r\n mean_res = np.mean(np.abs([x_res.mean(), y_res.mean()]))\r\n dist /= mean_res\r\n\r\n # compute distance-based weights\r\n dist += dist_offset # avoid zero distances\r\n weights = 1 / np.power(dist, power)\r\n weights = weights / np.sum(weights, axis=1, keepdims=True)\r\n\r\n # interpolate\r\n output_array = np.sum(\r\n values[inds, :] * weights[..., None],\r\n axis=1,\r\n )\r\n\r\n # reshape to final grid size\r\n output_array = output_array.reshape(ygrid.size, xgrid.size, nvar)\r\n\r\n return np.moveaxis(output_array, -1, 0).squeeze()\r\n\r\n\r\n@prepare_interpolator()\r\ndef rbfinterp2d(xy_coord, values, xgrid, ygrid, **kwargs):\r\n \"\"\"\r\n Radial basis function interpolation of a sparse (multivariate) array.\r\n\r\n .. _ndarray:\\\r\n https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html\r\n .. _`scipy.interpolate.Rbf`:\\\r\n https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.Rbf.html\r\n\r\n This method wraps the `scipy.interpolate.Rbf`_ class.\r\n\r\n Parameters\r\n ----------\r\n xy_coord: ndarray_\r\n Array of shape (n, 2) containing the coordinates of the data points in\r\n a 2-dimensional space.\r\n values: ndarray_\r\n Array of shape (n) or (n, m) containing the values of the data points,\r\n where *n* is the number of data points and *m* the number of co-located\r\n variables. All values in ``values`` are required to be finite.\r\n xgrid, ygrid: ndarray_\r\n 1-D arrays representing the coordinates of the 2-D output grid.\r\n\r\n Other Parameters\r\n ----------------\r\n Any of the parameters from the original `scipy.interpolate.Rbf`_ class.\r\n {extra_kwargs_doc}\r\n\r\n Returns\r\n -------\r\n output_array: ndarray_\r\n The interpolated field(s) having shape (``ygrid.size``, ``xgrid.size``)\r\n or (*m*, ``ygrid.size``, ``xgrid.size``).\r\n \"\"\"\r\n deprecated_args = [\"rbfunction\", \"k\"]\r\n deprecated_args = [arg for arg in deprecated_args if arg in list(kwargs.keys())]\r\n if deprecated_args:\r\n warnings.warn(\r\n \"rbfinterp2d: The following keyword arguments are deprecated:\\n\"\r\n + str(deprecated_args),\r\n DeprecationWarning,\r\n )\r\n\r\n if values.ndim == 1:\r\n kwargs[\"mode\"] = \"1-D\"\r\n else:\r\n kwargs[\"mode\"] = \"N-D\"\r\n\r\n xgridv, ygridv = np.meshgrid(xgrid, ygrid)\r\n rbfi = _Rbf_cached(*np.split(xy_coord, xy_coord.shape[1], 1), values, **kwargs)\r\n output_array = rbfi(xgridv, ygridv)\r\n\r\n return np.moveaxis(output_array, -1, 0).squeeze()\r\n\r\n\r\n@memoize()\r\ndef _cKDTree_cached(*args, **kwargs):\r\n \"\"\"Add LRU cache to cKDTree class.\"\"\"\r\n return cKDTree(*args)\r\n\r\n\r\n@memoize()\r\ndef _Rbf_cached(*args, **kwargs):\r\n \"\"\"Add LRU cache to Rbf class.\"\"\"\r\n return Rbf(*args, **kwargs)\r\n" }, { "path": "pysteps/utils/pca.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.utils.pca\n\nPrincipal component analysis for pysteps.\n\n.. autosummary::\n :toctree: ../generated/\n\n pca_transform\n pca_backtransform\n\"\"\"\n\nimport numpy as np\nfrom pysteps.exceptions import MissingOptionalDependency\n\ntry:\n from sklearn import decomposition\n\n SKLEARN_IMPORTED = True\nexcept ImportError:\n SKLEARN_IMPORTED = False\n\n\ndef pca_transform(\n forecast_ens: np.ndarray,\n mask: np.ndarray | None = None,\n pca_params: dict | None = None,\n get_params: bool = False,\n **kwargs: dict,\n):\n \"\"\"\n Transform ensemble forecasts from physical space into principal component (PC) space.\n\n Parameters\n ----------\n forecast_ens : np.ndarray\n Array of shape (n_ens, n_features) containing the ensemble forecasts\n in physical space.\n mask : np.ndarray, optional\n Mask to transform only grid points at which at least 10 ensemble\n members have forecast precipitation, to fulfill the Lien criterion\n (Lien et al., 2013) mentioned in Nerini et al., 2019.\n The default is None.\n pca_params : dict, optional\n Preconstructed Principal Component Analysis (PCA) object. If given,\n this is used instead of fitting a new PCA. The default is None.\n get_params : bool, optional\n If True, return the PCA parameters in addition to the transformed data.\n The default is False.\n n_components : int\n Number of principal components to retain.\n svd_solver : {'auto', 'full', 'covariance_eigh', 'arpack', 'randomized'}\n Solver to use for the singular value decomposition. For details, see\n the documentation of ``sklearn.decomposition.PCA``.\n\n Returns\n -------\n forecast_ens_pc : np.ndarray\n Array of shape (n_components, n_ens) containing the ensemble forecasts\n transformed into PC space. If no mask is given, the full dataset is\n transformed; otherwise only the mask-filtered values are transformed.\n pca_params : dict, optional\n Dictionary containing the PCA parameters, returned if\n ``get_params=True``. The dictionary has the following keys:\n\n principal_components : np.ndarray\n Array of shape (n_components, n_features) containing the\n principal component vectors in feature space.\n mean : np.ndarray\n Array of shape (n_features,) containing the per-feature\n empirical mean estimated from the input data.\n explained_variance : np.ndarray\n Array of shape (n_features,) containing the per-feature\n explained variance ratio.\n \"\"\"\n\n # Test import of sklean\n if not SKLEARN_IMPORTED:\n raise MissingOptionalDependency(\n \"scikit-learn package is required for principal component analysis \"\n \"but it is not installed\"\n )\n\n # Input data have to be two-dimensional\n if forecast_ens.ndim != 2:\n raise ValueError(\"Input array should be two-dimensional!\")\n\n if pca_params is None:\n # Check whether n_components and svd_solver are given as keyword arguments\n n_components = kwargs.get(\"n_components\", forecast_ens.shape[0])\n svd_solver = kwargs.get(\"svd_solver\", \"full\")\n\n # Initialize PCA and fit it to the input data\n pca = decomposition.PCA(n_components=n_components, svd_solver=svd_solver)\n pca.fit(forecast_ens)\n\n # Create output dictionary and save principal components and mean\n pca_params = {}\n pca_params[\"principal_components\"] = pca.components_\n pca_params[\"mean\"] = pca.mean_\n pca_params[\"explained_variance\"] = pca.explained_variance_ratio_\n\n else:\n # If output dict is given, check whether principal components and mean are included\n if not \"principal_components\" in pca_params.keys():\n raise KeyError(\"Output is not None but has no key 'principal_components'!\")\n if not \"mean\" in pca_params.keys():\n raise KeyError(\"Output is not None but has no key 'mean'!\")\n\n # Check whether PC and mean have the correct shape\n if forecast_ens.shape[1] != len(pca_params[\"mean\"]):\n raise ValueError(\"pca mean has not the same length as the input array!\")\n if forecast_ens.shape[1] != pca_params[\"principal_components\"].shape[1]:\n raise ValueError(\n \"principal components have not the same length as the input array\"\n )\n\n # If no mask is given, transform the full input data into PC space.\n if mask is None:\n forecast_ens_pc = np.dot(\n (forecast_ens - pca_params[\"mean\"]), pca_params[\"principal_components\"].T\n )\n else:\n forecast_ens_pc = np.dot(\n (forecast_ens[:, mask] - pca_params[\"mean\"][mask]),\n pca_params[\"principal_components\"][:, mask].T,\n )\n\n if get_params:\n return forecast_ens_pc, pca_params\n else:\n return forecast_ens_pc\n\n\ndef pca_backtransform(forecast_ens_pc: np.ndarray, pca_params: dict):\n \"\"\"\n Reconstruct ensemble forecasts from principal component (PC) space back into physical space.\n\n Parameters\n ----------\n forecast_ens_pc : np.ndarray\n Array of shape (n_components, n_ens) containing the ensemble forecasts\n represented in PC space.\n pca_params : dict\n Parameters of the PCA transformation. The dictionary contains the following keys:\n\n principal_components : np.ndarray\n Array of shape (n_components, n_features) containing the principal\n component vectors in feature space.\n mean : np.ndarray\n Array of shape (n_features,) containing the per-feature empirical mean\n estimated from the training data.\n\n Returns\n -------\n forecast_ens : np.ndarray\n Array of shape (n_ens, n_features) containing the ensemble forecasts\n reconstructed in physical space.\n \"\"\"\n\n # If output dict is given, check whether principal components and mean are included\n if not \"principal_components\" in pca_params.keys():\n raise KeyError(\"Output is not None but has no key 'principal_components'!\")\n if not \"mean\" in pca_params.keys():\n raise KeyError(\"Output is not None but has no key 'mean'!\")\n\n # Check whether PC and forecast_ens_pc have the correct shape\n if forecast_ens_pc.shape[1] != pca_params[\"principal_components\"].shape[0]:\n raise ValueError(\"pca mean has not the same length as the input array!\")\n\n # Transform forecast_ens_pc back into physical space.\n forecast_ens = (\n np.dot(forecast_ens_pc, pca_params[\"principal_components\"]) + pca_params[\"mean\"]\n )\n\n return forecast_ens\n" }, { "path": "pysteps/utils/reprojection.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.utils.reprojection\n==========================\n\nReprojection tools to reproject grids and adjust the grid cell size of an\ninput field to a destination field.\n\n.. autosummary::\n :toctree: ../generated/\n\n reproject_grids\n\"\"\"\n\nfrom pysteps.exceptions import MissingOptionalDependency\nfrom scipy.interpolate import griddata\n\nimport numpy as np\n\ntry:\n from rasterio import Affine as A\n from rasterio.warp import reproject, Resampling\n\n RASTERIO_IMPORTED = True\nexcept ImportError:\n RASTERIO_IMPORTED = False\n\ntry:\n import pyproj\n\n PYPROJ_IMPORTED = True\nexcept ImportError:\n PYPROJ_IMPORTED = False\n\n\ndef reproject_grids(src_array, dst_array, metadata_src, metadata_dst):\n \"\"\"\n Reproject precipitation fields to the domain of another precipitation field.\n\n Parameters\n ----------\n src_array: array-like\n Three-dimensional array of shape (t, x, y) containing a time series of\n precipitation fields. These precipitation fields will be reprojected.\n dst_array: array-like\n Array containing a precipitation field or a time series of precipitation\n fields. The src_array will be reprojected to the domain of\n dst_array.\n metadata_src: dict\n Metadata dictionary containing the projection, x- and ypixelsize, x1 and\n y2 attributes of the src_array as described in the documentation of\n :py:mod:`pysteps.io.importers`.\n metadata_dst: dict\n Metadata dictionary containing the projection, x- and ypixelsize, x1 and\n y2 attributes of the dst_array.\n\n Returns\n -------\n r_rprj: array-like\n Three-dimensional array of shape (t, x, y) containing the precipitation\n fields of src_array, but reprojected to the domain of dst_array.\n metadata: dict\n Metadata dictionary containing the projection, x- and ypixelsize, x1 and\n y2 attributes of the reprojected src_array.\n \"\"\"\n\n if not RASTERIO_IMPORTED:\n raise MissingOptionalDependency(\n \"rasterio package is required for the reprojection module, but it is \"\n \"not installed\"\n )\n\n # Extract the grid info from src_array\n src_crs = metadata_src[\"projection\"]\n x1_src = metadata_src[\"x1\"]\n y2_src = metadata_src[\"y2\"]\n xpixelsize_src = metadata_src[\"xpixelsize\"]\n ypixelsize_src = metadata_src[\"ypixelsize\"]\n src_transform = A.translation(float(x1_src), float(y2_src)) * A.scale(\n float(xpixelsize_src), float(-ypixelsize_src)\n )\n\n # Extract the grid info from dst_array\n dst_crs = metadata_dst[\"projection\"]\n x1_dst = metadata_dst[\"x1\"]\n y2_dst = metadata_dst[\"y2\"]\n xpixelsize_dst = metadata_dst[\"xpixelsize\"]\n ypixelsize_dst = metadata_dst[\"ypixelsize\"]\n dst_transform = A.translation(float(x1_dst), float(y2_dst)) * A.scale(\n float(xpixelsize_dst), float(-ypixelsize_dst)\n )\n\n # Initialise the reprojected array\n r_rprj = np.zeros((src_array.shape[0], dst_array.shape[-2], dst_array.shape[-1]))\n\n # For every timestep, reproject the precipitation field of src_array to\n # the domain of dst_array\n if metadata_src[\"yorigin\"] != metadata_dst[\"yorigin\"]:\n src_array = src_array[:, ::-1, :]\n\n for i in range(src_array.shape[0]):\n reproject(\n src_array[i, :, :],\n r_rprj[i, :, :],\n src_transform=src_transform,\n src_crs=src_crs,\n dst_transform=dst_transform,\n dst_crs=dst_crs,\n resampling=Resampling.nearest,\n dst_nodata=np.nan,\n )\n\n # Update the metadata\n metadata = metadata_src.copy()\n\n for key in [\n \"projection\",\n \"yorigin\",\n \"xpixelsize\",\n \"ypixelsize\",\n \"x1\",\n \"x2\",\n \"y1\",\n \"y2\",\n \"cartesian_unit\",\n ]:\n metadata[key] = metadata_dst[key]\n\n return r_rprj, metadata\n\n\ndef unstructured2regular(src_array, metadata_src, metadata_dst):\n \"\"\"\n Reproject unstructured data onto a regular grid on the assumption that\n both src data and dst grid have the same projection.\n\n Parameters\n ----------\n src_array: np.ndarray\n Three-dimensional array of shape (t, n_ens, n_gridcells) containing a\n time series of precipitation ensemble forecasts. These precipitation\n fields will be reprojected.\n metadata_src: dict\n Metadata dictionary containing the projection, clon, clat, and ngridcells\n and attributes of the src_array as described in the documentation of\n :py:mod:`pysteps.io.importers`.\n metadata_dst: dict\n Metadata dictionary containing the projection, x- and ypixelsize, x1 and\n y2 attributes of the dst_array.\n\n Returns\n -------\n tuple\n A tuple containing:\n - r_rprj: np.ndarray\n Four dimensional array of shape (t, n_ens, x, y) containing the\n precipitation fields of src_array, but reprojected to the grid\n of dst_array.\n - metadata: dict\n Dictionary containing geospatial metadat such as:\n - 'projection' : PROJ.4 string defining the stereographic projection.\n - 'xpixelsize', 'ypixelsize': Pixel size in meters.\n - 'x1', 'y1': Carthesian coordinates of the lower-left corner.\n - 'x2', 'y2': Carthesian coordinates of the upper-right corner.\n - 'cartesian_unit': Unit of the coordinate system (meters).\n \"\"\"\n\n if not PYPROJ_IMPORTED:\n raise MissingOptionalDependency(\n \"pyproj package is required to reproject DWD's NWP data\"\n \"but it is not installed\"\n )\n\n if not \"clon\" in metadata_src.keys():\n raise KeyError(\"Center longitude (clon) is missing in metadata_src\")\n if not \"clat\" in metadata_src.keys():\n raise KeyError(\"Center latitude (clat) is missing in metadata_src\")\n\n # Get number of grid cells\n Nc = metadata_src[\"clon\"].shape[0]\n ic_in = np.arange(Nc)\n\n # Get cartesian coordinates of destination grid\n x_dst = np.arange(\n np.float32(metadata_dst[\"x1\"]),\n np.float32(metadata_dst[\"x2\"]),\n metadata_dst[\"xpixelsize\"],\n )\n\n y_dst = np.arange(\n np.float32(metadata_dst[\"y1\"]),\n np.float32(metadata_dst[\"y2\"]),\n metadata_dst[\"ypixelsize\"],\n )\n\n # Create destination grid\n if metadata_dst[\"yorigin\"] == \"upper\":\n y_dst = y_dst[::-1]\n xx_dst, yy_dst = np.meshgrid(x_dst, y_dst)\n s_out = yy_dst.shape\n\n # Extract the grid info of src_array assuming the same projection of src and dst\n pr = pyproj.Proj(metadata_dst[\"projection\"])\n x_src, y_src = pr(metadata_src[\"clon\"], metadata_src[\"clat\"])\n\n # Create array of x-y pairs for interpolation\n P_in = np.stack((x_src, y_src)).T\n P_out = np.array((xx_dst.flatten(), yy_dst.flatten())).T\n\n # Nearest neighbor interpolation of x-y pairs\n ic_out = (\n griddata(P_in, ic_in.flatten(), P_out, method=\"nearest\")\n .reshape(s_out)\n .astype(int)\n )\n\n # Apply interpolation on all time steps and ensemble members\n r_rprj = np.array(\n [\n [src_array[i, j][ic_out] for j in range(src_array.shape[1])]\n for i in range(src_array.shape[0])\n ]\n )\n\n # Update the src metadata\n metadata = metadata_src.copy()\n\n for key in [\n \"projection\",\n \"yorigin\",\n \"xpixelsize\",\n \"ypixelsize\",\n \"x1\",\n \"x2\",\n \"y1\",\n \"y2\",\n \"cartesian_unit\",\n ]:\n metadata[key] = metadata_dst[key]\n\n return r_rprj, metadata\n" }, { "path": "pysteps/utils/spectral.py", "content": "\"\"\"\npysteps.utils.spectral\n======================\n\nUtility methods for processing and analyzing precipitation fields in the\nFourier domain.\n\n.. autosummary::\n :toctree: ../generated/\n\n corrcoef\n mean\n rapsd\n remove_rain_norain_discontinuity\n std\n\"\"\"\n\nimport numpy as np\nfrom . import arrays\n\n\ndef corrcoef(X, Y, shape, use_full_fft=False):\n \"\"\"\n Compute the correlation coefficient between two-dimensional arrays in\n the spectral domain.\n\n Parameters\n ----------\n X: array_like\n A complex array representing the Fourier transform of a two-dimensional\n array.\n Y: array_like\n A complex array representing the Fourier transform of a two-dimensional\n array.\n shape: tuple\n A two-element tuple specifying the shape of the original input arrays\n in the spatial domain.\n use_full_fft: bool\n If True, X and Y represent the full FFTs of the original arrays.\n Otherwise, they are assumed to contain only the symmetric part, i.e.\n in the format returned by numpy.fft.rfft2.\n\n Returns\n -------\n out: float\n The correlation coefficient. Gives the same result as\n numpy.corrcoef(X.flatten(), Y.flatten())[0, 1].\n \"\"\"\n if len(X.shape) != 2:\n raise ValueError(\"X is not a two-dimensional array\")\n\n if len(Y.shape) != 2:\n raise ValueError(\"Y is not a two-dimensional array\")\n\n if X.shape != Y.shape:\n raise ValueError(\n \"dimension mismatch between X and Y: \"\n + \"X.shape=%d,%d , \" % (X.shape[0], X.shape[1])\n + \"Y.shape=%d,%d\" % (Y.shape[0], Y.shape[1])\n )\n\n n = np.real(np.sum(X * np.conj(Y))) - np.real(X[0, 0] * Y[0, 0])\n d1 = np.sum(np.abs(X) ** 2) - np.real(X[0, 0]) ** 2\n d2 = np.sum(np.abs(Y) ** 2) - np.real(Y[0, 0]) ** 2\n\n if not use_full_fft:\n if shape[1] % 2 == 1:\n n += np.real(np.sum(X[:, 1:] * np.conj(Y[:, 1:])))\n d1 += np.sum(np.abs(X[:, 1:]) ** 2)\n d2 += np.sum(np.abs(Y[:, 1:]) ** 2)\n else:\n n += np.real(np.sum(X[:, 1:-1] * np.conj(Y[:, 1:-1])))\n d1 += np.sum(np.abs(X[:, 1:-1]) ** 2)\n d2 += np.sum(np.abs(Y[:, 1:-1]) ** 2)\n\n return n / np.sqrt(d1 * d2)\n\n\ndef mean(X, shape):\n \"\"\"\n Compute the mean value of a two-dimensional array in the spectral domain.\n\n Parameters\n ----------\n X: array_like\n A complex array representing the Fourier transform of a two-dimensional\n array.\n shape: tuple\n A two-element tuple specifying the shape of the original input array\n in the spatial domain.\n\n Returns\n -------\n out: float\n The mean value.\n \"\"\"\n return np.real(X[0, 0]) / (shape[0] * shape[1])\n\n\ndef rapsd(\n field, fft_method=None, return_freq=False, d=1.0, normalize=False, **fft_kwargs\n):\n \"\"\"\n Compute radially averaged power spectral density (RAPSD) from the given\n 2D input field.\n\n Parameters\n ----------\n field: array_like\n A 2d array of shape (m, n) containing the input field.\n fft_method: object\n A module or object implementing the same methods as numpy.fft and\n scipy.fftpack. If set to None, field is assumed to represent the\n shifted discrete Fourier transform of the input field, where the\n origin is at the center of the array\n (see numpy.fft.fftshift or scipy.fftpack.fftshift).\n return_freq: bool\n Whether to also return the Fourier frequencies.\n d: scalar\n Sample spacing (inverse of the sampling rate). Defaults to 1.\n Applicable if return_freq is 'True'.\n normalize: bool\n If True, normalize the power spectrum so that it sums to one.\n\n Returns\n -------\n out: ndarray\n One-dimensional array containing the RAPSD. The length of the array is\n int(l/2) (if l is even) or int(l/2)+1 (if l is odd), where l=max(m,n).\n freq: ndarray\n One-dimensional array containing the Fourier frequencies.\n\n References\n ----------\n :cite:`RC2011`\n \"\"\"\n\n if len(field.shape) != 2:\n raise ValueError(\n f\"{len(field.shape)} dimensions are found, but the number \"\n \"of dimensions should be 2\"\n )\n\n if np.sum(np.isnan(field)) > 0:\n raise ValueError(\"input field should not contain nans\")\n\n m, n = field.shape\n\n yc, xc = arrays.compute_centred_coord_array(m, n)\n r_grid = np.sqrt(xc * xc + yc * yc).round()\n l = max(field.shape[0], field.shape[1])\n\n if l % 2 == 1:\n r_range = np.arange(0, int(l / 2) + 1)\n else:\n r_range = np.arange(0, int(l / 2))\n\n if fft_method is not None:\n psd = fft_method.fftshift(fft_method.fft2(field, **fft_kwargs))\n psd = np.abs(psd) ** 2 / psd.size\n else:\n psd = field\n\n result = []\n for r in r_range:\n mask = r_grid == r\n psd_vals = psd[mask]\n result.append(np.mean(psd_vals))\n\n result = np.array(result)\n\n if normalize:\n result /= np.sum(result)\n\n if return_freq:\n freq = np.fft.fftfreq(l, d=d)\n freq = freq[r_range]\n return result, freq\n else:\n return result\n\n\ndef remove_rain_norain_discontinuity(R):\n \"\"\"Function to remove the rain/no-rain discontinuity.\n It can be used before computing Fourier filters to reduce\n the artificial increase of power at high frequencies\n caused by the discontinuity.\n\n Parameters\n ----------\n R: array-like\n Array of any shape to be transformed.\n\n Returns\n -------\n R: array-like\n Array of any shape containing the transformed data.\n \"\"\"\n R = R.copy()\n zerovalue = np.nanmin(R)\n threshold = np.nanmin(R[R > zerovalue])\n R[R > zerovalue] -= threshold - zerovalue\n R -= np.nanmin(R)\n\n return R\n\n\ndef std(X, shape, use_full_fft=False):\n \"\"\"\n Compute the standard deviation of a two-dimensional array in the\n spectral domain.\n\n Parameters\n ----------\n X: array_like\n A complex array representing the Fourier transform of a two-dimensional\n array.\n shape: tuple\n A two-element tuple specifying the shape of the original input array\n in the spatial domain.\n use_full_fft: bool\n If True, X represents the full FFT of the original array. Otherwise, it\n is assumed to contain only the symmetric part, i.e. in the format\n returned by numpy.fft.rfft2.\n\n Returns\n -------\n out: float\n The standard deviation.\n \"\"\"\n res = np.sum(np.abs(X) ** 2) - np.real(X[0, 0]) ** 2\n if not use_full_fft:\n if shape[1] % 2 == 1:\n res += np.sum(np.abs(X[:, 1:]) ** 2)\n else:\n res += np.sum(np.abs(X[:, 1:-1]) ** 2)\n\n return np.sqrt(res / (shape[0] * shape[1]) ** 2)\n" }, { "path": "pysteps/utils/tapering.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.utils.tapering\n======================\n\nImplementations of window functions for computing of the FFT.\n\n.. autosummary::\n :toctree: ../generated/\n\n compute_mask_window_function\n compute_window_function\n\"\"\"\n\nimport numpy as np\nfrom scipy.spatial import cKDTree\n\n\ndef compute_mask_window_function(mask, func, **kwargs):\n \"\"\"\n Compute window function for a two-dimensional area defined by a\n non-rectangular mask. The window function is computed based on the distance\n to the nearest boundary point of the mask. Window function-specific\n parameters are given as keyword arguments.\n\n Parameters\n ----------\n mask: array_like\n Two-dimensional boolean array containing the mask.\n Pixels with True/False are inside/outside the mask.\n func: str\n The name of the window function. The currently implemented function is\n 'tukey'.\n\n Returns\n -------\n out: array\n Array containing the tapering weights.\n \"\"\"\n R = _compute_mask_distances(mask)\n\n if func == \"hann\":\n raise NotImplementedError(\"Hann function has not been implemented\")\n elif func == \"tukey\":\n r_max = kwargs.get(\"r_max\", 10.0)\n\n return _tukey_masked(R, r_max, np.isfinite(R))\n else:\n raise ValueError(\"invalid window function '%s'\" % func)\n\n\ndef compute_window_function(m, n, func, **kwargs):\n \"\"\"\n Compute window function for a two-dimensional rectangular region. Window\n function-specific parameters are given as keyword arguments.\n\n Parameters\n ----------\n m: int\n Height of the array.\n n: int\n Width of the array.\n func: str\n The name of the window function.\n The currently implemented functions are\n 'hann' and 'tukey'.\n\n Other Parameters\n ----------------\n alpha: float\n Applicable if func is 'tukey'.\n\n Notes\n -----\n Two-dimensional tapering weights are computed from one-dimensional window\n functions using w(r), where r is the distance from the center of the\n region.\n\n Returns\n -------\n out: array\n Array of shape (m, n) containing the tapering weights.\n \"\"\"\n X, Y = np.meshgrid(np.arange(n), np.arange(m))\n R = np.sqrt(((X / n) - 0.5) ** 2 + ((Y / m) - 0.5) ** 2)\n\n if func == \"hann\":\n return _hann(R)\n elif func == \"tukey\":\n alpha = kwargs.get(\"alpha\", 0.2)\n\n return _tukey(R, alpha)\n else:\n raise ValueError(\"invalid window function '%s'\" % func)\n\n\ndef _compute_mask_distances(mask):\n X, Y = np.meshgrid(np.arange(mask.shape[1]), np.arange(mask.shape[0]))\n\n tree = cKDTree(np.vstack([X[~mask], Y[~mask]]).T)\n r, i = tree.query(np.vstack([X[mask], Y[mask]]).T, k=1)\n\n R = np.ones(mask.shape) * np.nan\n R[Y[mask], X[mask]] = r\n\n return R\n\n\ndef _hann(R):\n W = np.ones_like(R)\n mask = R > 0.5\n\n W[mask] = 0.0\n W[~mask] = 0.5 * (1.0 - np.cos(2.0 * np.pi * (R[~mask] + 0.5)))\n\n return W\n\n\ndef _tukey(R, alpha):\n W = np.ones_like(R)\n\n mask1 = R < 0.5\n mask2 = R > 0.5 * (1.0 - alpha)\n mask = np.logical_and(mask1, mask2)\n W[mask] = 0.5 * (\n 1.0 + np.cos(np.pi * (R[mask] / (alpha * 0.5) - 1.0 / alpha + 1.0))\n )\n mask = R >= 0.5\n W[mask] = 0.0\n\n return W\n\n\ndef _tukey_masked(R, r_max, mask):\n W = np.ones_like(R)\n\n mask_r = R < r_max\n mask_ = np.logical_and(mask, mask_r)\n W[mask_] = 0.5 * (1.0 + np.cos(np.pi * (R[mask_] / r_max - 1.0)))\n W[~mask] = np.nan\n\n return W\n" }, { "path": "pysteps/utils/transformation.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\npysteps.utils.transformation\r\n============================\r\n\r\nMethods for transforming data values.\r\n\r\n.. autosummary::\r\n :toctree: ../generated/\r\n\r\n boxcox_transform\r\n dB_transform\r\n NQ_transform\r\n sqrt_transform\r\n\"\"\"\r\n\r\nimport numpy as np\r\nimport scipy.stats as scipy_stats\r\nimport warnings\r\nfrom scipy.interpolate import interp1d\r\n\r\nwarnings.filterwarnings(\r\n \"ignore\", category=RuntimeWarning\r\n) # To deactivate warnings for comparison operators with NaNs\r\n\r\n\r\ndef boxcox_transform(\r\n R, metadata=None, Lambda=None, threshold=None, zerovalue=None, inverse=False\r\n):\r\n \"\"\"\r\n The one-parameter Box-Cox transformation.\r\n\r\n The Box-Cox transform is a well-known power transformation introduced by\r\n Box and Cox (1964). In its one-parameter version, the Box-Cox transform\r\n takes the form T(x) = ln(x) for Lambda = 0,\r\n or T(x) = (x**Lambda - 1)/Lambda otherwise.\r\n\r\n Default parameters will produce a log transform (i.e. Lambda=0).\r\n\r\n Parameters\r\n ----------\r\n R: array-like\r\n Array of any shape to be transformed.\r\n metadata: dict, optional\r\n Metadata dictionary containing the transform, zerovalue and threshold\r\n attributes as described in the documentation of\r\n :py:mod:`pysteps.io.importers`.\r\n Lambda: float, optional\r\n Parameter Lambda of the Box-Cox transformation.\r\n It is 0 by default, which produces the log transformation.\r\n\r\n Choose Lambda < 1 for positively skewed data, Lambda > 1 for negatively\r\n skewed data.\r\n threshold: float, optional\r\n The value that is used for thresholding with the same units as R.\r\n If None, the threshold contained in metadata is used.\r\n If no threshold is found in the metadata,\r\n a value of 0.1 is used as default.\r\n zerovalue: float, optional\r\n The value to be assigned to no rain pixels as defined by the threshold.\r\n It is equal to the threshold - 1 by default.\r\n inverse: bool, optional\r\n If set to True, it performs the inverse transform. False by default.\r\n\r\n Returns\r\n -------\r\n R: array-like\r\n Array of any shape containing the (back-)transformed units.\r\n metadata: dict\r\n The metadata with updated attributes.\r\n\r\n References\r\n ----------\r\n Box, G. E. and Cox, D. R. (1964), An Analysis of Transformations. Journal\r\n of the Royal Statistical Society: Series B (Methodological), 26: 211-243.\r\n doi:10.1111/j.2517-6161.1964.tb00553.x\r\n \"\"\"\r\n\r\n R = R.copy()\r\n\r\n if metadata is None:\r\n if inverse:\r\n metadata = {\"transform\": \"BoxCox\"}\r\n else:\r\n metadata = {\"transform\": None}\r\n\r\n else:\r\n metadata = metadata.copy()\r\n\r\n if not inverse:\r\n if metadata[\"transform\"] == \"BoxCox\":\r\n return R, metadata\r\n\r\n if Lambda is None:\r\n Lambda = metadata.get(\"BoxCox_lambda\", 0.0)\r\n\r\n if threshold is None:\r\n threshold = metadata.get(\"threshold\", 0.1)\r\n\r\n zeros = R < threshold\r\n\r\n # Apply Box-Cox transform\r\n if Lambda == 0.0:\r\n R[~zeros] = np.log(R[~zeros])\r\n threshold = np.log(threshold)\r\n\r\n else:\r\n R[~zeros] = (R[~zeros] ** Lambda - 1) / Lambda\r\n threshold = (threshold**Lambda - 1) / Lambda\r\n\r\n # Set value for zeros\r\n if zerovalue is None:\r\n zerovalue = threshold - 1 # TODO: set to a more meaningful value\r\n R[zeros] = zerovalue\r\n\r\n metadata[\"transform\"] = \"BoxCox\"\r\n metadata[\"BoxCox_lambda\"] = Lambda\r\n metadata[\"zerovalue\"] = zerovalue\r\n metadata[\"threshold\"] = threshold\r\n\r\n elif inverse:\r\n if metadata[\"transform\"] not in [\"BoxCox\", \"log\"]:\r\n return R, metadata\r\n\r\n if Lambda is None:\r\n Lambda = metadata.pop(\"BoxCox_lambda\", 0.0)\r\n if threshold is None:\r\n threshold = metadata.get(\"threshold\", -10.0)\r\n if zerovalue is None:\r\n zerovalue = 0.0\r\n\r\n # Apply inverse Box-Cox transform\r\n if Lambda == 0.0:\r\n R = np.exp(R)\r\n threshold = np.exp(threshold)\r\n\r\n else:\r\n R = np.exp(np.log(Lambda * R + 1) / Lambda)\r\n threshold = np.exp(np.log(Lambda * threshold + 1) / Lambda)\r\n\r\n R[R < threshold] = zerovalue\r\n\r\n metadata[\"transform\"] = None\r\n metadata[\"zerovalue\"] = zerovalue\r\n metadata[\"threshold\"] = threshold\r\n\r\n return R, metadata\r\n\r\n\r\ndef dB_transform(R, metadata=None, threshold=None, zerovalue=None, inverse=False):\r\n \"\"\"Methods to transform precipitation intensities to/from dB units.\r\n\r\n Parameters\r\n ----------\r\n R: array-like\r\n Array of any shape to be (back-)transformed.\r\n metadata: dict, optional\r\n Metadata dictionary containing the transform, zerovalue and threshold\r\n attributes as described in the documentation of\r\n :py:mod:`pysteps.io.importers`.\r\n threshold: float, optional\r\n Optional value that is used for thresholding with the same units as R.\r\n If None, the threshold contained in metadata is used.\r\n If no threshold is found in the metadata,\r\n a value of 0.1 is used as default.\r\n zerovalue: float, optional\r\n The value to be assigned to no rain pixels as defined by the threshold.\r\n It is equal to the threshold - 1 by default.\r\n inverse: bool, optional\r\n If set to True, it performs the inverse transform. False by default.\r\n\r\n Returns\r\n -------\r\n R: array-like\r\n Array of any shape containing the (back-)transformed units.\r\n metadata: dict\r\n The metadata with updated attributes.\r\n \"\"\"\r\n\r\n R = R.copy()\r\n\r\n if metadata is None:\r\n if inverse:\r\n metadata = {\"transform\": \"dB\"}\r\n else:\r\n metadata = {\"transform\": None}\r\n\r\n else:\r\n metadata = metadata.copy()\r\n\r\n # to dB units\r\n if not inverse:\r\n if metadata[\"transform\"] == \"dB\":\r\n return R, metadata\r\n\r\n if threshold is None:\r\n threshold = metadata.get(\"threshold\", 0.1)\r\n\r\n zeros = R < threshold\r\n\r\n # Convert to dB\r\n R[~zeros] = 10.0 * np.log10(R[~zeros])\r\n threshold = 10.0 * np.log10(threshold)\r\n\r\n # Set value for zeros\r\n if zerovalue is None:\r\n zerovalue = threshold - 5 # TODO: set to a more meaningful value\r\n R[zeros] = zerovalue\r\n\r\n metadata[\"transform\"] = \"dB\"\r\n metadata[\"zerovalue\"] = zerovalue\r\n metadata[\"threshold\"] = threshold\r\n\r\n return R, metadata\r\n\r\n # from dB units\r\n elif inverse:\r\n if metadata[\"transform\"] != \"dB\":\r\n return R, metadata\r\n\r\n if threshold is None:\r\n threshold = metadata.get(\"threshold\", -10.0)\r\n if zerovalue is None:\r\n zerovalue = 0.0\r\n\r\n R = 10.0 ** (R / 10.0)\r\n threshold = 10.0 ** (threshold / 10.0)\r\n R[R < threshold] = zerovalue\r\n\r\n metadata[\"transform\"] = None\r\n metadata[\"threshold\"] = threshold\r\n metadata[\"zerovalue\"] = zerovalue\r\n\r\n return R, metadata\r\n\r\n\r\ndef NQ_transform(R, metadata=None, inverse=False, **kwargs):\r\n \"\"\"\r\n The normal quantile transformation as in Bogner et al (2012).\r\n Zero rain vales are set to zero in norm space.\r\n\r\n Parameters\r\n ----------\r\n R: array-like\r\n Array of any shape to be transformed.\r\n metadata: dict, optional\r\n Metadata dictionary containing the transform, zerovalue and threshold\r\n attributes as described in the documentation of\r\n :py:mod:`pysteps.io.importers`.\r\n inverse: bool, optional\r\n If set to True, it performs the inverse transform. False by default.\r\n\r\n Other Parameters\r\n ----------------\r\n a: float, optional\r\n The offset fraction to be used for plotting positions;\r\n typically in (0,1).\r\n The default is 0., that is, it spaces the points evenly in the uniform\r\n distribution.\r\n\r\n Returns\r\n -------\r\n R: array-like\r\n Array of any shape containing the (back-)transformed units.\r\n metadata: dict\r\n The metadata with updated attributes.\r\n\r\n References\r\n ----------\r\n Bogner, K., Pappenberger, F., and Cloke, H. L.: Technical Note: The normal\r\n quantile transformation and its application in a flood forecasting system,\r\n Hydrol. Earth Syst. Sci., 16, 1085-1094,\r\n https://doi.org/10.5194/hess-16-1085-2012, 2012.\r\n \"\"\"\r\n\r\n # defaults\r\n a = kwargs.get(\"a\", 0.0)\r\n\r\n R = R.copy()\r\n shape0 = R.shape\r\n R = R.ravel().astype(float)\r\n idxNan = np.isnan(R)\r\n R_ = R[~idxNan]\r\n\r\n if metadata is None:\r\n if inverse:\r\n metadata = {\"transform\": \"NQT\"}\r\n else:\r\n metadata = {\"transform\": None}\r\n metadata[\"zerovalue\"] = np.min(R_)\r\n\r\n else:\r\n metadata = metadata.copy()\r\n\r\n if not inverse:\r\n # Plotting positions\r\n # https://en.wikipedia.org/wiki/Q%E2%80%93Q_plot#Plotting_position\r\n n = R_.size\r\n Rpp = ((np.arange(n) + 1 - a) / (n + 1 - 2 * a)).reshape(R_.shape)\r\n\r\n # NQ transform\r\n Rqn = scipy_stats.norm.ppf(Rpp)\r\n R__ = np.interp(R_, R_[np.argsort(R_)], Rqn)\r\n\r\n # set zero rain to 0 in norm space\r\n R__[R[~idxNan] == metadata[\"zerovalue\"]] = 0\r\n\r\n # build inverse transform\r\n metadata[\"inqt\"] = interp1d(\r\n Rqn, R_[np.argsort(R_)], bounds_error=False, fill_value=(R_.min(), R_.max())\r\n )\r\n\r\n metadata[\"transform\"] = \"NQT\"\r\n metadata[\"zerovalue\"] = 0\r\n metadata[\"threshold\"] = R__[R__ > 0].min()\r\n\r\n else:\r\n f = metadata.pop(\"inqt\")\r\n R__ = f(R_)\r\n metadata[\"transform\"] = None\r\n metadata[\"zerovalue\"] = R__.min()\r\n metadata[\"threshold\"] = R__[R__ > R__.min()].min()\r\n\r\n R[~idxNan] = R__\r\n\r\n return R.reshape(shape0), metadata\r\n\r\n\r\ndef sqrt_transform(R, metadata=None, inverse=False, **kwargs):\r\n \"\"\"\r\n Square-root transform.\r\n\r\n Parameters\r\n ----------\r\n R: array-like\r\n Array of any shape to be transformed.\r\n metadata: dict, optional\r\n Metadata dictionary containing the transform, zerovalue and threshold\r\n attributes as described in the documentation of\r\n :py:mod:`pysteps.io.importers`.\r\n inverse: bool, optional\r\n If set to True, it performs the inverse transform. False by default.\r\n\r\n Returns\r\n -------\r\n R: array-like\r\n Array of any shape containing the (back-)transformed units.\r\n metadata: dict\r\n The metadata with updated attributes.\r\n\r\n \"\"\"\r\n\r\n R = R.copy()\r\n\r\n if metadata is None:\r\n if inverse:\r\n metadata = {\"transform\": \"sqrt\"}\r\n else:\r\n metadata = {\"transform\": None}\r\n metadata[\"zerovalue\"] = np.nan\r\n metadata[\"threshold\"] = np.nan\r\n else:\r\n metadata = metadata.copy()\r\n\r\n if not inverse:\r\n # sqrt transform\r\n R = np.sqrt(R)\r\n\r\n metadata[\"transform\"] = \"sqrt\"\r\n metadata[\"zerovalue\"] = np.sqrt(metadata[\"zerovalue\"])\r\n metadata[\"threshold\"] = np.sqrt(metadata[\"threshold\"])\r\n else:\r\n # inverse sqrt transform\r\n R = R**2\r\n\r\n metadata[\"transform\"] = None\r\n metadata[\"zerovalue\"] = metadata[\"zerovalue\"] ** 2\r\n metadata[\"threshold\"] = metadata[\"threshold\"] ** 2\r\n\r\n return R, metadata\r\n" }, { "path": "pysteps/verification/__init__.py", "content": "# -- coding: utf-8 --\n\"\"\"Methods for verification of deterministic, probabilistic and ensemble forecasts.\"\"\"\n\nfrom .interface import get_method\nfrom .detcatscores import *\nfrom .detcontscores import *\nfrom .ensscores import *\nfrom .plots import *\nfrom .probscores import *\nfrom .spatialscores import *\nfrom .salscores import *\n" }, { "path": "pysteps/verification/detcatscores.py", "content": "# -- coding: utf-8 --\n\"\"\"\npysteps.verification.detcatscores\n=================================\n\nForecast evaluation and skill scores for deterministic categorial (dichotomous)\nforecasts.\n\n.. autosummary::\n :toctree: ../generated/\n\n det_cat_fct\n det_cat_fct_init\n det_cat_fct_accum\n det_cat_fct_merge\n det_cat_fct_compute\n\"\"\"\n\nimport collections\nimport numpy as np\n\n\ndef det_cat_fct(pred, obs, thr, scores=\"\", axis=None):\n \"\"\"\n Calculate simple and skill scores for deterministic categorical\n (dichotomous) forecasts.\n\n Parameters\n ----------\n pred: array_like\n Array of predictions. NaNs are ignored.\n obs: array_like\n Array of verifying observations. NaNs are ignored.\n thr: float\n The threshold that is applied to predictions and observations in order\n to define events vs no events (yes/no).\n scores: {string, list of strings}, optional\n The name(s) of the scores. The default, scores=\"\", will compute all\n available scores.\n The available score names are:\n\n .. tabularcolumns:: |p{2cm}|L|\n\n +------------+--------------------------------------------------------+\n | Name | Description |\n +============+========================================================+\n | ACC | accuracy (proportion correct) |\n +------------+--------------------------------------------------------+\n | BIAS | frequency bias |\n +------------+--------------------------------------------------------+\n | CSI | critical success index (threat score) |\n +------------+--------------------------------------------------------+\n | ETS | equitable threat score |\n +------------+--------------------------------------------------------+\n | F1 | the harmonic mean of precision and sensitivity |\n +------------+--------------------------------------------------------+\n | FA | false alarm rate (prob. of false detection, fall-out, |\n | | false positive rate) |\n +------------+--------------------------------------------------------+\n | FAR | false alarm ratio (false discovery rate) |\n +------------+--------------------------------------------------------+\n | GSS | Gilbert skill score (equitable threat score) |\n +------------+--------------------------------------------------------+\n | HK | Hanssen-Kuipers discriminant (Pierce skill score) |\n +------------+--------------------------------------------------------+\n | HSS | Heidke skill score |\n +------------+--------------------------------------------------------+\n | MCC | Matthews correlation coefficient |\n +------------+--------------------------------------------------------+\n | POD | probability of detection (hit rate, sensitivity, |\n | | recall, true positive rate) |\n +------------+--------------------------------------------------------+\n | SEDI | symmetric extremal dependency index |\n +------------+--------------------------------------------------------+\n\n axis: None or int or tuple of ints, optional\n Axis or axes along which a score is integrated. The default, axis=None,\n will integrate all of the elements of the input arrays.\\n\n If axis is -1 (or any negative integer),\n the integration is not performed\n and scores are computed on all of the elements in the input arrays.\\n\n If axis is a tuple of ints, the integration is performed on all of the\n axes specified in the tuple.\n\n Returns\n -------\n result: dict\n Dictionary containing the verification results.\n\n See also\n --------\n pysteps.verification.detcontscores.det_cont_fct\n \"\"\"\n\n contab = det_cat_fct_init(thr, axis)\n det_cat_fct_accum(contab, pred, obs)\n return det_cat_fct_compute(contab, scores)\n\n\ndef det_cat_fct_init(thr, axis=None):\n \"\"\"\n Initialize a contingency table object.\n\n Parameters\n ----------\n thr: float\n threshold that is applied to predictions and observations in order\n to define events vs no events (yes/no).\n axis: None or int or tuple of ints, optional\n Axis or axes along which a score is integrated. The default, axis=None,\n will integrate all of the elements of the input arrays.\\n\n If axis is -1 (or any negative integer),\n the integration is not performed\n and scores are computed on all of the elements in the input arrays.\\n\n If axis is a tuple of ints, the integration is performed on all of the\n axes specified in the tuple.\n\n Returns\n -------\n out: dict\n The contingency table object.\n \"\"\"\n\n contab = {}\n\n # catch case of axis passed as integer\n def get_iterable(x):\n if x is None or (\n isinstance(x, collections.abc.Iterable) and not isinstance(x, int)\n ):\n return x\n else:\n return (x,)\n\n contab[\"thr\"] = thr\n contab[\"axis\"] = get_iterable(axis)\n contab[\"hits\"] = None\n contab[\"false_alarms\"] = None\n contab[\"misses\"] = None\n contab[\"correct_negatives\"] = None\n\n return contab\n\n\ndef det_cat_fct_accum(contab, pred, obs):\n \"\"\"Accumulate the frequency of \"yes\" and \"no\" forecasts and observations\n in the contingency table.\n\n Parameters\n ----------\n contab: dict\n A contingency table object initialized with\n pysteps.verification.detcatscores.det_cat_fct_init.\n pred: array_like\n Array of predictions. NaNs are ignored.\n obs: array_like\n Array of verifying observations. NaNs are ignored.\n \"\"\"\n\n pred = np.asarray(pred.copy())\n obs = np.asarray(obs.copy())\n axis = tuple(range(pred.ndim)) if contab[\"axis\"] is None else contab[\"axis\"]\n\n # checks\n if pred.shape != obs.shape:\n raise ValueError(\n \"the shape of pred does not match the shape of obs %s!=%s\"\n % (pred.shape, obs.shape)\n )\n\n if pred.ndim <= np.max(axis):\n raise ValueError(\n \"axis %d is out of bounds for array of dimension %d\"\n % (np.max(axis), len(pred.shape))\n )\n\n idims = [dim not in axis for dim in range(pred.ndim)]\n nshape = tuple(np.array(pred.shape)[np.array(idims)])\n if contab[\"hits\"] is None:\n # initialize the count arrays in the contingency table\n contab[\"hits\"] = np.zeros(nshape, dtype=int)\n contab[\"false_alarms\"] = np.zeros(nshape, dtype=int)\n contab[\"misses\"] = np.zeros(nshape, dtype=int)\n contab[\"correct_negatives\"] = np.zeros(nshape, dtype=int)\n\n else:\n # check dimensions\n if contab[\"hits\"].shape != nshape:\n raise ValueError(\n \"the shape of the input arrays does not match \"\n + \"the shape of the \"\n + \"contingency table %s!=%s\" % (nshape, contab[\"hits\"].shape)\n )\n\n # add dummy axis in case integration is not required\n if np.max(axis) < 0:\n pred = pred[None, :]\n obs = obs[None, :]\n axis = (0,)\n axis = tuple([a for a in axis if a >= 0])\n\n # apply threshold\n predb = pred > contab[\"thr\"]\n obsb = obs > contab[\"thr\"]\n\n # calculate hits, misses, false positives, correct rejects\n H_idx = np.logical_and(predb == 1, obsb == 1)\n F_idx = np.logical_and(predb == 1, obsb == 0)\n M_idx = np.logical_and(predb == 0, obsb == 1)\n R_idx = np.logical_and(predb == 0, obsb == 0)\n\n # accumulate in the contingency table\n contab[\"hits\"] += np.nansum(H_idx.astype(int), axis=axis)\n contab[\"misses\"] += np.nansum(M_idx.astype(int), axis=axis)\n contab[\"false_alarms\"] += np.nansum(F_idx.astype(int), axis=axis)\n contab[\"correct_negatives\"] += np.nansum(R_idx.astype(int), axis=axis)\n\n\ndef det_cat_fct_merge(contab_1, contab_2):\n \"\"\"\n Merge two contingency table objects.\n\n Parameters\n ----------\n contab_1: dict\n A contingency table object initialized with\n :py:func:`pysteps.verification.detcatscores.det_cat_fct_init`\n and populated with\n :py:func:`pysteps.verification.detcatscores.det_cat_fct_accum`.\n contab_2: dict\n Another contingency table object initialized with\n :py:func:`pysteps.verification.detcatscores.det_cat_fct_init`\n and populated with\n :py:func:`pysteps.verification.detcatscores.det_cat_fct_accum`.\n\n Returns\n -------\n out: dict\n The merged contingency table object.\n \"\"\"\n\n # checks\n if contab_1[\"thr\"] != contab_2[\"thr\"]:\n raise ValueError(\n \"cannot merge: the thresholds are not same %s!=%s\"\n % (contab_1[\"thr\"], contab_2[\"thr\"])\n )\n if contab_1[\"axis\"] != contab_2[\"axis\"]:\n raise ValueError(\n \"cannot merge: the axis are not same %s!=%s\"\n % (contab_1[\"axis\"], contab_2[\"axis\"])\n )\n if contab_1[\"hits\"] is None or contab_2[\"hits\"] is None:\n raise ValueError(\"cannot merge: no data found\")\n\n # merge the contingency tables\n contab = contab_1.copy()\n contab[\"hits\"] += contab_2[\"hits\"]\n contab[\"misses\"] += contab_2[\"misses\"]\n contab[\"false_alarms\"] += contab_2[\"false_alarms\"]\n contab[\"correct_negatives\"] += contab_2[\"correct_negatives\"]\n\n return contab\n\n\ndef det_cat_fct_compute(contab, scores=\"\"):\n \"\"\"\n Compute simple and skill scores for deterministic categorical\n (dichotomous) forecasts from a contingency table object.\n\n Parameters\n ----------\n contab: dict\n A contingency table object initialized with\n pysteps.verification.detcatscores.det_cat_fct_init and populated with\n pysteps.verification.detcatscores.det_cat_fct_accum.\n scores: {string, list of strings}, optional\n The name(s) of the scores. The default, scores=\"\", will compute all\n available scores.\n The available score names a\n\n .. tabularcolumns:: |p{2cm}|L|\n\n +------------+--------------------------------------------------------+\n | Name | Description |\n +============+========================================================+\n | ACC | accuracy (proportion correct) |\n +------------+--------------------------------------------------------+\n | BIAS | frequency bias |\n +------------+--------------------------------------------------------+\n | CSI | critical success index (threat score) |\n +------------+--------------------------------------------------------+\n | ETS | equitable threat score |\n +------------+--------------------------------------------------------+\n | F1 | the harmonic mean of precision and sensitivity |\n +------------+--------------------------------------------------------+\n | FA | false alarm rate (prob. of false detection, fall-out, |\n | | false positive rate) |\n +------------+--------------------------------------------------------+\n | FAR | false alarm ratio (false discovery rate) |\n +------------+--------------------------------------------------------+\n | GSS | Gilbert skill score (equitable threat score) |\n +------------+--------------------------------------------------------+\n | HK | Hanssen-Kuipers discriminant (Pierce skill score) |\n +------------+--------------------------------------------------------+\n | HSS | Heidke skill score |\n +------------+--------------------------------------------------------+\n | MCC | Matthews correlation coefficient |\n +------------+--------------------------------------------------------+\n | POD | probability of detection (hit rate, sensitivity, |\n | | recall, true positive rate) |\n +------------+--------------------------------------------------------+\n | SEDI | symmetric extremal dependency index |\n +------------+--------------------------------------------------------+\n\n Returns\n -------\n result: dict\n Dictionary containing the verification results.\n \"\"\"\n\n # catch case of single score passed as string\n def get_iterable(x):\n if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):\n return x\n else:\n return (x,)\n\n scores = get_iterable(scores)\n\n H = 1.0 * contab[\"hits\"] # true positives\n M = 1.0 * contab[\"misses\"] # false negatives\n F = 1.0 * contab[\"false_alarms\"] # false positives\n R = 1.0 * contab[\"correct_negatives\"] # true negatives\n\n result = {}\n for score in scores:\n # catch None passed as score\n if score is None:\n continue\n\n score_ = score.lower()\n\n # simple scores\n POD = H / (H + M)\n FAR = F / (H + F)\n FA = F / (F + R)\n s = (H + M) / (H + M + F + R)\n\n if score_ in [\"pod\", \"\"]:\n # probability of detection\n result[\"POD\"] = POD\n if score_ in [\"far\", \"\"]:\n # false alarm ratio\n result[\"FAR\"] = FAR\n if score_ in [\"fa\", \"\"]:\n # false alarm rate (prob of false detection)\n result[\"FA\"] = FA\n if score_ in [\"acc\", \"\"]:\n # accuracy (fraction correct)\n ACC = (H + R) / (H + M + F + R)\n result[\"ACC\"] = ACC\n if score_ in [\"csi\", \"\"]:\n # critical success index\n CSI = H / (H + M + F)\n result[\"CSI\"] = CSI\n if score_ in [\"bias\", \"\"]:\n # frequency bias\n B = (H + F) / (H + M)\n result[\"BIAS\"] = B\n\n # skill scores\n if score_ in [\"hss\", \"\"]:\n # Heidke Skill Score (-1 < HSS < 1) < 0 implies no skill\n HSS = 2 * (H * R - F * M) / ((H + M) * (M + R) + (H + F) * (F + R))\n result[\"HSS\"] = HSS\n if score_ in [\"hk\", \"\"]:\n # Hanssen-Kuipers Discriminant\n HK = POD - FA\n result[\"HK\"] = HK\n if score_ in [\"gss\", \"ets\", \"\"]:\n # Gilbert Skill Score\n GSS = (POD - FA) / ((1 - s * POD) / (1 - s) + FA * (1 - s) / s)\n if score_ == \"ets\":\n result[\"ETS\"] = GSS\n else:\n result[\"GSS\"] = GSS\n if score_ in [\"sedi\", \"\"]:\n # Symmetric extremal dependence index\n SEDI = (np.log(FA) - np.log(POD) + np.log(1 - POD) - np.log(1 - FA)) / (\n np.log(FA) + np.log(POD) + np.log(1 - POD) + np.log(1 - FA)\n )\n result[\"SEDI\"] = SEDI\n if score_ in [\"mcc\", \"\"]:\n # Matthews correlation coefficient\n MCC = (H * R - F * M) / np.sqrt((H + F) * (H + M) * (R + F) * (R + M))\n result[\"MCC\"] = MCC\n if score_ in [\"f1\", \"\"]:\n # F1 score\n F1 = 2 * H / (2 * H + F + M)\n result[\"F1\"] = F1\n\n return result\n" }, { "path": "pysteps/verification/detcontscores.py", "content": "# -- coding: utf-8 --\n\"\"\"\npysteps.verification.detcontscores\n==================================\n\nForecast evaluation and skill scores for deterministic continuous forecasts.\n\n.. autosummary::\n :toctree: ../generated/\n\n det_cont_fct\n det_cont_fct_init\n det_cont_fct_accum\n det_cont_fct_merge\n det_cont_fct_compute\n\"\"\"\n\nimport collections\nimport numpy as np\nfrom scipy.stats import spearmanr\n\n\ndef det_cont_fct(pred, obs, scores=\"\", axis=None, conditioning=None, thr=0.0):\n \"\"\"\n Calculate simple and skill scores for deterministic continuous forecasts.\n\n Parameters\n ----------\n pred: array_like\n Array of predictions. NaNs are ignored.\n obs: array_like\n Array of verifying observations. NaNs are ignored.\n scores: {string, list of strings}, optional\n The name(s) of the scores. The default, scores=\"\", will compute all\n available scores.\n The available score names are:\n\n .. tabularcolumns:: |p{2cm}|L|\n\n +------------+--------------------------------------------------------+\n | Name | Description |\n +============+========================================================+\n | beta1 | linear regression slope (type 1 conditional bias) |\n +------------+--------------------------------------------------------+\n | beta2 | linear regression slope (type 2 conditional bias) |\n +------------+--------------------------------------------------------+\n | corr_p | pearson's correleation coefficien (linear correlation) |\n +------------+--------------------------------------------------------+\n | corr_s* | spearman's correlation coefficient (rank correlation) |\n +------------+--------------------------------------------------------+\n | DRMSE | debiased root mean squared error |\n +------------+--------------------------------------------------------+\n | MAE | mean absolute error |\n +------------+--------------------------------------------------------+\n | ME | mean error or bias |\n +------------+--------------------------------------------------------+\n | MSE | mean squared error |\n +------------+--------------------------------------------------------+\n | NMSE | normalized mean squared error |\n +------------+--------------------------------------------------------+\n | RMSE | root mean squared error |\n +------------+--------------------------------------------------------+\n | RV | reduction of variance |\n | | (Brier Score, Nash-Sutcliffe Efficiency) |\n +------------+--------------------------------------------------------+\n | scatter* | half the distance between the 16% and 84% percentiles |\n | | of the weighted cumulative error distribution, |\n | | where error = dB(pred/obs), |\n | | as in Germann et al. (2006) |\n +------------+--------------------------------------------------------+\n\n axis: {int, tuple of int, None}, optional\n Axis or axes along which a score is integrated. The default, axis=None,\n will integrate all of the elements of the input arrays.\\n\n If axis is -1 (or any negative integer),\n the integration is not performed\n and scores are computed on all of the elements in the input arrays.\\n\n If axis is a tuple of ints, the integration is performed on all of the\n axes specified in the tuple.\n conditioning: {None, \"single\", \"double\"}, optional\n The type of conditioning used for the verification.\n The default, conditioning=None, includes all pairs. With\n conditioning=\"single\", only pairs with either pred or obs > thr are\n included. With conditioning=\"double\", only pairs with both pred and\n obs > thr are included.\n thr: float\n Optional threshold value for conditioning. Defaults to 0.\n\n Returns\n -------\n result: dict\n Dictionary containing the verification results.\n\n Notes\n -----\n Multiplicative scores can be computed by passing log-tranformed values.\n Note that \"scatter\" is the only score that will be computed in dB units of\n the multiplicative error, i.e.: 10*log10(pred/obs).\n\n beta1 measures the degree of conditional bias of the observations given the\n forecasts (type 1).\n\n beta2 measures the degree of conditional bias of the forecasts given the\n observations (type 2).\n\n The normalized MSE is computed as\n NMSE = E[(pred - obs)^2]/E[(pred + obs)^2].\n\n The debiased RMSE is computed as DRMSE = sqrt(MSE - ME^2).\n\n The reduction of variance score is computed as RV = 1 - MSE/Var(obs).\n\n Score names denoted by * can only be computed offline, meaning that the\n these cannot be computed using _init, _accum and _compute methods of this\n module.\n\n References\n ----------\n Germann, U. , Galli, G. , Boscacci, M. and Bolliger, M. (2006), Radar\n precipitation measurement in a mountainous region. Q.J.R. Meteorol. Soc.,\n 132: 1669-1692. doi:10.1256/qj.05.190\n\n Potts, J. (2012), Chapter 2 - Basic concepts. Forecast verification: a\n practitioner’s guide in atmospheric sciences, I. T. Jolliffe, and D. B.\n Stephenson, Eds., Wiley-Blackwell, 11–29.\n\n See also\n --------\n pysteps.verification.detcatscores.det_cat_fct\n \"\"\"\n\n # catch case of single score passed as string\n def get_iterable(x):\n if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):\n return x\n else:\n return (x,)\n\n scores = get_iterable(scores)\n\n # split between online and offline scores\n loffline = [\"scatter\", \"corr_s\"]\n onscores = [\n score for score in scores if str(score).lower() not in loffline or score == \"\"\n ]\n offscores = [\n score for score in scores if str(score).lower() in loffline or score == \"\"\n ]\n\n # unique lists\n onscores = _uniquelist(onscores)\n offscores = _uniquelist(offscores)\n\n # online scores\n onresult = {}\n if onscores:\n err = det_cont_fct_init(axis=axis, conditioning=conditioning, thr=thr)\n det_cont_fct_accum(err, pred, obs)\n onresult = det_cont_fct_compute(err, onscores)\n\n # offline scores\n offresult = {}\n if offscores:\n pred = np.asarray(pred.copy())\n obs = np.asarray(obs.copy())\n\n if pred.shape != obs.shape:\n raise ValueError(\n \"the shape of pred does not match the shape of obs %s!=%s\"\n % (pred.shape, obs.shape)\n )\n\n # conditioning\n if conditioning is not None:\n if conditioning == \"single\":\n idx = np.logical_or(obs > thr, pred > thr)\n elif conditioning == \"double\":\n idx = np.logical_and(obs > thr, pred > thr)\n else:\n raise ValueError(\"unkown conditioning %s\" % conditioning)\n obs[~idx] = np.nan\n pred[~idx] = np.nan\n\n for score in offscores:\n # catch None passed as score\n if score is None:\n continue\n\n score_ = score.lower()\n\n # spearman corr (rank correlation)\n if score_ in [\"corr_s\", \"spearmanr\", \"\"]:\n corr_s = _spearmanr(pred, obs, axis=axis)\n offresult[\"corr_s\"] = corr_s\n\n # scatter\n if score_ in [\"scatter\", \"\"]:\n scatter = _scatter(pred, obs, axis=axis)\n offresult[\"scatter\"] = scatter\n\n # pull all results together\n result = onresult\n result.update(offresult)\n\n return result\n\n\ndef det_cont_fct_init(axis=None, conditioning=None, thr=0.0):\n \"\"\"\n Initialize a verification error object.\n\n Parameters\n ----------\n axis: {int, tuple of int, None}, optional\n Axis or axes along which a score is integrated. The default, axis=None,\n will integrate all of the elements of the input arrays.\\n\n If axis is -1 (or any negative integer),\n the integration is not performed\n and scores are computed on all of the elements in the input arrays.\\n\n If axis is a tuple of ints, the integration is performed on all of the\n axes specified in the tuple.\n conditioning: {None, \"single\", \"double\"}, optional\n The type of conditioning used for the verification.\n The default, conditioning=None, includes all pairs. With\n conditioning=\"single\", only pairs with either pred or obs > thr are\n included. With conditioning=\"double\", only pairs with both pred and\n obs > thr are included.\n thr: float\n Optional threshold value for conditioning. Defaults to 0.\n\n Returns\n -------\n out: dict\n The verification error object.\n \"\"\"\n\n err = {}\n\n # catch case of axis passed as integer\n def get_iterable(x):\n if x is None or (\n isinstance(x, collections.abc.Iterable) and not isinstance(x, int)\n ):\n return x\n else:\n return (x,)\n\n err[\"axis\"] = get_iterable(axis)\n err[\"conditioning\"] = conditioning\n err[\"thr\"] = thr\n err[\"cov\"] = None\n err[\"vobs\"] = None\n err[\"vpred\"] = None\n err[\"mobs\"] = None\n err[\"mpred\"] = None\n err[\"me\"] = None\n err[\"mse\"] = None\n err[\"mss\"] = None # mean square sum, i.e. E[(pred + obs)^2]\n err[\"mae\"] = None\n err[\"n\"] = None\n\n return err\n\n\ndef det_cont_fct_accum(err, pred, obs):\n \"\"\"Accumulate the forecast error in the verification error object.\n\n Parameters\n ----------\n err: dict\n A verification error object initialized with\n :py:func:`pysteps.verification.detcontscores.det_cont_fct_init`.\n pred: array_like\n Array of predictions. NaNs are ignored.\n obs: array_like\n Array of verifying observations. NaNs are ignored.\n\n References\n ----------\n Chan, Tony F.; Golub, Gene H.; LeVeque, Randall J. (1979), \"Updating\n Formulae and a Pairwise Algorithm for Computing Sample Variances.\",\n Technical Report STAN-CS-79-773, Department of Computer Science,\n Stanford University.\n\n Schubert, Erich; Gertz, Michael (2018-07-09). \"Numerically stable parallel\n computation of (co-)variance\". ACM: 10. doi:10.1145/3221269.3223036.\n \"\"\"\n\n pred = np.asarray(pred.copy())\n obs = np.asarray(obs.copy())\n axis = tuple(range(pred.ndim)) if err[\"axis\"] is None else err[\"axis\"]\n\n # checks\n if pred.shape != obs.shape:\n raise ValueError(\n \"the shape of pred does not match the shape of obs %s!=%s\"\n % (pred.shape, obs.shape)\n )\n\n if pred.ndim <= np.max(axis):\n raise ValueError(\n \"axis %d is out of bounds for array of dimension %d\"\n % (np.max(axis), len(pred.shape))\n )\n\n idims = [dim not in axis for dim in range(pred.ndim)]\n nshape = tuple(np.array(pred.shape)[np.array(idims)])\n if err[\"cov\"] is None:\n # initialize the error arrays in the verification object\n err[\"cov\"] = np.zeros(nshape)\n err[\"vobs\"] = np.zeros(nshape)\n err[\"vpred\"] = np.zeros(nshape)\n err[\"mobs\"] = np.zeros(nshape)\n err[\"mpred\"] = np.zeros(nshape)\n err[\"me\"] = np.zeros(nshape)\n err[\"mse\"] = np.zeros(nshape)\n err[\"mss\"] = np.zeros(nshape)\n err[\"mae\"] = np.zeros(nshape)\n err[\"n\"] = np.zeros(nshape)\n\n else:\n # check dimensions\n if err[\"cov\"].shape != nshape:\n raise ValueError(\n \"the shape of the input arrays does not match \"\n + \"the shape of the \"\n + \"verification object %s!=%s\" % (nshape, err[\"cov\"].shape)\n )\n\n # conditioning\n if err[\"conditioning\"] is not None:\n if err[\"conditioning\"] == \"single\":\n idx = np.logical_or(obs > err[\"thr\"], pred > err[\"thr\"])\n elif err[\"conditioning\"] == \"double\":\n idx = np.logical_and(obs > err[\"thr\"], pred > err[\"thr\"])\n else:\n raise ValueError(\"unkown conditioning %s\" % err[\"conditioning\"])\n obs[~idx] = np.nan\n pred[~idx] = np.nan\n\n # add dummy axis in case integration is not required\n if np.max(axis) < 0:\n pred = pred[None, :]\n obs = obs[None, :]\n axis = (0,)\n axis = tuple([a for a in axis if a >= 0])\n\n # compute residuals\n res = pred - obs\n sum = pred + obs\n n = np.sum(np.isfinite(res), axis=axis)\n\n # new means\n mobs = np.nanmean(obs, axis=axis)\n mpred = np.nanmean(pred, axis=axis)\n me = np.nanmean(res, axis=axis)\n mse = np.nanmean(res**2, axis=axis)\n mss = np.nanmean(sum**2, axis=axis)\n mae = np.nanmean(np.abs(res), axis=axis)\n\n # expand axes for broadcasting\n for ax in sorted(axis):\n mobs = np.expand_dims(mobs, ax)\n mpred = np.expand_dims(mpred, ax)\n\n # new cov matrix\n cov = np.nanmean((obs - mobs) * (pred - mpred), axis=axis)\n vobs = np.nanmean(np.abs(obs - mobs) ** 2, axis=axis)\n vpred = np.nanmean(np.abs(pred - mpred) ** 2, axis=axis)\n\n mobs = mobs.squeeze()\n mpred = mpred.squeeze()\n\n # update variances\n _parallel_var(err[\"mobs\"], err[\"n\"], err[\"vobs\"], mobs, n, vobs)\n _parallel_var(err[\"mpred\"], err[\"n\"], err[\"vpred\"], mpred, n, vpred)\n\n # update covariance\n _parallel_cov(err[\"cov\"], err[\"mobs\"], err[\"mpred\"], err[\"n\"], cov, mobs, mpred, n)\n\n # update means\n _parallel_mean(err[\"mobs\"], err[\"n\"], mobs, n)\n _parallel_mean(err[\"mpred\"], err[\"n\"], mpred, n)\n _parallel_mean(err[\"me\"], err[\"n\"], me, n)\n _parallel_mean(err[\"mse\"], err[\"n\"], mse, n)\n _parallel_mean(err[\"mss\"], err[\"n\"], mss, n)\n _parallel_mean(err[\"mae\"], err[\"n\"], mae, n)\n\n # update number of samples\n err[\"n\"] += n\n\n\ndef det_cont_fct_merge(err_1, err_2):\n \"\"\"\n Merge two verification error objects.\n\n Parameters\n ----------\n err_1: dict\n A verification error object initialized with\n :py:func:`pysteps.verification.detcontscores.det_cont_fct_init`\n and populated with\n :py:func:`pysteps.verification.detcontscores.det_cont_fct_accum`.\n err_2: dict\n Another verification error object initialized with\n :py:func:`pysteps.verification.detcontscores.det_cont_fct_init`\n and populated with\n :py:func:`pysteps.verification.detcontscores.det_cont_fct_accum`.\n\n Returns\n -------\n out: dict\n The merged verification error object.\n \"\"\"\n\n # checks\n if err_1[\"axis\"] != err_2[\"axis\"]:\n raise ValueError(\n \"cannot merge: the axis are not same %s!=%s\"\n % (err_1[\"axis\"], err_2[\"axis\"])\n )\n if err_1[\"conditioning\"] != err_2[\"conditioning\"]:\n raise ValueError(\n \"cannot merge: the conditioning is not same %s!=%s\"\n % (err_1[\"conditioning\"], err_2[\"conditioning\"])\n )\n if err_1[\"thr\"] != err_2[\"thr\"]:\n raise ValueError(\n \"cannot merge: the threshold is not same %s!=%s\"\n % (err_1[\"thr\"], err_2[\"thr\"])\n )\n if err_1[\"cov\"] is None or err_2[\"cov\"] is None:\n raise ValueError(\"cannot merge: no data found\")\n\n # merge the two verification error objects\n err = err_1.copy()\n\n # update variances\n _parallel_var(\n err[\"mobs\"], err[\"n\"], err[\"vobs\"], err_2[\"mobs\"], err_2[\"n\"], err_2[\"vobs\"]\n )\n _parallel_var(\n err[\"mpred\"],\n err[\"n\"],\n err[\"vpred\"],\n err_2[\"mpred\"],\n err_2[\"n\"],\n err_2[\"vpred\"],\n )\n\n # update covariance\n _parallel_cov(\n err[\"cov\"],\n err[\"mobs\"],\n err[\"mpred\"],\n err[\"n\"],\n err_2[\"cov\"],\n err_2[\"mobs\"],\n err_2[\"mpred\"],\n err_2[\"n\"],\n )\n\n # update means\n _parallel_mean(err[\"mobs\"], err[\"n\"], err_2[\"mobs\"], err_2[\"n\"])\n _parallel_mean(err[\"mpred\"], err[\"n\"], err_2[\"mpred\"], err_2[\"n\"])\n _parallel_mean(err[\"me\"], err[\"n\"], err_2[\"me\"], err_2[\"n\"])\n _parallel_mean(err[\"mse\"], err[\"n\"], err_2[\"mse\"], err_2[\"n\"])\n _parallel_mean(err[\"mss\"], err[\"n\"], err_2[\"mss\"], err_2[\"n\"])\n _parallel_mean(err[\"mae\"], err[\"n\"], err_2[\"mae\"], err_2[\"n\"])\n\n # update number of samples\n err[\"n\"] += err_2[\"n\"]\n\n return err\n\n\ndef det_cont_fct_compute(err, scores=\"\"):\n \"\"\"\n Compute simple and skill scores for deterministic continuous forecasts\n from a verification error object.\n\n Parameters\n ----------\n err: dict\n A verification error object initialized with\n :py:func:`pysteps.verification.detcontscores.det_cont_fct_init` and\n populated with\n :py:func:`pysteps.verification.detcontscores.det_cont_fct_accum`.\n scores: {string, list of strings}, optional\n The name(s) of the scores. The default, scores=\"\", will compute all\n available scores.\n The available score names are:\n\n .. tabularcolumns:: |p{2cm}|L|\n\n +------------+--------------------------------------------------------+\n | Name | Description |\n +============+========================================================+\n | beta1 | linear regression slope (type 1 conditional bias) |\n +------------+--------------------------------------------------------+\n | beta2 | linear regression slope (type 2 conditional bias) |\n +------------+--------------------------------------------------------+\n | corr_p | pearson's correleation coefficien (linear correlation) |\n +------------+--------------------------------------------------------+\n | DRMSE | debiased root mean squared error, i.e. |\n | | :math:`DRMSE = \\\\sqrt{RMSE - ME^2}` |\n +------------+--------------------------------------------------------+\n | MAE | mean absolute error |\n +------------+--------------------------------------------------------+\n | ME | mean error or bias |\n +------------+--------------------------------------------------------+\n | MSE | mean squared error |\n +------------+--------------------------------------------------------+\n | NMSE | normalized mean squared error |\n +------------+--------------------------------------------------------+\n | RMSE | root mean squared error |\n +------------+--------------------------------------------------------+\n | RV | reduction of variance |\n | | (Brier Score, Nash-Sutcliffe Efficiency), i.e. |\n | | :math:`RV = 1 - \\\\frac{MSE}{s^2_o}` |\n +------------+--------------------------------------------------------+\n\n Returns\n -------\n result: dict\n Dictionary containing the verification results.\n \"\"\"\n\n # catch case of single score passed as string\n def get_iterable(x):\n if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):\n return x\n else:\n return (x,)\n\n scores = get_iterable(scores)\n\n result = {}\n for score in scores:\n # catch None passed as score\n if score is None:\n continue\n\n score_ = score.lower()\n\n # bias (mean error, systematic error)\n if score_ in [\"bias\", \"me\", \"\"]:\n bias = err[\"me\"]\n result[\"ME\"] = bias\n\n # mean absolute error\n if score_ in [\"mae\", \"\"]:\n MAE = err[\"mae\"]\n result[\"MAE\"] = MAE\n\n # mean squared error\n if score_ in [\"mse\", \"\"]:\n MSE = err[\"mse\"]\n result[\"MSE\"] = MSE\n\n # normalized mean squared error\n if score_ in [\"nmse\", \"\"]:\n NMSE = err[\"mse\"] / err[\"mss\"]\n result[\"NMSE\"] = NMSE\n\n # root mean squared error\n if score_ in [\"rmse\", \"\"]:\n RMSE = np.sqrt(err[\"mse\"])\n result[\"RMSE\"] = RMSE\n\n # linear correlation coeff (pearson corr)\n if score_ in [\"corr_p\", \"pearsonr\", \"\"]:\n corr_p = err[\"cov\"] / np.sqrt(err[\"vobs\"]) / np.sqrt(err[\"vpred\"])\n result[\"corr_p\"] = corr_p\n\n # beta1 (linear regression slope)\n if score_ in [\"beta\", \"beta1\", \"\"]:\n beta1 = err[\"cov\"] / err[\"vpred\"]\n result[\"beta1\"] = beta1\n\n # beta2 (linear regression slope)\n if score_ in [\"beta2\", \"\"]:\n beta2 = err[\"cov\"] / err[\"vobs\"]\n result[\"beta2\"] = beta2\n\n # debiased RMSE\n if score_ in [\"drmse\", \"\"]:\n RMSE_d = np.sqrt(err[\"mse\"] - err[\"me\"] ** 2)\n result[\"DRMSE\"] = RMSE_d\n\n # reduction of variance\n # (Brier Score, Nash-Sutcliffe efficiency coefficient,\n # MSE skill score)\n if score_ in [\"rv\", \"brier_score\", \"nse\", \"\"]:\n RV = 1.0 - err[\"mse\"] / err[\"vobs\"]\n result[\"RV\"] = RV\n\n return result\n\n\ndef _parallel_mean(avg_a, count_a, avg_b, count_b):\n \"\"\"Update avg_a with avg_b.\"\"\"\n idx = count_b > 0\n avg_a[idx] = (count_a[idx] * avg_a[idx] + count_b[idx] * avg_b[idx]) / (\n count_a[idx] + count_b[idx]\n )\n\n\ndef _parallel_var(avg_a, count_a, var_a, avg_b, count_b, var_b):\n \"\"\"\n Update var_a with var_b.\n source: https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance\n \"\"\"\n idx = count_b > 0\n delta = avg_b - avg_a\n m_a = var_a * count_a\n m_b = var_b * count_b\n var_a[idx] = (\n m_a[idx]\n + m_b[idx]\n + delta[idx] ** 2 * count_a[idx] * count_b[idx] / (count_a[idx] + count_b[idx])\n )\n var_a[idx] = var_a[idx] / (count_a[idx] + count_b[idx])\n\n\ndef _parallel_cov(cov_a, avg_xa, avg_ya, count_a, cov_b, avg_xb, avg_yb, count_b):\n \"\"\"Update cov_a with cov_b.\"\"\"\n idx = count_b > 0\n deltax = avg_xb - avg_xa\n deltay = avg_yb - avg_ya\n c_a = cov_a * count_a\n c_b = cov_b * count_b\n cov_a[idx] = (\n c_a[idx]\n + c_b[idx]\n + deltax[idx]\n * deltay[idx]\n * count_a[idx]\n * count_b[idx]\n / (count_a[idx] + count_b[idx])\n )\n cov_a[idx] = cov_a[idx] / (count_a[idx] + count_b[idx])\n\n\ndef _uniquelist(mylist):\n used = set()\n return [x for x in mylist if x not in used and (used.add(x) or True)]\n\n\ndef _scatter(pred, obs, axis=None):\n pred = pred.copy()\n obs = obs.copy()\n\n # catch case of axis passed as integer\n def get_iterable(x):\n if x is None or (\n isinstance(x, collections.abc.Iterable) and not isinstance(x, int)\n ):\n return x\n else:\n return (x,)\n\n axis = get_iterable(axis)\n\n # reshape arrays as 2d matrices\n # rows: samples; columns: variables\n axis = tuple(range(pred.ndim)) if axis is None else axis\n axis = tuple(np.sort(axis))\n for ax in axis:\n pred = np.rollaxis(pred, ax, 0)\n obs = np.rollaxis(obs, ax, 0)\n shp_rows = pred.shape[: len(axis)]\n shp_cols = pred.shape[len(axis) :]\n pred = np.reshape(pred, (np.prod(shp_rows), -1))\n obs = np.reshape(obs, (np.prod(shp_rows), -1))\n\n # compute multiplicative erros in dB\n q = 10 * np.log10(pred / obs)\n\n # nans are given zero weight and are set equal to (min value - 1)\n idkeep = np.isfinite(q)\n q[~idkeep] = q[idkeep].min() - 1\n obs[~idkeep] = 0\n\n # compute scatter along rows\n xs = np.sort(q, axis=0)\n xs = np.vstack((xs[0, :], xs))\n ixs = np.argsort(q, axis=0)\n ws = np.take_along_axis(obs, ixs, axis=0)\n ws = np.vstack((ws[0, :] * 0.0, ws))\n wsc = np.cumsum(ws, axis=0) / np.sum(ws, axis=0)\n xint = np.zeros((2, xs.shape[1]))\n for i in range(xint.shape[1]):\n xint[:, i] = np.interp([0.16, 0.84], wsc[:, i], xs[:, i])\n scatter = (xint[1, :] - xint[0, :]) / 2.0\n\n # reshape back\n scatter = scatter.reshape(shp_cols)\n\n return float(scatter) if scatter.size == 1 else scatter\n\n\ndef _spearmanr(pred, obs, axis=None):\n pred = pred.copy()\n obs = obs.copy()\n\n # catch case of axis passed as integer\n def get_iterable(x):\n if x is None or (\n isinstance(x, collections.abc.Iterable) and not isinstance(x, int)\n ):\n return x\n else:\n return (x,)\n\n axis = get_iterable(axis)\n\n # reshape arrays as 2d matrices\n # rows: samples; columns: variables\n axis = tuple(range(pred.ndim)) if axis is None else axis\n axis = tuple(np.sort(axis))\n for ax in axis:\n pred = np.rollaxis(pred, ax, 0)\n obs = np.rollaxis(obs, ax, 0)\n shp_rows = pred.shape[: len(axis)]\n shp_cols = pred.shape[len(axis) :]\n pred = np.reshape(pred, (np.prod(shp_rows), -1))\n obs = np.reshape(obs, (np.prod(shp_rows), -1))\n\n # apply only with more than 2 valid samples\n # although this does not seem to solve the error\n # \"ValueError: The input must have at least 3 entries!\" ...\n corr_s = np.zeros(pred.shape[1]) * np.nan\n nsamp = np.sum(np.logical_and(np.isfinite(pred), np.isfinite(obs)), axis=0)\n idx = nsamp > 2\n if np.any(idx):\n corr_s_ = spearmanr(pred[:, idx], obs[:, idx], axis=0, nan_policy=\"omit\")[0]\n\n if corr_s_.size > 1:\n corr_s[idx] = np.diag(corr_s_, idx.sum())\n else:\n corr_s = corr_s_\n\n return float(corr_s) if corr_s.size == 1 else corr_s.reshape(shp_cols)\n" }, { "path": "pysteps/verification/ensscores.py", "content": "# -- coding: utf-8 --\n\"\"\"\npysteps.verification.ensscores\n==============================\n\nEvaluation and skill scores for ensemble forecasts.\n\n.. autosummary::\n :toctree: ../generated/\n\n ensemble_skill\n ensemble_spread\n rankhist\n rankhist_init\n rankhist_accum\n rankhist_compute\n\"\"\"\n\nimport numpy as np\nfrom .interface import get_method\n\n\ndef ensemble_skill(X_f, X_o, metric, **kwargs):\n \"\"\"\n Compute mean ensemble skill for a given skill metric.\n\n Parameters\n ----------\n X_f: array-like\n Array of shape (l,m,n) containing the forecast fields of shape (m,n)\n from l ensemble members.\n X_o: array_like\n Array of shape (m,n) containing the observed field corresponding to\n the forecast.\n metric: str\n The deterministic skill metric to be used (list available in\n :func:`~pysteps.verification.interface.get_method`).\n\n Returns\n -------\n out: float\n The mean skill of all ensemble members that is used as defintion of\n ensemble skill (as in Zacharov and Rezcova 2009 with the FSS).\n\n References\n ----------\n :cite:`ZR2009`\n \"\"\"\n\n if len(X_f.shape) != 3:\n raise ValueError(\n \"the number of dimensions of X_f must be equal to 3, \"\n + \"but %i dimensions were passed\" % len(X_f.shape)\n )\n if X_f.shape[1:] != X_o.shape:\n raise ValueError(\n \"the shape of X_f does not match the shape of \"\n + \"X_o (%d,%d)!=(%d,%d)\"\n % (X_f.shape[1], X_f.shape[2], X_o.shape[0], X_o.shape[1])\n )\n\n compute_skill = get_method(metric, type=\"deterministic\")\n\n lolo = X_f.shape[0]\n skill = []\n for member in range(lolo):\n skill_ = compute_skill(X_f[member, :, :], X_o, **kwargs)\n if isinstance(skill_, dict):\n skill_ = skill_[metric]\n skill.append(skill_)\n\n return np.mean(skill)\n\n\ndef ensemble_spread(X_f, metric, **kwargs):\n \"\"\"\n Compute mean ensemble spread for a given skill metric.\n\n Parameters\n ----------\n X_f: array-like\n Array of shape (l,m,n) containing the forecast fields of shape (m,n)\n from l ensemble members.\n metric: str\n The deterministic skill metric to be used (list available in\n :func:`~pysteps.verification.interface.get_method`).\n\n Returns\n -------\n out: float\n The mean skill compted between all possible pairs of\n the ensemble members,\n which can be used as definition of mean ensemble spread (as in Zacharov\n and Rezcova 2009 with the FSS).\n\n References\n ----------\n :cite:`ZR2009`\n \"\"\"\n if len(X_f.shape) != 3:\n raise ValueError(\n \"the number of dimensions of X_f must be equal to 3, \"\n + \"but %i dimensions were passed\" % len(X_f.shape)\n )\n if X_f.shape[0] < 2:\n raise ValueError(\n \"the number of members in X_f must be greater than 1,\"\n + \" but %i members were passed\" % X_f.shape[0]\n )\n\n compute_spread = get_method(metric, type=\"deterministic\")\n\n lolo = X_f.shape[0]\n spread = []\n for member in range(lolo):\n for othermember in range(member + 1, lolo):\n spread_ = compute_spread(\n X_f[member, :, :], X_f[othermember, :, :], **kwargs\n )\n if isinstance(spread_, dict):\n spread_ = spread_[metric]\n spread.append(spread_)\n\n return np.mean(spread)\n\n\ndef rankhist(X_f, X_o, X_min=None, normalize=True):\n \"\"\"\n Compute a rank histogram counts and optionally normalize the histogram.\n\n Parameters\n ----------\n X_f: array-like\n Array of shape (k,m,n,...) containing the values from an ensemble\n forecast of k members with shape (m,n,...).\n X_o: array_like\n Array of shape (m,n,...) containing the observed values corresponding\n to the forecast.\n X_min: {float,None}\n Threshold for minimum intensity. Forecast-observation pairs, where all\n ensemble members and verifying observations are below X_min, are not\n counted in the rank histogram.\n If set to None, thresholding is not used.\n normalize: {bool, True}\n If True, normalize the rank histogram so that\n the bin counts sum to one.\n \"\"\"\n\n X_f = X_f.copy()\n X_o = X_o.copy()\n num_ens_members = X_f.shape[0]\n rhist = rankhist_init(num_ens_members, X_min)\n rankhist_accum(rhist, X_f, X_o)\n return rankhist_compute(rhist, normalize)\n\n\ndef rankhist_init(num_ens_members, X_min=None):\n \"\"\"\n Initialize a rank histogram object.\n\n Parameters\n ----------\n num_ens_members: int\n Number ensemble members in the forecasts to accumulate into the rank\n histogram.\n X_min: {float,None}\n Threshold for minimum intensity. Forecast-observation pairs, where all\n ensemble members and verifying observations are below X_min, are not\n counted in the rank histogram.\n If set to None, thresholding is not used.\n\n Returns\n -------\n out: dict\n The rank histogram object.\n \"\"\"\n rankhist = {}\n\n rankhist[\"num_ens_members\"] = num_ens_members\n rankhist[\"n\"] = np.zeros(num_ens_members + 1, dtype=int)\n rankhist[\"X_min\"] = X_min\n\n return rankhist\n\n\ndef rankhist_accum(rankhist, X_f, X_o):\n \"\"\"Accumulate forecast-observation pairs to the given rank histogram.\n\n Parameters\n ----------\n rankhist: dict\n The rank histogram object.\n X_f: array-like\n Array of shape (k,m,n,...) containing the values from an ensemble\n forecast of k members with shape (m,n,...).\n X_o: array_like\n Array of shape (m,n,...) containing the observed values corresponding\n to the forecast.\n \"\"\"\n if X_f.shape[0] != rankhist[\"num_ens_members\"]:\n raise ValueError(\n \"the number of ensemble members in X_f does not \"\n + \"match the number of members in the rank \"\n + \"histogram (%d!=%d)\" % (X_f.shape[0], rankhist[\"num_ens_members\"])\n )\n\n X_f = np.vstack([X_f[i, :].flatten() for i in range(X_f.shape[0])]).T\n X_o = X_o.flatten()\n\n X_min = rankhist[\"X_min\"]\n\n mask = np.logical_and(np.isfinite(X_o), np.all(np.isfinite(X_f), axis=1))\n # ignore pairs where the verifying observations and all ensemble members\n # are below the threshold X_min\n if X_min is not None:\n mask_nz = np.logical_or(X_o >= X_min, np.any(X_f >= X_min, axis=1))\n mask = np.logical_and(mask, mask_nz)\n\n X_f = X_f[mask, :].copy()\n X_o = X_o[mask].copy()\n if X_min is not None:\n X_f[X_f < X_min] = X_min - 1\n X_o[X_o < X_min] = X_min - 1\n\n X_o = np.reshape(X_o, (len(X_o), 1))\n\n X_c = np.hstack([X_f, X_o])\n X_c.sort(axis=1)\n\n idx1 = np.where(X_c == X_o)\n _, idx2, idx_counts = np.unique(idx1[0], return_index=True, return_counts=True)\n bin_idx_1 = idx1[1][idx2]\n\n bin_idx = list(bin_idx_1[np.where(idx_counts == 1)[0]])\n\n # handle ties, where the verifying observation lies between ensemble\n # members having the same value\n idxdup = np.where(idx_counts > 1)[0]\n if len(idxdup) > 0:\n X_c_ = np.fliplr(X_c)\n idx1 = np.where(X_c_ == X_o)\n _, idx2 = np.unique(idx1[0], return_index=True)\n bin_idx_2 = X_f.shape[1] - idx1[1][idx2]\n\n idxr = np.random.uniform(low=0.0, high=1.0, size=len(idxdup))\n idxr = bin_idx_1[idxdup] + idxr * (bin_idx_2[idxdup] + 1 - bin_idx_1[idxdup])\n bin_idx.extend(idxr.astype(int))\n\n for bi in bin_idx:\n rankhist[\"n\"][bi] += 1\n\n\ndef rankhist_compute(rankhist, normalize=True):\n \"\"\"\n Return the rank histogram counts and optionally normalize the histogram.\n\n Parameters\n ----------\n rankhist: dict\n A rank histogram object created with rankhist_init.\n normalize: bool\n If True, normalize the rank histogram so that\n the bin counts sum to one.\n\n Returns\n -------\n out: array_like\n The counts for the n+1 bins in the rank histogram,\n where n is the number of ensemble members.\n \"\"\"\n if normalize:\n return 1.0 * rankhist[\"n\"] / sum(rankhist[\"n\"])\n else:\n return rankhist[\"n\"]\n" }, { "path": "pysteps/verification/interface.py", "content": "# -- coding: utf-8 --\n\"\"\"\npysteps.verification.interface\n==============================\n\nInterface for the verification module.\n\n.. autosummary::\n :toctree: ../generated/\n\n get_method\n\"\"\"\n\n\ndef get_method(name, type=\"deterministic\"):\n \"\"\"\n Return a callable function for the method corresponding to the given\n verification score.\n\n Parameters\n ----------\n name : str\n Name of the verification method. The available options are:\\n\\\n\n type: deterministic\n\n .. tabularcolumns:: |p{2cm}|L|\n\n +------------+--------------------------------------------------------+\n | Name | Description |\n +============+========================================================+\n | ACC | accuracy (proportion correct) |\n +------------+--------------------------------------------------------+\n | BIAS | frequency bias |\n +------------+--------------------------------------------------------+\n | CSI | critical success index (threat score) |\n +------------+--------------------------------------------------------+\n | F1 | the harmonic mean of precision and sensitivity |\n +------------+--------------------------------------------------------+\n | FA | false alarm rate (prob. of false detection, fall-out, |\n | | false positive rate) |\n +------------+--------------------------------------------------------+\n | FAR | false alarm ratio (false discovery rate) |\n +------------+--------------------------------------------------------+\n | GSS | Gilbert skill score (equitable threat score) |\n +------------+--------------------------------------------------------+\n | HK | Hanssen-Kuipers discriminant (Pierce skill score) |\n +------------+--------------------------------------------------------+\n | HSS | Heidke skill score |\n +------------+--------------------------------------------------------+\n | MCC | Matthews correlation coefficient |\n +------------+--------------------------------------------------------+\n | POD | probability of detection (hit rate, sensitivity, |\n | | recall, true positive rate) |\n +------------+--------------------------------------------------------+\n | SEDI | symmetric extremal dependency index |\n +------------+--------------------------------------------------------+\n | beta1 | linear regression slope (type 1 conditional bias) |\n +------------+--------------------------------------------------------+\n | beta2 | linear regression slope (type 2 conditional bias) |\n +------------+--------------------------------------------------------+\n | corr_p | pearson's correleation coefficien (linear correlation) |\n +------------+--------------------------------------------------------+\n | corr_s* | spearman's correlation coefficient (rank correlation) |\n +------------+--------------------------------------------------------+\n | DRMSE | debiased root mean squared error |\n +------------+--------------------------------------------------------+\n | MAE | mean absolute error of residuals |\n +------------+--------------------------------------------------------+\n | ME | mean error or bias of residuals |\n +------------+--------------------------------------------------------+\n | MSE | mean squared error |\n +------------+--------------------------------------------------------+\n | NMSE | normalized mean squared error |\n +------------+--------------------------------------------------------+\n | RMSE | root mean squared error |\n +------------+--------------------------------------------------------+\n | RV | reduction of variance |\n | | (Brier Score, Nash-Sutcliffe Efficiency) |\n +------------+--------------------------------------------------------+\n | scatter* | half the distance between the 16% and 84% percentiles |\n | | of the weighted cumulative error distribution, |\n | | where error = dB(pred/obs), |\n | | as in Germann et al. (2006) |\n +------------+--------------------------------------------------------+\n | binary_mse| binary MSE |\n +------------+--------------------------------------------------------+\n | FSS | fractions skill score |\n +------------+--------------------------------------------------------+\n | SAL | Structure-Amplitude-Location score |\n +------------+--------------------------------------------------------+\n\n type: ensemble\n\n .. tabularcolumns:: |p{2cm}|L|\n\n +------------+--------------------------------------------------------+\n | Name | Description |\n +============+========================================================+\n | ens_skill | mean ensemble skill |\n +------------+--------------------------------------------------------+\n | ens_spread | mean ensemble spread |\n +------------+--------------------------------------------------------+\n | rankhist | rank histogram |\n +------------+--------------------------------------------------------+\n\n type: probabilistic\n\n .. tabularcolumns:: |p{2cm}|L|\n\n +------------+--------------------------------------------------------+\n | Name | Description |\n +============+========================================================+\n | CRPS | continuous ranked probability score |\n +------------+--------------------------------------------------------+\n | reldiag | reliability diagram |\n +------------+--------------------------------------------------------+\n | ROC | ROC curve |\n +------------+--------------------------------------------------------+\n\n type : {'deterministic', 'ensemble', 'probabilistic'}, optional\n Type of the verification method.\n\n Notes\n -----\n Multiplicative scores can be computed by passing log-tranformed values.\n Note that \"scatter\" is the only score that will be computed in dB units of\n the multiplicative error, i.e.: 10*log10(pred/obs).\n\n beta1 measures the degree of conditional bias of the observations given the\n forecasts (type 1).\n\n beta2 measures the degree of conditional bias of the forecasts given the\n observations (type 2).\n\n The normalized MSE is computed as\n NMSE = E[(pred - obs)^2]/E[(pred + obs)^2].\n\n The debiased RMSE is computed as DRMSE = sqrt(RMSE - ME^2).\n\n The reduction of variance score is computed as RV = 1 - MSE/Var(obs).\n\n Score names denoted by * can only be computed offline, meaning that the\n these cannot be computed using _init, _accum and _compute methods of this\n module.\n\n References\n ----------\n Germann, U. , Galli, G. , Boscacci, M. and Bolliger, M. (2006), Radar\n precipitation measurement in a mountainous region. Q.J.R. Meteorol. Soc.,\n 132: 1669-1692. doi:10.1256/qj.05.190\n\n Potts, J. (2012), Chapter 2 - Basic concepts. Forecast verification: a\n practitioner’s guide in atmospheric sciences, I. T. Jolliffe, and D. B.\n Stephenson, Eds., Wiley-Blackwell, 11–29.\n \"\"\"\n\n if name is None:\n name = \"none\"\n if type is None:\n type = \"none\"\n\n name = name.lower()\n type = type.lower()\n\n if type == \"deterministic\":\n from .detcatscores import det_cat_fct\n from .detcontscores import det_cont_fct\n from .spatialscores import binary_mse, fss\n from .salscores import sal\n\n # categorical\n if name in [\n \"acc\",\n \"bias\",\n \"csi\",\n \"f1\",\n \"fa\",\n \"far\",\n \"gss\",\n \"hk\",\n \"hss\",\n \"mcc\",\n \"pod\",\n \"sedi\",\n ]:\n\n def f(fct, obs, **kwargs):\n return det_cat_fct(fct, obs, kwargs.pop(\"thr\"), [name])\n\n return f\n\n # continuous\n elif name in [\n \"beta\",\n \"beta1\",\n \"beta2\",\n \"corr_p\",\n \"corr_s\",\n \"drmse\",\n \"mae\",\n \"mse\",\n \"me\",\n \"nmse\",\n \"rmse\",\n \"rv\",\n \"scatter\",\n ]:\n\n def f(fct, obs, **kwargs):\n return det_cont_fct(fct, obs, [name], **kwargs)\n\n return f\n\n # spatial\n elif name == \"binary_mse\":\n return binary_mse\n elif name == \"fss\":\n return fss\n elif name == \"sal\":\n return sal\n\n else:\n raise ValueError(\"unknown deterministic method %s\" % name)\n\n elif type == \"ensemble\":\n from .ensscores import ensemble_skill, ensemble_spread, rankhist\n\n if name == \"ens_skill\":\n return ensemble_skill\n elif name == \"ens_spread\":\n return ensemble_spread\n elif name == \"rankhist\":\n return rankhist\n else:\n raise ValueError(\"unknown ensemble method %s\" % name)\n\n elif type == \"probabilistic\":\n from .probscores import CRPS, reldiag, ROC_curve\n\n if name == \"crps\":\n return CRPS\n elif name == \"reldiag\":\n return reldiag\n elif name == \"roc\":\n return ROC_curve\n else:\n raise ValueError(\"unknown probabilistic method %s\" % name)\n\n else:\n raise ValueError(\"unknown type %s\" % name)\n" }, { "path": "pysteps/verification/lifetime.py", "content": "# -- coding: utf-8 --\r\n\"\"\"\r\npysteps.verification.lifetime\r\n=============================\r\n\r\nEstimation of precipitation lifetime from\r\na decaying verification score function\r\n(e.g. autocorrelation function).\r\n\r\n.. autosummary::\r\n :toctree: ../generated/\r\n\r\n lifetime\r\n lifetime_init\r\n lifetime_accum\r\n lifetime_compute\r\n\"\"\"\r\n\r\nfrom math import exp\r\nimport numpy as np\r\nfrom scipy.integrate import simpson\r\n\r\n\r\ndef lifetime(X_s, X_t, rule=\"1/e\"):\r\n \"\"\"\r\n Compute the average lifetime by integrating the correlation function\r\n as a function of lead time. When not using the 1/e rule, the correlation\r\n function must be long enough to converge to 0, otherwise the lifetime is\r\n underestimated. The correlation function can be either empirical or\r\n theoretical, e.g. derived using the function 'ar_acf'\r\n in timeseries/autoregression.py.\r\n\r\n Parameters\r\n ----------\r\n X_s: array-like\r\n Array with the correlation function.\r\n Works also with other decaying scores that are defined\r\n in the range [0,1]=[min_skill,max_skill].\r\n X_t: array-like\r\n Array with the forecast lead times in the desired unit,\r\n e.g. [min, hour].\r\n rule: str {'1/e', 'trapz', 'simpson'}, optional\r\n Name of the method to integrate the correlation curve. \\n\r\n '1/e' uses the 1/e rule and assumes an exponential decay. It linearly\r\n interpolates the time when the correlation goes below the value 1/e.\r\n When all values are > 1/e it returns the max lead time.\r\n When all values are < 1/e it returns the min lead time. \\n\r\n 'trapz' uses the trapezoidal rule for integration.\\n\r\n 'simpson' uses the Simpson's rule for integration.\r\n\r\n Returns\r\n -------\r\n lf: float\r\n Estimated lifetime with same units of X_t.\r\n \"\"\"\r\n X_s = X_s.copy()\r\n X_t = X_t.copy()\r\n life = lifetime_init(rule)\r\n lifetime_accum(life, X_s, X_t)\r\n return lifetime_compute(life)\r\n\r\n\r\ndef lifetime_init(rule=\"1/e\"):\r\n \"\"\"\r\n Initialize a lifetime object.\r\n\r\n Parameters\r\n ----------\r\n rule: str {'1/e', 'trapz', 'simpson'}, optional\r\n Name of the method to integrate the correlation curve. \\n\r\n '1/e' uses the 1/e rule and assumes an exponential decay. It linearly\r\n interpolates the time when the correlation goes below the value 1/e.\r\n When all values are > 1/e it returns the max lead time.\r\n When all values are < 1/e it returns the min lead time.\\n\r\n 'trapz' uses the trapezoidal rule for integration.\\n\r\n 'simpson' uses the Simpson's rule for integration.\r\n\r\n Returns\r\n -------\r\n out: dict\r\n The lifetime object.\r\n \"\"\"\r\n list_rules = [\"trapz\", \"simpson\", \"1/e\"]\r\n if rule not in list_rules:\r\n raise ValueError(\r\n \"Unknown rule %s for integration.\\n\" % rule\r\n + \"The available methods are: \"\r\n + str(list_rules)\r\n )\r\n\r\n lifetime = {}\r\n lifetime[\"lifetime_sum\"] = 0.0\r\n lifetime[\"n\"] = 0.0\r\n lifetime[\"rule\"] = rule\r\n return lifetime\r\n\r\n\r\ndef lifetime_accum(lifetime, X_s, X_t):\r\n \"\"\"\r\n Compute the lifetime by integrating the correlation function\r\n and accumulate the result into the given lifetime object.\r\n\r\n Parameters\r\n ----------\r\n X_s: array-like\r\n Array with the correlation function.\r\n Works also with other decaying scores that are defined\r\n in the range [0,1]=[min_skill,max_skill].\r\n X_t: array-like\r\n Array with the forecast lead times in the desired unit,\r\n e.g. [min, hour].\r\n \"\"\"\r\n if lifetime[\"rule\"] == \"trapz\":\r\n lf = np.trapz(X_s, x=X_t)\r\n elif lifetime[\"rule\"] == \"simpson\":\r\n lf = simpson(X_s, x=X_t)\r\n elif lifetime[\"rule\"] == \"1/e\":\r\n euler_number = 1.0 / exp(1.0)\r\n X_s_ = np.array(X_s)\r\n\r\n is_euler_reached = np.sum(X_s_ <= euler_number) > 0\r\n if is_euler_reached:\r\n idx_b = np.argmax(X_s_ <= euler_number)\r\n if idx_b > 0:\r\n idx_a = idx_b - 1\r\n fraction_score = (\r\n (euler_number - X_s[idx_b])\r\n * (X_t[idx_a] - X_t[idx_b])\r\n / (X_s[idx_a] - X_s[idx_b])\r\n )\r\n lf = X_t[idx_b] + fraction_score\r\n else:\r\n # if all values are below the 1/e value, return min lead time\r\n lf = np.min(X_t)\r\n else:\r\n # if all values are above the 1/e value, return max lead time\r\n lf = np.max(X_t)\r\n\r\n lifetime[\"lifetime_sum\"] += lf\r\n lifetime[\"n\"] += 1\r\n\r\n\r\ndef lifetime_compute(lifetime):\r\n \"\"\"\r\n Compute the average value from the lifetime object.\r\n\r\n Parameters\r\n ----------\r\n lifetime: dict\r\n A lifetime object created with lifetime_init.\r\n\r\n Returns\r\n -------\r\n out: float\r\n The computed lifetime.\r\n \"\"\"\r\n return 1.0 * lifetime[\"lifetime_sum\"] / lifetime[\"n\"]\r\n" }, { "path": "pysteps/verification/plots.py", "content": "# -- coding: utf-8 --\n\"\"\"\npysteps.verification.plots\n==========================\n\nMethods for plotting verification results.\n\n.. autosummary::\n :toctree: ../generated/\n\n plot_intensityscale\n plot_rankhist\n plot_reldiag\n plot_ROC\n\"\"\"\n\nfrom matplotlib import cm\nimport matplotlib.pylab as plt\nfrom mpl_toolkits.axes_grid1.inset_locator import inset_axes\nimport numpy as np\nfrom pysteps.verification import ensscores, probscores, spatialscores\n\n\ndef plot_intensityscale(intscale, fig=None, vminmax=None, kmperpixel=None, unit=None):\n \"\"\"\n Plot a intensity-scale verification table with a color bar and axis\n labels.\n\n Parameters\n ----------\n\n intscale: dict\n The intensity-scale object initialized with\n :py:func:`pysteps.verification.spatialscores.intensity_scale_init`\n and accumulated with\n :py:func:`pysteps.verification.spatialscores.intensity_scale_accum`.\n\n fig: matplotlib.figure.Figure, optional\n The figure object to use for plotting. If not supplied, a new\n figure is created.\n\n vminmax: tuple of floats, optional\n The minimum and maximum values for the intensity-scale skill score\n in the plot.\n Defaults to the data extent.\n\n kmperpixel: float, optional\n The conversion factor from pixels to kilometers. If supplied,\n the unit of the shown spatial scales is km instead of pixels.\n\n unit: string, optional\n The unit of the intensity thresholds.\n \"\"\"\n if fig is None:\n fig = plt.figure()\n\n ax = fig.gca()\n\n SS = spatialscores.intensity_scale_compute(intscale)\n\n vmin = vmax = None\n if vminmax is not None:\n vmin = np.min(vminmax)\n vmax = np.max(vminmax)\n im = ax.imshow(SS, vmin=vmin, vmax=vmax, interpolation=\"nearest\", cmap=cm.jet)\n cb = fig.colorbar(im)\n cb.set_label(intscale[\"label\"])\n\n if unit is None:\n ax.set_xlabel(\"Intensity threshold\")\n else:\n ax.set_xlabel(\"Intensity threshold [%s]\" % unit)\n if kmperpixel is None:\n ax.set_ylabel(\"Spatial scale [pixels]\")\n else:\n ax.set_ylabel(\"Spatial scale [km]\")\n\n ax.set_xticks(np.arange(SS.shape[1]))\n ax.set_xticklabels(intscale[\"thrs\"])\n ax.set_yticks(np.arange(SS.shape[0]))\n if kmperpixel is None:\n scales = intscale[\"scales\"]\n else:\n scales = np.array(intscale[\"scales\"]) * kmperpixel\n ax.set_yticklabels(scales)\n\n\ndef plot_rankhist(rankhist, ax=None):\n \"\"\"\n Plot a rank histogram.\n\n Parameters\n ----------\n rankhist: dict\n A rank histogram object created by ensscores.rankhist_init.\n ax: axis handle, optional\n Axis handle for the figure. If set to None, the handle is taken from\n the current figure (matplotlib.pylab.gca()).\n\n \"\"\"\n if ax is None:\n ax = plt.gca()\n\n r = ensscores.rankhist_compute(rankhist)\n x = np.linspace(0, 1, rankhist[\"num_ens_members\"] + 1)\n ax.bar(x, r, width=1.0 / len(x), align=\"edge\", color=\"gray\", edgecolor=\"black\")\n\n ax.set_xticks(x[::3] + (x[1] - x[0]))\n ax.set_xticklabels(np.arange(1, len(x) + 1)[::3])\n ax.set_xlim(0, 1 + 1.0 / len(x))\n ax.set_ylim(0, np.max(r) * 1.25)\n\n ax.set_xlabel(\"Rank of observation (among ensemble members)\")\n ax.set_ylabel(\"Relative frequency\")\n\n ax.grid(True, axis=\"y\", ls=\":\")\n\n\ndef plot_reldiag(reldiag, ax=None):\n \"\"\"\n Plot a reliability diagram.\n\n Parameters\n ----------\n reldiag: dict\n A reldiag object created by probscores.reldiag_init.\n ax: axis handle, optional\n Axis handle for the figure. If set to None, the handle is taken from\n the current figure (matplotlib.pylab.gca()).\n\n \"\"\"\n if ax is None:\n ax = plt.gca()\n\n # Plot the reliability diagram.\n f = 1.0 * reldiag[\"Y_sum\"] / reldiag[\"num_idx\"]\n r = 1.0 * reldiag[\"X_sum\"] / reldiag[\"num_idx\"]\n\n mask = np.logical_and(np.isfinite(r), np.isfinite(f))\n\n ax.plot(r[mask], f[mask], \"kD-\")\n ax.plot([0, 1], [0, 1], \"k--\")\n\n ax.set_xlim(0, 1)\n ax.set_ylim(0, 1)\n\n ax.grid(True, ls=\":\")\n\n ax.set_xlabel(\"Forecast probability\")\n ax.set_ylabel(\"Observed relative frequency\")\n\n # Plot sharpness diagram into an inset figure.\n iax = inset_axes(ax, width=\"35%\", height=\"20%\", loc=4, borderpad=3.5)\n bw = reldiag[\"bin_edges\"][2] - reldiag[\"bin_edges\"][1]\n iax.bar(\n reldiag[\"bin_edges\"][:-1],\n reldiag[\"sample_size\"],\n width=bw,\n align=\"edge\",\n color=\"gray\",\n edgecolor=\"black\",\n )\n iax.set_yscale(\"log\")\n iax.set_xticks(reldiag[\"bin_edges\"])\n iax.set_xticklabels([\"%.1f\" % max(v, 1e-6) for v in reldiag[\"bin_edges\"]])\n yt_min = int(max(np.floor(np.log10(min(reldiag[\"sample_size\"][:-1]))), 1))\n yt_max = int(np.ceil(np.log10(max(reldiag[\"sample_size\"][:-1]))))\n t = [pow(10.0, k) for k in range(yt_min, yt_max)]\n\n iax.set_yticks([int(t_) for t_ in t])\n iax.set_xlim(0.0, 1.0)\n iax.set_ylim(t[0], 5 * t[-1])\n iax.set_ylabel(\"log10(samples)\")\n iax.yaxis.tick_right()\n iax.yaxis.set_label_position(\"right\")\n iax.tick_params(axis=\"both\", which=\"major\", labelsize=6)\n\n\ndef plot_ROC(ROC, ax=None, opt_prob_thr=False):\n \"\"\"\n Plot a ROC curve.\n\n Parameters\n ----------\n ROC: dict\n A ROC curve object created by probscores.ROC_curve_init.\n ax: axis handle, optional\n Axis handle for the figure. If set to None, the handle is taken from\n the current figure (matplotlib.pylab.gca()).\n opt_prob_thr: bool, optional\n If set to True, plot the optimal probability threshold that maximizes\n the difference between the hit rate (POD) and false alarm rate (POFD).\n\n \"\"\"\n if ax is None:\n ax = plt.gca()\n\n POFD, POD, area = probscores.ROC_curve_compute(ROC, compute_area=True)\n p_thr = ROC[\"prob_thrs\"]\n\n ax.plot([0, 1], [0, 1], \"k--\")\n ax.set_xlim(0, 1)\n ax.set_ylim(0, 1)\n ax.set_xlabel(\"False alarm rate (POFD)\")\n ax.set_ylabel(\"Probability of detection (POD)\")\n ax.grid(True, ls=\":\")\n\n ax.plot(POFD, POD, \"kD-\")\n\n if opt_prob_thr:\n opt_prob_thr_idx = np.argmax(np.array(POD) - np.array(POFD))\n ax.scatter(\n [POFD[opt_prob_thr_idx]],\n [POD[opt_prob_thr_idx]],\n c=\"r\",\n s=150,\n facecolors=None,\n edgecolors=\"r\",\n )\n\n for p_thr_, x, y in zip(p_thr, POFD, POD):\n ax.text(x + 0.02, y - 0.02, \"%.2f\" % p_thr_, fontsize=7)\n" }, { "path": "pysteps/verification/probscores.py", "content": "# -- coding: utf-8 --\n\"\"\"\npysteps.verification.probscores\n===============================\n\nEvaluation and skill scores for probabilistic forecasts.\n\n.. autosummary::\n :toctree: ../generated/\n\n CRPS\n CRPS_init\n CRPS_accum\n CRPS_compute\n reldiag\n reldiag_init\n reldiag_accum\n reldiag_compute\n ROC_curve\n ROC_curve_init\n ROC_curve_accum\n ROC_curve_compute\n\"\"\"\n\nimport numpy as np\n\n\ndef CRPS(X_f, X_o):\n \"\"\"\n Compute the continuous ranked probability score (CRPS).\n\n Parameters\n ----------\n X_f: array_like\n Array of shape (k,m,n,...) containing the values from an ensemble\n forecast of k members with shape (m,n,...).\n X_o: array_like\n Array of shape (m,n,...) containing the observed values corresponding\n to the forecast.\n\n Returns\n -------\n out: float\n The computed CRPS.\n\n References\n ----------\n :cite:`Her2000`\n \"\"\"\n\n X_f = X_f.copy()\n X_o = X_o.copy()\n crps = CRPS_init()\n CRPS_accum(crps, X_f, X_o)\n return CRPS_compute(crps)\n\n\ndef CRPS_init():\n \"\"\"\n Initialize a CRPS object.\n\n Returns\n -------\n out: dict\n The CRPS object.\n \"\"\"\n return {\"CRPS_sum\": 0.0, \"n\": 0.0}\n\n\ndef CRPS_accum(CRPS, X_f, X_o):\n \"\"\"\n Compute the average continuous ranked probability score (CRPS) for a set\n of forecast ensembles and the corresponding observations and accumulate the\n result to the given CRPS object.\n\n Parameters\n ----------\n CRPS: dict\n The CRPS object.\n X_f: array_like\n Array of shape (k,m,n,...) containing the values from an ensemble\n forecast of k members with shape (m,n,...).\n X_o: array_like\n Array of shape (m,n,...) containing the observed values corresponding\n to the forecast.\n\n References\n ----------\n :cite:`Her2000`\n \"\"\"\n X_f = np.vstack([X_f[i, :].flatten() for i in range(X_f.shape[0])]).T\n X_o = X_o.flatten()\n\n mask = np.logical_and(np.all(np.isfinite(X_f), axis=1), np.isfinite(X_o))\n\n X_f = X_f[mask, :].copy()\n X_f.sort(axis=1)\n X_o = X_o[mask]\n\n n = X_f.shape[0]\n m = X_f.shape[1]\n\n alpha = np.zeros((n, m + 1))\n beta = np.zeros((n, m + 1))\n\n for i in range(1, m):\n mask = X_o > X_f[:, i]\n alpha[mask, i] = X_f[mask, i] - X_f[mask, i - 1]\n beta[mask, i] = 0.0\n\n mask = np.logical_and(X_f[:, i] > X_o, X_o > X_f[:, i - 1])\n alpha[mask, i] = X_o[mask] - X_f[mask, i - 1]\n beta[mask, i] = X_f[mask, i] - X_o[mask]\n\n mask = X_o < X_f[:, i - 1]\n alpha[mask, i] = 0.0\n beta[mask, i] = X_f[mask, i] - X_f[mask, i - 1]\n\n mask = X_o < X_f[:, 0]\n alpha[mask, 0] = 0.0\n beta[mask, 0] = X_f[mask, 0] - X_o[mask]\n\n mask = X_f[:, -1] < X_o\n alpha[mask, -1] = X_o[mask] - X_f[mask, -1]\n beta[mask, -1] = 0.0\n\n p = 1.0 * np.arange(m + 1) / m\n res = np.sum(alpha * p**2.0 + beta * (1.0 - p) ** 2.0, axis=1)\n\n CRPS[\"CRPS_sum\"] += np.sum(res)\n CRPS[\"n\"] += len(res)\n\n\ndef CRPS_compute(CRPS):\n \"\"\"\n Compute the averaged values from the given CRPS object.\n\n Parameters\n ----------\n CRPS: dict\n A CRPS object created with CRPS_init.\n\n Returns\n -------\n out: float\n The computed CRPS.\n \"\"\"\n return 1.0 * CRPS[\"CRPS_sum\"] / CRPS[\"n\"]\n\n\ndef reldiag(P_f, X_o, X_min, n_bins=10, min_count=10):\n \"\"\"\n Compute the x- and y- coordinates of the points in the reliability diagram.\n\n Parameters\n ----------\n P_f: array-like\n Forecast probabilities for exceeding the intensity threshold specified\n in the reliability diagram object.\n X_o: array-like\n Observed values.\n X_min: float\n Precipitation intensity threshold for yes/no prediction.\n n_bins: int\n Number of bins to use in the reliability diagram.\n min_count: int\n Minimum number of samples required for each bin. A zero value is assigned\n if the number of samples in a bin is smaller than bin_count.\n\n Returns\n -------\n out: tuple\n Two-element tuple containing the x- and y-coordinates of the points in\n the reliability diagram.\n \"\"\"\n\n P_f = P_f.copy()\n X_o = X_o.copy()\n rdiag = reldiag_init(X_min, n_bins, min_count)\n reldiag_accum(rdiag, P_f, X_o)\n return reldiag_compute(rdiag)\n\n\ndef reldiag_init(X_min, n_bins=10, min_count=10):\n \"\"\"\n Initialize a reliability diagram object.\n\n Parameters\n ----------\n X_min: float\n Precipitation intensity threshold for yes/no prediction.\n n_bins: int\n Number of bins to use in the reliability diagram.\n min_count: int\n Minimum number of samples required for each bin. A zero value is assigned\n if the number of samples in a bin is smaller than bin_count.\n\n Returns\n -------\n out: dict\n The reliability diagram object.\n\n References\n ----------\n :cite:`BS2007`\n \"\"\"\n reldiag = {}\n\n reldiag[\"X_min\"] = X_min\n reldiag[\"bin_edges\"] = np.linspace(-1e-6, 1 + 1e-6, int(n_bins + 1))\n reldiag[\"n_bins\"] = n_bins\n reldiag[\"X_sum\"] = np.zeros(n_bins)\n reldiag[\"Y_sum\"] = np.zeros(n_bins, dtype=int)\n reldiag[\"num_idx\"] = np.zeros(n_bins, dtype=int)\n reldiag[\"sample_size\"] = np.zeros(n_bins, dtype=int)\n reldiag[\"min_count\"] = min_count\n\n return reldiag\n\n\ndef reldiag_accum(reldiag, P_f, X_o):\n \"\"\"Accumulate the given probability-observation pairs into the reliability\n diagram.\n\n Parameters\n ----------\n reldiag: dict\n A reliability diagram object created with reldiag_init.\n P_f: array-like\n Forecast probabilities for exceeding the intensity threshold specified\n in the reliability diagram object.\n X_o: array-like\n Observed values.\n \"\"\"\n mask = np.logical_and(np.isfinite(P_f), np.isfinite(X_o))\n\n P_f = P_f[mask]\n X_o = X_o[mask]\n\n idx = np.digitize(P_f, reldiag[\"bin_edges\"], right=True)\n\n x = []\n y = []\n num_idx = []\n ss = []\n\n for k in range(1, len(reldiag[\"bin_edges\"])):\n I_k = np.where(idx == k)[0]\n if len(I_k) >= reldiag[\"min_count\"]:\n X_o_above_thr = (X_o[I_k] >= reldiag[\"X_min\"]).astype(int)\n y.append(np.sum(X_o_above_thr))\n x.append(np.sum(P_f[I_k]))\n num_idx.append(len(I_k))\n ss.append(len(I_k))\n else:\n y.append(0.0)\n x.append(0.0)\n num_idx.append(0.0)\n ss.append(0)\n\n reldiag[\"X_sum\"] += np.array(x)\n reldiag[\"Y_sum\"] += np.array(y, dtype=int)\n reldiag[\"num_idx\"] += np.array(num_idx, dtype=int)\n reldiag[\"sample_size\"] += ss\n\n\ndef reldiag_compute(reldiag):\n \"\"\"\n Compute the x- and y- coordinates of the points in the reliability diagram.\n\n Parameters\n ----------\n reldiag: dict\n A reliability diagram object created with reldiag_init.\n\n Returns\n -------\n out: tuple\n Two-element tuple containing the x- and y-coordinates of the points in\n the reliability diagram.\n \"\"\"\n f = 1.0 * reldiag[\"Y_sum\"] / reldiag[\"num_idx\"]\n r = 1.0 * reldiag[\"X_sum\"] / reldiag[\"num_idx\"]\n\n return r, f\n\n\ndef ROC_curve(P_f, X_o, X_min, n_prob_thrs=10, compute_area=False):\n \"\"\"\n Compute the ROC curve and its area from the given ROC object.\n\n Parameters\n ----------\n P_f: array_like\n Forecasted probabilities for exceeding the threshold specified in the ROC\n object. Non-finite values are ignored.\n X_o: array_like\n Observed values. Non-finite values are ignored.\n X_min: float\n Precipitation intensity threshold for yes/no prediction.\n n_prob_thrs: int\n The number of probability thresholds to use.\n The interval [0,1] is divided into n_prob_thrs evenly spaced values.\n compute_area: bool\n If True, compute the area under the ROC curve (between 0.5 and 1).\n\n Returns\n -------\n out: tuple\n A two-element tuple containing the probability of detection (POD) and\n probability of false detection (POFD) for the probability thresholds\n specified in the ROC curve object. If compute_area is True, return the\n area under the ROC curve as the third element of the tuple.\n \"\"\"\n\n P_f = P_f.copy()\n X_o = X_o.copy()\n roc = ROC_curve_init(X_min, n_prob_thrs)\n ROC_curve_accum(roc, P_f, X_o)\n return ROC_curve_compute(roc, compute_area)\n\n\ndef ROC_curve_init(X_min, n_prob_thrs=10):\n \"\"\"\n Initialize a ROC curve object.\n\n Parameters\n ----------\n X_min: float\n Precipitation intensity threshold for yes/no prediction.\n n_prob_thrs: int\n The number of probability thresholds to use.\n The interval [0,1] is divided into n_prob_thrs evenly spaced values.\n\n Returns\n -------\n out: dict\n The ROC curve object.\n \"\"\"\n ROC = {}\n\n ROC[\"X_min\"] = X_min\n ROC[\"hits\"] = np.zeros(n_prob_thrs, dtype=int)\n ROC[\"misses\"] = np.zeros(n_prob_thrs, dtype=int)\n ROC[\"false_alarms\"] = np.zeros(n_prob_thrs, dtype=int)\n ROC[\"corr_neg\"] = np.zeros(n_prob_thrs, dtype=int)\n ROC[\"prob_thrs\"] = np.linspace(0.0, 1.0, int(n_prob_thrs))\n\n return ROC\n\n\ndef ROC_curve_accum(ROC, P_f, X_o):\n \"\"\"Accumulate the given probability-observation pairs into the given ROC\n object.\n\n Parameters\n ----------\n ROC: dict\n A ROC curve object created with ROC_curve_init.\n P_f: array_like\n Forecasted probabilities for exceeding the threshold specified in the ROC\n object. Non-finite values are ignored.\n X_o: array_like\n Observed values. Non-finite values are ignored.\n \"\"\"\n mask = np.logical_and(np.isfinite(P_f), np.isfinite(X_o))\n\n P_f = P_f[mask]\n X_o = X_o[mask]\n\n for i, p in enumerate(ROC[\"prob_thrs\"]):\n mask = np.logical_and(P_f >= p, X_o >= ROC[\"X_min\"])\n ROC[\"hits\"][i] += np.sum(mask.astype(int))\n mask = np.logical_and(P_f < p, X_o >= ROC[\"X_min\"])\n ROC[\"misses\"][i] += np.sum(mask.astype(int))\n mask = np.logical_and(P_f >= p, X_o < ROC[\"X_min\"])\n ROC[\"false_alarms\"][i] += np.sum(mask.astype(int))\n mask = np.logical_and(P_f < p, X_o < ROC[\"X_min\"])\n ROC[\"corr_neg\"][i] += np.sum(mask.astype(int))\n\n\ndef ROC_curve_compute(ROC, compute_area=False):\n \"\"\"\n Compute the ROC curve and its area from the given ROC object.\n\n Parameters\n ----------\n ROC: dict\n A ROC curve object created with ROC_curve_init.\n compute_area: bool\n If True, compute the area under the ROC curve (between 0.5 and 1).\n\n Returns\n -------\n out: tuple\n A two-element tuple containing the probability of detection (POD) and\n probability of false detection (POFD) for the probability thresholds\n specified in the ROC curve object. If compute_area is True, return the\n area under the ROC curve as the third element of the tuple.\n \"\"\"\n POD_vals = []\n POFD_vals = []\n\n for i in range(len(ROC[\"prob_thrs\"])):\n POD_vals.append(1.0 * ROC[\"hits\"][i] / (ROC[\"hits\"][i] + ROC[\"misses\"][i]))\n POFD_vals.append(\n 1.0 * ROC[\"false_alarms\"][i] / (ROC[\"corr_neg\"][i] + ROC[\"false_alarms\"][i])\n )\n\n if compute_area:\n # Compute the total area of parallelepipeds under the ROC curve.\n area = (1.0 - POFD_vals[0]) * (1.0 + POD_vals[0]) / 2.0\n for i in range(len(ROC[\"prob_thrs\"]) - 1):\n area += (\n (POFD_vals[i] - POFD_vals[i + 1])\n * (POD_vals[i + 1] + POD_vals[i])\n / 2.0\n )\n area += POFD_vals[-1] * POD_vals[-1] / 2.0\n\n return POFD_vals, POD_vals, area\n else:\n return POFD_vals, POD_vals\n" }, { "path": "pysteps/verification/salscores.py", "content": "# -- coding: utf-8 --\n\"\"\"\npysteps.verification.salscores\n==============================\n\nThe Spatial-Amplitude-Location (SAL) score by :cite:`WPHF2008`.\n\n.. autosummary::\n :toctree: ../generated/\n\n sal\n sal_structure\n sal_amplitude\n sal_location\n\"\"\"\n\nfrom math import sqrt, hypot\n\nimport numpy as np\nfrom scipy.ndimage import center_of_mass\n\nfrom pysteps.exceptions import MissingOptionalDependency\nfrom pysteps.feature import tstorm as tstorm_detect\n\ntry:\n import pandas as pd\n\n PANDAS_IMPORTED = True\nexcept ImportError:\n PANDAS_IMPORTED = False\n\ntry:\n from skimage.measure import regionprops_table\n\n SKIMAGE_IMPORTED = True\nexcept ImportError:\n SKIMAGE_IMPORTED = False\n\n\n# regionprops property names changed with scikit-image v0.19, buld old names\n# will continue to work for backwards compatibility\n# see https://github.com/scikit-image/scikit-image/releases/tag/v0.19.0\nREGIONPROPS = [\n \"label\",\n \"weighted_centroid\",\n \"max_intensity\",\n \"intensity_image\",\n]\n\n\ndef sal(\n prediction,\n observation,\n thr_factor=0.067, # default to 1/15 as in the reference paper\n thr_quantile=0.95,\n tstorm_kwargs=None,\n):\n \"\"\"\n Compute the Structure-Amplitude-Location (SAL) spatial verification metric.\n\n Parameters\n ----------\n prediction: array-like\n Array of shape (m,n) with prediction data. NaNs are ignored.\n observation: array-like\n Array of shape (m,n) with observation data. NaNs are ignored.\n thr_factor: float, optional\n Factor used to compute the detection threshold as in eq. 1 of :cite:`WHZ2009`.\n If not None, this is used to identify coherent objects enclosed by the\n threshold contour `thr_factor * thr_quantile(precip)`.\n thr_quantile: float, optional\n The wet quantile between 0 and 1 used to define the detection threshold.\n Required if `thr_factor` is not None.\n tstorm_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the tstorm feature\n detection algorithm. If None, default values are used.\n See the documentation of :py:func:`pysteps.feature.tstorm.detection`.\n\n Returns\n -------\n sal: tuple of floats\n A 3-element tuple containing the structure, amplitude, location\n components of the SAL score.\n\n References\n ----------\n :cite:`WPHF2008`\n :cite:`WHZ2009`\n :cite:`Feldmann2021`\n\n Notes\n -----\n This implementation uses the thunderstorm detection algorithm by :cite:`Feldmann2021`\n for the identification of precipitation objects within the considered domain.\n\n See also\n --------\n :py:func:`pysteps.verification.salscores.sal_structure`,\n :py:func:`pysteps.verification.salscores.sal_amplitude`,\n :py:func:`pysteps.verification.salscores.sal_location`,\n :py:mod:`pysteps.feature.tstorm`\n \"\"\"\n prediction = np.copy(prediction)\n observation = np.copy(observation)\n structure = sal_structure(\n prediction, observation, thr_factor, thr_quantile, tstorm_kwargs\n )\n amplitude = sal_amplitude(prediction, observation)\n location = sal_location(\n prediction, observation, thr_factor, thr_quantile, tstorm_kwargs\n )\n return structure, amplitude, location\n\n\ndef sal_structure(\n prediction, observation, thr_factor=None, thr_quantile=None, tstorm_kwargs=None\n):\n \"\"\"\n Compute the structure component for SAL based on :cite:`WPHF2008`.\n\n Parameters\n ----------\n prediction: array-like\n Array of shape (m,n) with prediction data. NaNs are ignored.\n observation: array-like\n Array of shape (m,n) with observation data. NaNs are ignored.\n thr_factor: float, optional\n Factor used to compute the detection threshold as in eq. 1 of :cite:`WHZ2009`.\n If not None, this is used to identify coherent objects enclosed by the\n threshold contour `thr_factor * thr_quantile(precip)`.\n thr_quantile: float, optional\n The wet quantile between 0 and 1 used to define the detection threshold.\n Required if `thr_factor` is not None.\n tstorm_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the tstorm feature\n detection algorithm. If None, default values are used.\n See the documentation of :py:func:`pysteps.feature.tstorm.detection`.\n\n Returns\n -------\n structure: float\n The structure component with value between -2 to 2 and 0 denotes perfect\n forecast in terms of structure. The returned value is NaN if no objects are\n detected in neither the prediction nor the observation.\n\n See also\n --------\n :py:func:`pysteps.verification.salscores.sal`,\n :py:func:`pysteps.verification.salscores.sal_amplitude`,\n :py:func:`pysteps.verification.salscores.sal_location`,\n :py:mod:`pysteps.feature.tstorm`\n \"\"\"\n prediction_objects = _sal_detect_objects(\n prediction, thr_factor, thr_quantile, tstorm_kwargs\n )\n observation_objects = _sal_detect_objects(\n observation, thr_factor, thr_quantile, tstorm_kwargs\n )\n prediction_volume = _sal_scaled_volume(prediction_objects)\n observation_volume = _sal_scaled_volume(observation_objects)\n nom = prediction_volume - observation_volume\n denom = prediction_volume + observation_volume\n return np.divide(nom, (0.5 * denom))\n\n\ndef sal_amplitude(prediction, observation):\n \"\"\"\n Compute the amplitude component for SAL based on :cite:`WPHF2008`.\n\n This component is the normalized difference of the domain-averaged precipitation\n in observation and forecast.\n\n Parameters\n ----------\n prediction: array-like\n Array of shape (m,n) with prediction data. NaNs are ignored.\n observation: array-like\n Array of shape (m,n) with observation data. NaNs are ignored.\n\n Returns\n -------\n amplitude: float\n Amplitude parameter with value between -2 to 2 and 0 denotes perfect forecast in\n terms of amplitude. The returned value is NaN if no objects are detected in\n neither the prediction nor the observation.\n\n See also\n --------\n :py:func:`pysteps.verification.salscores.sal`,\n :py:func:`pysteps.verification.salscores.sal_structure`,\n :py:func:`pysteps.verification.salscores.sal_location`\n \"\"\"\n mean_obs = np.nanmean(observation)\n mean_pred = np.nanmean(prediction)\n return (mean_pred - mean_obs) / (0.5 * (mean_pred + mean_obs))\n\n\ndef sal_location(\n prediction, observation, thr_factor=None, thr_quantile=None, tstorm_kwargs=None\n):\n \"\"\"\n Compute the first parameter of location component for SAL based on\n :cite:`WPHF2008`.\n\n This parameter indicates the normalized distance between the center of mass in\n observation and forecast.\n\n Parameters\n ----------\n prediction: array-like\n Array of shape (m,n) with prediction data. NaNs are ignored.\n observation: array-like\n Array of shape (m,n) with observation data. NaNs are ignored.\n thr_factor: float, optional\n Factor used to compute the detection threshold as in eq. 1 of :cite:`WHZ2009`.\n If not None, this is used to identify coherent objects enclosed by the\n threshold contour `thr_factor * thr_quantile(precip)`.\n thr_quantile: float, optional\n The wet quantile between 0 and 1 used to define the detection threshold.\n Required if `thr_factor` is not None.\n tstorm_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the tstorm feature\n detection algorithm. If None, default values are used.\n See the documentation of :py:func:`pysteps.feature.tstorm.detection`.\n\n Returns\n -------\n location: float\n The location component with value between 0 to 2 and 0 denotes perfect forecast\n in terms of location. The returned value is NaN if no objects are detected in\n either the prediction or the observation.\n\n See also\n --------\n :py:func:`pysteps.verification.salscores.sal`,\n :py:func:`pysteps.verification.salscores.sal_structure`,\n :py:func:`pysteps.verification.salscores.sal_amplitude`,\n :py:mod:`pysteps.feature.tstorm`\n \"\"\"\n return _sal_l1_param(prediction, observation) + _sal_l2_param(\n prediction, observation, thr_factor, thr_quantile, tstorm_kwargs\n )\n\n\ndef _sal_l1_param(prediction, observation):\n \"\"\"\n Compute the first parameter of location component for SAL based on\n :cite:`WPHF2008`.\n\n This parameter indicates the normalized distance between the center of mass in\n observation and forecast.\n\n Parameters\n ----------\n prediction: array-like\n Array of shape (m,n) with prediction data. NaNs are ignored.\n observation: array-like\n Array of shape (m,n) with observation data. NaNs are ignored.\n\n Returns\n -------\n location_1: float\n The first parameter of location component which has a value between 0 to 1.\n \"\"\"\n maximum_distance = sqrt(\n ((observation.shape[0]) ** 2) + ((observation.shape[1]) ** 2)\n )\n obi = center_of_mass(np.nan_to_num(observation))\n fori = center_of_mass(np.nan_to_num(prediction))\n dist = hypot(fori[1] - obi[1], fori[0] - obi[0])\n return dist / maximum_distance\n\n\ndef _sal_l2_param(prediction, observation, thr_factor, thr_quantile, tstorm_kwargs):\n \"\"\"\n Calculate the second parameter of location component for SAL based on :cite:`WPHF2008`.\n\n Parameters\n ----------\n prediction: array-like\n Array of shape (m,n) with prediction data. NaNs are ignored.\n observation: array-like\n Array of shape (m,n) with observation data. NaNs are ignored.\n thr_factor: float\n Factor used to compute the detection threshold as in eq. 1 of :cite:`WHZ2009`.\n If not None, this is used to identify coherent objects enclosed by the\n threshold contour `thr_factor * thr_quantile(precip)`.\n thr_quantile: float\n The wet quantile between 0 and 1 used to define the detection threshold.\n Required if `thr_factor` is not None.\n tstorm_kwargs: dict\n Optional dictionary containing keyword arguments for the tstorm feature\n detection algorithm. If None, default values are used.\n See the documentation of :py:func:`pysteps.feature.tstorm.detection`.\n\n Returns\n -------\n location_2: float\n The secibd parameter of location component with value between 0 to 1.\n \"\"\"\n maximum_distance = sqrt(\n ((observation.shape[0]) ** 2) + ((observation.shape[1]) ** 2)\n )\n obs_r = _sal_weighted_distance(observation, thr_factor, thr_quantile, tstorm_kwargs)\n forc_r = _sal_weighted_distance(prediction, thr_factor, thr_quantile, tstorm_kwargs)\n\n location_2 = 2 * ((abs(obs_r - forc_r)) / maximum_distance)\n return float(location_2)\n\n\ndef _sal_detect_objects(precip, thr_factor, thr_quantile, tstorm_kwargs):\n \"\"\"\n Detect coherent precipitation objects using a multi-threshold approach from\n :cite:`Feldmann2021`.\n\n Parameters\n ----------\n precip: array-like\n Array of shape (m,n) containing input data. Nan values are ignored.\n thr_factor: float\n Factor used to compute the detection threshold as in eq. 1 of :cite:`WHZ2009`.\n If not None, this is used to identify coherent objects enclosed by the\n threshold contour `thr_factor * thr_quantile(precip)`.\n thr_quantile: float\n The wet quantile between 0 and 1 used to define the detection threshold.\n Required if `thr_factor` is not None.\n tstorm_kwargs: dict\n Optional dictionary containing keyword arguments for the tstorm feature\n detection algorithm. If None, default values are used.\n See the documentation of :py:func:`pysteps.feature.tstorm.detection`.\n\n Returns\n -------\n precip_objects: pd.DataFrame\n Dataframe containing all detected cells and their respective properties.\n \"\"\"\n if not PANDAS_IMPORTED:\n raise MissingOptionalDependency(\n \"The pandas package is required for the SAL \"\n \"verification method but it is not installed\"\n )\n if not SKIMAGE_IMPORTED:\n raise MissingOptionalDependency(\n \"The scikit-image package is required for the SAL \"\n \"verification method but it is not installed\"\n )\n if thr_factor is not None and thr_quantile is None:\n raise ValueError(\"You must pass thr_quantile, too\")\n if tstorm_kwargs is None:\n tstorm_kwargs = dict()\n if thr_factor is not None:\n zero_value = np.nanmin(precip)\n threshold = thr_factor * np.nanquantile(\n precip[precip > zero_value], thr_quantile\n )\n tstorm_kwargs = {\n \"minmax\": tstorm_kwargs.get(\"minmax\", threshold),\n \"maxref\": tstorm_kwargs.get(\"maxref\", threshold + 1e-5),\n \"mindiff\": tstorm_kwargs.get(\"mindiff\", 1e-5),\n \"minref\": tstorm_kwargs.get(\"minref\", threshold),\n }\n _, labels = tstorm_detect.detection(precip, **tstorm_kwargs)\n labels = labels.astype(int)\n precip_objects = pd.DataFrame(\n regionprops_table(labels, intensity_image=precip, properties=REGIONPROPS)\n )\n return precip_objects\n\n\ndef _sal_scaled_volume(precip_objects):\n \"\"\"\n Calculate the scaled volume based on :cite:`WPHF2008`.\n\n Parameters\n ----------\n precip_objects: pd.DataFrame\n Dataframe containing all detected cells and their respective properties\n as returned by the :py:func:`pysteps.verification.salsscores._sal_detect_objects`\n function.\n\n Returns\n -------\n total_scaled_volum: float\n The total scaled volume of precipitation objects.\n \"\"\"\n if not PANDAS_IMPORTED:\n raise MissingOptionalDependency(\n \"The pandas package is required for the SAL \"\n \"verification method but it is not installed\"\n )\n objects_volume_scaled = []\n for _, precip_object in precip_objects.iterrows():\n intensity_sum = np.nansum(precip_object.intensity_image)\n max_intensity = precip_object.max_intensity\n if intensity_sum == 0:\n intensity_vol = 0\n else:\n volume_scaled = intensity_sum / max_intensity\n tot_vol = intensity_sum * volume_scaled\n intensity_vol = tot_vol\n\n objects_volume_scaled.append(\n {\"intensity_vol\": intensity_vol, \"intensity_sum_obj\": intensity_sum}\n )\n df_vols = pd.DataFrame(objects_volume_scaled)\n\n if df_vols.empty or (df_vols[\"intensity_sum_obj\"] == 0).all():\n total_scaled_volum = 0\n else:\n total_scaled_volum = np.nansum(df_vols.intensity_vol) / np.nansum(\n df_vols.intensity_sum_obj\n )\n return total_scaled_volum\n\n\ndef _sal_weighted_distance(precip, thr_factor, thr_quantile, tstorm_kwargs):\n \"\"\"\n Compute the weighted averaged distance between the centers of mass of the\n individual objects and the center of mass of the total precipitation field.\n\n Parameters\n ----------\n precip: array-like\n Array of shape (m,n). NaNs are ignored.\n thr_factor: float\n Factor used to compute the detection threshold as in eq. 1 of :cite:`WHZ2009`.\n If not None, this is used to identify coherent objects enclosed by the\n threshold contour `thr_factor * thr_quantile(precip)`.\n thr_quantile: float\n The wet quantile between 0 and 1 used to define the detection threshold.\n Required if `thr_factor` is not None.\n tstorm_kwargs: dict\n Optional dictionary containing keyword arguments for the tstorm feature\n detection algorithm. If None, default values are used.\n See the documentation of :py:func:`pysteps.feature.tstorm.detection`.\n\n Returns\n -------\n weighted_distance: float\n The weighted averaged distance between the centers of mass of the\n individual objects and the center of mass of the total precipitation field.\n The returned value is NaN if no objects are detected.\n \"\"\"\n if not PANDAS_IMPORTED:\n raise MissingOptionalDependency(\n \"The pandas package is required for the SAL \"\n \"verification method but it is not installed\"\n )\n precip_objects = _sal_detect_objects(\n precip, thr_factor, thr_quantile, tstorm_kwargs\n )\n if len(precip_objects) == 0:\n return np.nan\n centroid_total = center_of_mass(np.nan_to_num(precip))\n r = []\n for i in precip_objects.label - 1:\n xd = (precip_objects[\"weighted_centroid-1\"][i] - centroid_total[1]) ** 2\n yd = (precip_objects[\"weighted_centroid-0\"][i] - centroid_total[0]) ** 2\n\n dst = sqrt(xd + yd)\n sumr = (np.nansum(precip_objects.intensity_image[i])) * dst\n\n sump = np.nansum(precip_objects.intensity_image[i])\n\n r.append({\"sum_dist\": sumr, \"sum_p\": sump})\n rr = pd.DataFrame(r)\n return (np.nansum(rr.sum_dist)) / (np.nansum(rr.sum_p))\n" }, { "path": "pysteps/verification/spatialscores.py", "content": "# -- coding: utf-8 --\n\"\"\"\npysteps.verification.spatialscores\n==================================\n\nSkill scores for spatial forecasts.\n\n.. autosummary::\n :toctree: ../generated/\n\n intensity_scale\n intensity_scale_init\n intensity_scale_accum\n intensity_scale_merge\n intensity_scale_compute\n binary_mse\n binary_mse_init\n binary_mse_accum\n binary_mse_merge\n binary_mse_compute\n fss\n fss_init\n fss_accum\n fss_merge\n fss_compute\n\"\"\"\n\nimport collections\nimport numpy as np\nfrom scipy.ndimage import uniform_filter\n\nfrom pysteps.exceptions import MissingOptionalDependency\nfrom pysteps.verification.salscores import sal # make SAL accessible from this module\n\ntry:\n import pywt\n\n pywt_imported = True\nexcept ImportError:\n pywt_imported = False\n\n\ndef intensity_scale(X_f, X_o, name, thrs, scales=None, wavelet=\"Haar\"):\n \"\"\"\n Compute an intensity-scale verification score.\n\n Parameters\n ----------\n X_f: array_like\n Array of shape (m, n) containing the forecast field.\n X_o: array_like\n Array of shape (m, n) containing the verification observation field.\n name: string\n A string indicating the name of the spatial verification score\n to be used:\n\n +------------+--------------------------------------------------------+\n | Name | Description |\n +============+========================================================+\n | FSS | Fractions skill score |\n +------------+--------------------------------------------------------+\n | BMSE | Binary mean squared error |\n +------------+--------------------------------------------------------+\n\n thrs: float or array_like\n Scalar or 1-D array of intensity thresholds for which to compute the\n verification.\n scales: float or array_like, optional\n Scalar or 1-D array of spatial scales in pixels,\n required if ``name=\"FSS\"``.\n wavelet: str, optional\n The name of the wavelet function to use in the BMSE.\n Defaults to the Haar wavelet, as described in Casati et al. 2004.\n See the documentation of PyWavelets for a list of available options.\n\n Returns\n -------\n out: array_like\n The two-dimensional array containing the intensity-scale skill scores\n for each spatial scale and intensity threshold.\n\n References\n ----------\n :cite:`CRS2004`, :cite:`RL2008`, :cite:`EWWM2013`\n\n See also\n --------\n pysteps.verification.spatialscores.binary_mse,\n pysteps.verification.spatialscores.fss\n \"\"\"\n\n intscale = intensity_scale_init(name, thrs, scales, wavelet)\n intensity_scale_accum(intscale, X_f, X_o)\n return intensity_scale_compute(intscale)\n\n\ndef intensity_scale_init(name, thrs, scales=None, wavelet=\"Haar\"):\n \"\"\"\n Initialize an intensity-scale verification object.\n\n Parameters\n ----------\n name: string\n A string indicating the name of the spatial verification score\n to be used:\n\n +------------+--------------------------------------------------------+\n | Name | Description |\n +============+========================================================+\n | FSS | Fractions skill score |\n +------------+--------------------------------------------------------+\n | BMSE | Binary mean squared error |\n +------------+--------------------------------------------------------+\n\n thrs: float or array_like\n Scalar or 1-D array of intensity thresholds for which to compute the\n verification.\n scales: float or array_like, optional\n Scalar or 1-D array of spatial scales in pixels,\n required if ``name=\"FSS\"``.\n wavelet: str, optional\n The name of the wavelet function, required if ``name=\"BMSE\"``.\n Defaults to the Haar wavelet, as described in Casati et al. 2004.\n See the documentation of PyWavelets for a list of available options.\n\n Returns\n -------\n out: dict\n The intensity-scale object.\n \"\"\"\n\n if name.lower() == \"fss\" and scales is None:\n message = \"an array of spatial scales must be provided for the FSS,\"\n message += \" but %s was passed\" % scales\n raise ValueError(message)\n\n if name.lower() == \"bmse\" and wavelet is None:\n message = \"the name of a wavelet must be provided for the BMSE,\"\n message += \" but %s was passed\" % wavelet\n raise ValueError(message)\n\n # catch scalars when passed as arguments\n def get_iterable(x):\n if isinstance(x, collections.abc.Iterable):\n return np.copy(x)\n else:\n return np.copy((x,))\n\n intscale = {}\n intscale[\"name\"] = name\n intscale[\"thrs\"] = np.sort(get_iterable(thrs))\n if scales is not None:\n intscale[\"scales\"] = np.sort(get_iterable(scales))[::-1]\n else:\n intscale[\"scales\"] = scales\n intscale[\"wavelet\"] = wavelet\n\n for i, thr in enumerate(intscale[\"thrs\"]):\n if name.lower() == \"bmse\":\n intscale[thr] = binary_mse_init(thr, intscale[\"wavelet\"])\n\n elif name.lower() == \"fss\":\n intscale[thr] = {}\n\n for j, scale in enumerate(intscale[\"scales\"]):\n intscale[thr][scale] = fss_init(thr, scale)\n\n if name.lower() == \"fss\":\n intscale[\"label\"] = \"Fractions skill score\"\n del intscale[\"wavelet\"]\n\n elif name.lower() == \"bmse\":\n intscale[\"label\"] = \"Binary MSE skill score\"\n intscale[\"scales\"] = None\n\n else:\n raise ValueError(\"unknown method %s\" % name)\n\n return intscale\n\n\ndef intensity_scale_accum(intscale, X_f, X_o):\n \"\"\"\n Compute and update the intensity-scale verification scores.\n\n Parameters\n ----------\n intscale: dict\n The intensity-scale object initialized with\n :py:func:`pysteps.verification.spatialscores.intensity_scale_init`.\n X_f: array_like\n Array of shape (m, n) containing the forecast field.\n X_o: array_like\n Array of shape (m, n) containing the verification observation field.\n \"\"\"\n\n name = intscale[\"name\"]\n thrs = intscale[\"thrs\"]\n scales = intscale[\"scales\"]\n\n for i, thr in enumerate(thrs):\n if name.lower() == \"bmse\":\n binary_mse_accum(intscale[thr], X_f, X_o)\n\n elif name.lower() == \"fss\":\n for j, scale in enumerate(scales):\n fss_accum(intscale[thr][scale], X_f, X_o)\n\n if scales is None:\n intscale[\"scales\"] = intscale[thrs[0]][\"scales\"]\n\n\ndef intensity_scale_merge(intscale_1, intscale_2):\n \"\"\"\n Merge two intensity-scale verification objects.\n\n Parameters\n ----------\n intscale_1: dict\n Am intensity-scale object initialized with\n :py:func:`pysteps.verification.spatialscores.intensity_scale_init`\n and populated with\n :py:func:`pysteps.verification.spatialscores.intensity_scale_accum`.\n intscale_2: dict\n Another intensity-scale object initialized with\n :py:func:`pysteps.verification.spatialscores.intensity_scale_init`\n and populated with\n :py:func:`pysteps.verification.spatialscores.intensity_scale_accum`.\n\n Returns\n -------\n out: dict\n The merged intensity-scale object.\n \"\"\"\n\n # checks\n if intscale_1[\"name\"] != intscale_2[\"name\"]:\n raise ValueError(\n \"cannot merge: the intensity scale methods are not same %s!=%s\"\n % (intscale_1[\"name\"], intscale_2[\"name\"])\n )\n\n intscale = intscale_1.copy()\n name = intscale[\"name\"]\n thrs = intscale[\"thrs\"]\n scales = intscale[\"scales\"]\n\n for i, thr in enumerate(thrs):\n if name.lower() == \"bmse\":\n intscale[thr] = binary_mse_merge(intscale[thr], intscale_2[thr])\n\n elif name.lower() == \"fss\":\n for j, scale in enumerate(scales):\n intscale[thr][scale] = fss_merge(\n intscale[thr][scale], intscale_2[thr][scale]\n )\n\n return intscale\n\n\ndef intensity_scale_compute(intscale):\n \"\"\"\n Return the intensity scale matrix.\n\n Parameters\n ----------\n intscale: dict\n The intensity-scale object initialized with\n :py:func:`pysteps.verification.spatialscores.intensity_scale_init`\n and accumulated with\n :py:func:`pysteps.verification.spatialscores.intensity_scale_accum`.\n\n Returns\n -------\n out: array_like\n The two-dimensional array of shape (j, k) containing\n the intensity-scale skill scores for **j** spatial scales and\n **k** intensity thresholds.\n \"\"\"\n\n name = intscale[\"name\"]\n thrs = intscale[\"thrs\"]\n scales = intscale[\"scales\"]\n\n SS = np.zeros((scales.size, thrs.size))\n\n for i, thr in enumerate(thrs):\n if name.lower() == \"bmse\":\n SS[:, i] = binary_mse_compute(intscale[thr], False)\n\n elif name.lower() == \"fss\":\n for j, scale in enumerate(scales):\n SS[j, i] = fss_compute(intscale[thr][scale])\n\n return SS\n\n\ndef binary_mse(X_f, X_o, thr, wavelet=\"haar\", return_scales=True):\n \"\"\"\n Compute the MSE of the binary error as a function of spatial scale.\n\n This method uses PyWavelets for decomposing the error field between the\n forecasts and observations into multiple spatial scales.\n\n Parameters\n ----------\n X_f: array_like\n Array of shape (m, n) containing the forecast field.\n X_o: array_like\n Array of shape (m, n) containing the verification observation field.\n thr: sequence\n The intensity threshold for which to compute the verification.\n wavelet: str, optional\n The name of the wavelet function to use. Defaults to the Haar wavelet,\n as described in Casati et al. 2004. See the documentation of PyWavelets\n for a list of available options.\n return_scales: bool, optional\n Whether to return the spatial scales resulting from the wavelet\n decomposition.\n\n Returns\n -------\n SS: array\n One-dimensional array containing the binary MSE for each spatial scale.\n scales: list, optional\n If ``return_scales=True``, return the spatial scales in pixels resulting\n from the wavelet decomposition.\n\n References\n ----------\n :cite:`CRS2004`\n \"\"\"\n\n bmse = binary_mse_init(thr, wavelet)\n binary_mse_accum(bmse, X_f, X_o)\n return binary_mse_compute(bmse, return_scales)\n\n\ndef binary_mse_init(thr, wavelet=\"haar\"):\n \"\"\"\n Initialize a binary MSE (BMSE) verification object.\n\n Parameters\n ----------\n thr: float\n The intensity threshold.\n wavelet: str, optional\n The name of the wavelet function to use. Defaults to the Haar wavelet,\n as described in Casati et al. 2004. See the documentation of PyWavelets\n for a list of available options.\n\n Returns\n -------\n bmse: dict\n The initialized BMSE verification object.\n \"\"\"\n\n bmse = dict(thr=thr, wavelet=wavelet, scales=None, mse=None, eps=0, n=0)\n\n return bmse\n\n\ndef binary_mse_accum(bmse, X_f, X_o):\n \"\"\"Accumulate forecast-observation pairs to an BMSE object.\n\n Parameters\n -----------\n bmse: dict\n The BMSE object initialized with\n :py:func:`pysteps.verification.spatialscores.binary_mse_init`.\n X_f: array_like\n Array of shape (m, n) containing the forecast field.\n X_o: array_like\n Array of shape (m, n) containing the observation field.\n \"\"\"\n if not pywt_imported:\n raise MissingOptionalDependency(\n \"PyWavelets package is required for the binary MSE spatial \"\n \"verification method but it is not installed\"\n )\n\n if len(X_f.shape) != 2 or len(X_o.shape) != 2 or X_f.shape != X_o.shape:\n message = \"X_f and X_o must be two-dimensional arrays\"\n message += \" having the same shape\"\n raise ValueError(message)\n\n thr = bmse[\"thr\"]\n wavelet = bmse[\"wavelet\"]\n\n X_f = X_f.copy()\n X_f[~np.isfinite(X_f)] = thr - 1\n X_o = X_o.copy()\n X_o[~np.isfinite(X_o)] = thr - 1\n\n w = pywt.Wavelet(wavelet)\n\n I_f = (X_f >= thr).astype(float)\n I_o = (X_o >= thr).astype(float)\n\n E_decomp = _wavelet_decomp(I_f - I_o, w)\n\n n_scales = len(E_decomp)\n if bmse[\"scales\"] is None:\n bmse[\"scales\"] = pow(2, np.arange(n_scales))[::-1]\n bmse[\"mse\"] = np.zeros(n_scales)\n\n # update eps\n eps = 1.0 * np.sum((X_o >= thr).astype(int)) / X_o.size\n if np.isfinite(eps):\n bmse[\"eps\"] = (bmse[\"eps\"] * bmse[\"n\"] + eps) / (bmse[\"n\"] + 1)\n\n # update mse\n for j in range(n_scales):\n mse = np.mean(E_decomp[j] ** 2)\n if np.isfinite(mse):\n bmse[\"mse\"][j] = (bmse[\"mse\"][j] * bmse[\"n\"] + mse) / (bmse[\"n\"] + 1)\n\n bmse[\"n\"] += 1\n\n\ndef binary_mse_merge(bmse_1, bmse_2):\n \"\"\"\n Merge two BMSE objects.\n\n Parameters\n ----------\n bmse_1: dict\n A BMSE object initialized with\n :py:func:`pysteps.verification.spatialscores.binary_mse_init`.\n and populated with\n :py:func:`pysteps.verification.spatialscores.binary_mse_accum`.\n bmse_2: dict\n Another BMSE object initialized with\n :py:func:`pysteps.verification.spatialscores.binary_mse_init`.\n and populated with\n :py:func:`pysteps.verification.spatialscores.binary_mse_accum`.\n\n Returns\n -------\n out: dict\n The merged BMSE object.\n \"\"\"\n\n # checks\n if bmse_1[\"thr\"] != bmse_2[\"thr\"]:\n raise ValueError(\n \"cannot merge: the thresholds are not same %s!=%s\"\n % (bmse_1[\"thr\"], bmse_2[\"thr\"])\n )\n if bmse_1[\"wavelet\"] != bmse_2[\"wavelet\"]:\n raise ValueError(\n \"cannot merge: the wavelets are not same %s!=%s\"\n % (bmse_1[\"wavelet\"], bmse_2[\"wavelet\"])\n )\n if list(bmse_1[\"scales\"]) != list(bmse_2[\"scales\"]):\n raise ValueError(\n \"cannot merge: the scales are not same %s!=%s\"\n % (bmse_1[\"scales\"], bmse_2[\"scales\"])\n )\n\n # merge the BMSE objects\n bmse = bmse_1.copy()\n bmse[\"eps\"] = (bmse[\"eps\"] * bmse[\"n\"] + bmse_2[\"eps\"] * bmse_2[\"n\"]) / (\n bmse[\"n\"] + bmse_2[\"n\"]\n )\n for j, scale in enumerate(bmse[\"scales\"]):\n bmse[\"mse\"][j] = (\n bmse[\"mse\"][j] * bmse[\"n\"] + bmse_2[\"mse\"][j] * bmse_2[\"n\"]\n ) / (bmse[\"n\"] + bmse_2[\"n\"])\n bmse[\"n\"] += bmse_2[\"n\"]\n\n return bmse\n\n\ndef binary_mse_compute(bmse, return_scales=True):\n \"\"\"\n Compute the BMSE.\n\n Parameters\n ----------\n bmse: dict\n The BMSE object initialized with\n :py:func:`pysteps.verification.spatialscores.binary_mse_init`\n and accumulated with\n :py:func:`pysteps.verification.spatialscores.binary_mse_accum`.\n return_scales: bool, optional\n Whether to return the spatial scales resulting from the wavelet\n decomposition.\n\n Returns\n -------\n BMSE: array_like\n One-dimensional array containing the binary MSE for each spatial scale.\n scales: list, optional\n If ``return_scales=True``, return the spatial scales in pixels resulting\n from the wavelet decomposition.\n \"\"\"\n\n scales = bmse[\"scales\"]\n n_scales = len(scales)\n eps = bmse[\"eps\"]\n\n BMSE = np.zeros(n_scales)\n for j in range(n_scales):\n mse = bmse[\"mse\"][j]\n BMSE[j] = 1 - mse / (2 * eps * (1 - eps) / n_scales)\n\n BMSE[~np.isfinite(BMSE)] = np.nan\n\n if return_scales:\n return BMSE, scales\n else:\n return BMSE\n\n\ndef fss(X_f, X_o, thr, scale):\n \"\"\"\n Compute the fractions skill score (FSS) for a deterministic forecast field\n and the corresponding observation field.\n\n Parameters\n ----------\n X_f: array_like\n Array of shape (m, n) containing the forecast field.\n X_o: array_like\n Array of shape (m, n) containing the observation field.\n thr: float\n The intensity threshold.\n scale: int\n The spatial scale in pixels. In practice, the scale represents the size\n of the moving window that it is used to compute the fraction of pixels\n above the threshold.\n\n Returns\n -------\n out: float\n The fractions skill score between 0 and 1.\n\n References\n ----------\n :cite:`RL2008`, :cite:`EWWM2013`\n \"\"\"\n\n fss = fss_init(thr, scale)\n fss_accum(fss, X_f, X_o)\n return fss_compute(fss)\n\n\ndef fss_init(thr, scale):\n \"\"\"\n Initialize a fractions skill score (FSS) verification object.\n\n Parameters\n ----------\n thr: float\n The intensity threshold.\n scale: float\n The spatial scale in pixels. In practice, the scale represents the size\n of the moving window that it is used to compute the fraction of pixels\n above the threshold.\n\n Returns\n -------\n fss: dict\n The initialized FSS verification object.\n \"\"\"\n fss = dict(thr=thr, scale=scale, sum_fct_sq=0.0, sum_fct_obs=0.0, sum_obs_sq=0.0)\n\n return fss\n\n\ndef fss_accum(fss, X_f, X_o):\n \"\"\"Accumulate forecast-observation pairs to an FSS object.\n\n Parameters\n -----------\n fss: dict\n The FSS object initialized with\n :py:func:`pysteps.verification.spatialscores.fss_init`.\n X_f: array_like\n Array of shape (m, n) containing the forecast field.\n X_o: array_like\n Array of shape (m, n) containing the observation field.\n \"\"\"\n if len(X_f.shape) != 2 or len(X_o.shape) != 2 or X_f.shape != X_o.shape:\n message = \"X_f and X_o must be two-dimensional arrays\"\n message += \" having the same shape\"\n raise ValueError(message)\n\n X_f = X_f.copy()\n X_f[~np.isfinite(X_f)] = fss[\"thr\"] - 1\n X_o = X_o.copy()\n X_o[~np.isfinite(X_o)] = fss[\"thr\"] - 1\n\n # Convert to binary fields with the given intensity threshold\n I_f = (X_f >= fss[\"thr\"]).astype(float)\n I_o = (X_o >= fss[\"thr\"]).astype(float)\n\n # Compute fractions of pixels above the threshold within a square\n # neighboring area by applying a 2D moving average to the binary fields\n if fss[\"scale\"] > 1:\n S_f = uniform_filter(I_f, size=fss[\"scale\"], mode=\"constant\", cval=0.0)\n S_o = uniform_filter(I_o, size=fss[\"scale\"], mode=\"constant\", cval=0.0)\n else:\n S_f = I_f\n S_o = I_o\n\n fss[\"sum_obs_sq\"] += np.nansum(S_o**2)\n fss[\"sum_fct_obs\"] += np.nansum(S_f * S_o)\n fss[\"sum_fct_sq\"] += np.nansum(S_f**2)\n\n\ndef fss_merge(fss_1, fss_2):\n \"\"\"\n Merge two FSS objects.\n\n Parameters\n ----------\n fss_1: dict\n A FSS object initialized with\n :py:func:`pysteps.verification.spatialscores.fss_init`.\n and populated with\n :py:func:`pysteps.verification.spatialscores.fss_accum`.\n fss_2: dict\n Another FSS object initialized with\n :py:func:`pysteps.verification.spatialscores.fss_init`.\n and populated with\n :py:func:`pysteps.verification.spatialscores.fss_accum`.\n\n Returns\n -------\n out: dict\n The merged FSS object.\n \"\"\"\n\n # checks\n if fss_1[\"thr\"] != fss_2[\"thr\"]:\n raise ValueError(\n \"cannot merge: the thresholds are not same %s!=%s\"\n % (fss_1[\"thr\"], fss_2[\"thr\"])\n )\n if fss_1[\"scale\"] != fss_2[\"scale\"]:\n raise ValueError(\n \"cannot merge: the scales are not same %s!=%s\"\n % (fss_1[\"scale\"], fss_2[\"scale\"])\n )\n\n # merge the FSS objects\n fss = fss_1.copy()\n fss[\"sum_obs_sq\"] += fss_2[\"sum_obs_sq\"]\n fss[\"sum_fct_obs\"] += fss_2[\"sum_fct_obs\"]\n fss[\"sum_fct_sq\"] += fss_2[\"sum_fct_sq\"]\n\n return fss\n\n\ndef fss_compute(fss):\n \"\"\"\n Compute the FSS.\n\n Parameters\n ----------\n fss: dict\n An FSS object initialized with\n :py:func:`pysteps.verification.spatialscores.fss_init`\n and accumulated with\n :py:func:`pysteps.verification.spatialscores.fss_accum`.\n\n Returns\n -------\n out: float\n The computed FSS value.\n \"\"\"\n numer = fss[\"sum_fct_sq\"] - 2.0 * fss[\"sum_fct_obs\"] + fss[\"sum_obs_sq\"]\n denom = fss[\"sum_fct_sq\"] + fss[\"sum_obs_sq\"]\n\n return 1.0 - numer / denom\n\n\ndef _wavelet_decomp(X, w):\n c = pywt.wavedec2(X, w)\n\n X_out = []\n for k in range(len(c)):\n c_ = c[:]\n for k_ in set(range(len(c))).difference([k]):\n c_[k_] = tuple([np.zeros_like(v) for v in c[k_]])\n X_k = pywt.waverec2(c_, w)\n X_out.append(X_k)\n\n return X_out\n" }, { "path": "pysteps/visualization/__init__.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"Methods for plotting precipitation and motion fields.\"\"\"\n\nfrom .motionfields import *\nfrom .precipfields import *\nfrom .animations import *\nfrom .spectral import *\nfrom .thunderstorms import *\n" }, { "path": "pysteps/visualization/animations.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\npysteps.visualization.animations\r\n================================\r\n\r\nFunctions to produce animations for pysteps.\r\n\r\n.. autosummary::\r\n :toctree: ../generated/\r\n\r\n animate\r\n\"\"\"\r\n\r\nimport os\r\nimport warnings\r\n\r\nimport matplotlib.pylab as plt\r\nimport pysteps as st\r\n\r\nPRECIP_VALID_TYPES = (\"ensemble\", \"mean\", \"prob\")\r\nMOTION_VALID_METHODS = (\"quiver\", \"streamplot\")\r\n\r\n\r\ndef animate(\r\n precip_obs,\r\n precip_fct=None,\r\n timestamps_obs=None,\r\n timestep_min=None,\r\n motion_field=None,\r\n ptype=\"ensemble\",\r\n motion_plot=\"quiver\",\r\n geodata=None,\r\n title=None,\r\n prob_thr=None,\r\n display_animation=True,\r\n nloops=1,\r\n time_wait=0.2,\r\n savefig=False,\r\n fig_dpi=100,\r\n fig_format=\"png\",\r\n path_outputs=\"\",\r\n precip_kwargs=None,\r\n motion_kwargs=None,\r\n map_kwargs=None,\r\n):\r\n \"\"\"\r\n Function to animate observations and forecasts in pysteps.\r\n\r\n It also allows to export the individual frames as figures, which\r\n is useful for constructing animated GIFs or similar.\r\n\r\n .. _Axes: https://matplotlib.org/api/axes_api.html#matplotlib.axes.Axes\r\n\r\n Parameters\r\n ----------\r\n precip_obs: array-like\r\n Three-dimensional array containing the time series of observed\r\n precipitation fields.\r\n precip_fct: array-like, optional\r\n The three or four-dimensional (for ensembles) array\r\n containing the time series of forecasted precipitation field.\r\n timestamps_obs: list of datetimes, optional\r\n List of datetime objects corresponding to the time stamps of\r\n the fields in precip_obs.\r\n timestep_min: float, optional\r\n The time resolution in minutes of the forecast.\r\n motion_field: array-like, optional\r\n Three-dimensional array containing the u and v components of\r\n the motion field.\r\n motion_plot: string, optional\r\n The method to plot the motion field. See plot methods in\r\n :py:mod:`pysteps.visualization.motionfields`.\r\n geodata: dictionary or None, optional\r\n Dictionary containing geographical information about\r\n the field.\r\n If geodata is not None, it must contain the following key-value pairs:\r\n\r\n .. tabularcolumns:: |p{1.5cm}|L|\r\n\r\n +----------------+----------------------------------------------------+\r\n | Key | Value |\r\n +================+====================================================+\r\n | projection | PROJ.4-compatible projection definition |\r\n +----------------+----------------------------------------------------+\r\n | x1 | x-coordinate of the lower-left corner of the data |\r\n | | raster |\r\n +----------------+----------------------------------------------------+\r\n | y1 | y-coordinate of the lower-left corner of the data |\r\n | | raster |\r\n +----------------+----------------------------------------------------+\r\n | x2 | x-coordinate of the upper-right corner of the data |\r\n | | raster |\r\n +----------------+----------------------------------------------------+\r\n | y2 | y-coordinate of the upper-right corner of the data |\r\n | | raster |\r\n +----------------+----------------------------------------------------+\r\n | yorigin | a string specifying the location of the first |\r\n | | element in the data raster w.r.t. y-axis: |\r\n | | 'upper' = upper border, 'lower' = lower border |\r\n +----------------+----------------------------------------------------+\r\n\r\n title: str or None, optional\r\n If not None, print the string as title on top of the plot.\r\n ptype: {'ensemble', 'mean', 'prob'}, str, optional\r\n Type of the plot to animate. 'ensemble' = ensemble members,\r\n 'mean' = ensemble mean, 'prob' = exceedance probability\r\n (using threshold defined in prob_thrs).\r\n prob_thr: float, optional\r\n Intensity threshold for the exceedance probability maps. Applicable\r\n if ptype = 'prob'.\r\n display_animation: bool, optional\r\n If set to True, display the animation (set to False if only\r\n interested in saving the animation frames).\r\n nloops: int, optional\r\n The number of loops in the animation.\r\n time_wait: float, optional\r\n The time in seconds between one frame and the next. Applicable\r\n if display_animation is True.\r\n savefig: bool, optional\r\n If set to True, save the individual frames into path_outputs.\r\n fig_dpi: float, optional\r\n The resolution in dots per inch. Applicable if savefig is True.\r\n fig_format: str, optional\r\n Filename extension. Applicable if savefig is True.\r\n path_outputs: string, optional\r\n Path to folder where to save the frames. Applicable if savefig is True.\r\n precip_kwargs: dict, optional\r\n Optional parameters that are supplied to\r\n :py:func:`pysteps.visualization.precipfields.plot_precip_field`.\r\n motion_kwargs: dict, optional\r\n Optional parameters that are supplied to\r\n :py:func:`pysteps.visualization.motionfields.quiver` or\r\n :py:func:`pysteps.visualization.motionfields.streamplot`.\r\n map_kwargs: dict, optional\r\n Optional parameters that need to be passed to\r\n :py:func:`pysteps.visualization.basemaps.plot_geography`.\r\n\r\n Returns\r\n -------\r\n None\r\n \"\"\"\r\n\r\n if precip_kwargs is None:\r\n precip_kwargs = {}\r\n\r\n if motion_kwargs is None:\r\n motion_kwargs = {}\r\n\r\n if map_kwargs is None:\r\n map_kwargs = {}\r\n\r\n if precip_fct is not None:\r\n if len(precip_fct.shape) == 3:\r\n precip_fct = precip_fct[None, ...]\r\n n_lead_times = precip_fct.shape[1]\r\n n_members = precip_fct.shape[0]\r\n else:\r\n n_lead_times = 0\r\n n_members = 1\r\n\r\n if title is not None and isinstance(title, str):\r\n title_first_line = title + \"\\n\"\r\n else:\r\n title_first_line = \"\"\r\n\r\n if motion_plot not in MOTION_VALID_METHODS:\r\n raise ValueError(\r\n f\"Invalid motion plot method '{motion_plot}'.\"\r\n f\"Supported: {str(MOTION_VALID_METHODS)}\"\r\n )\r\n\r\n if ptype not in PRECIP_VALID_TYPES:\r\n raise ValueError(\r\n f\"Invalid precipitation type '{ptype}'.\"\r\n f\"Supported: {str(PRECIP_VALID_TYPES)}\"\r\n )\r\n\r\n if timestamps_obs is not None:\r\n if len(timestamps_obs) != precip_obs.shape[0]:\r\n raise ValueError(\r\n f\"The number of timestamps does not match the size of precip_obs: \"\r\n f\"{len(timestamps_obs)} != {precip_obs.shape[0]}\"\r\n )\r\n if precip_fct is not None:\r\n reftime_str = timestamps_obs[-1].strftime(\"%Y%m%d%H%M\")\r\n else:\r\n reftime_str = timestamps_obs[0].strftime(\"%Y%m%d%H%M\")\r\n else:\r\n reftime_str = None\r\n\r\n if ptype == \"prob\" and prob_thr is None:\r\n raise ValueError(\"ptype 'prob' needs a prob_thr value\")\r\n\r\n if ptype != \"ensemble\":\r\n n_members = 1\r\n\r\n n_obs = precip_obs.shape[0]\r\n\r\n loop = 0\r\n while loop < nloops:\r\n for n in range(n_members):\r\n for i in range(n_obs + n_lead_times):\r\n plt.clf()\r\n\r\n # Observations\r\n if i < n_obs and (display_animation or n == 0):\r\n title = title_first_line + \"Analysis\"\r\n if timestamps_obs is not None:\r\n title += (\r\n f\" valid for {timestamps_obs[i].strftime('%Y-%m-%d %H:%M')}\"\r\n )\r\n\r\n plt.clf()\r\n if ptype == \"prob\":\r\n prob_field = st.postprocessing.ensemblestats.excprob(\r\n precip_obs[None, i, ...], prob_thr\r\n )\r\n ax = st.plt.plot_precip_field(\r\n prob_field,\r\n ptype=\"prob\",\r\n geodata=geodata,\r\n probthr=prob_thr,\r\n title=title,\r\n map_kwargs=map_kwargs,\r\n **precip_kwargs,\r\n )\r\n else:\r\n ax = st.plt.plot_precip_field(\r\n precip_obs[i, :, :],\r\n geodata=geodata,\r\n title=title,\r\n map_kwargs=map_kwargs,\r\n **precip_kwargs,\r\n )\r\n\r\n if motion_field is not None:\r\n if motion_plot == \"quiver\":\r\n st.plt.quiver(\r\n motion_field, ax=ax, geodata=geodata, **motion_kwargs\r\n )\r\n elif motion_plot == \"streamplot\":\r\n st.plt.streamplot(\r\n motion_field, ax=ax, geodata=geodata, **motion_kwargs\r\n )\r\n\r\n if savefig & (loop == 0):\r\n figtags = [reftime_str, ptype, f\"f{i:02d}\"]\r\n figname = \"_\".join([tag for tag in figtags if tag])\r\n filename = os.path.join(path_outputs, f\"{figname}.{fig_format}\")\r\n plt.savefig(filename, bbox_inches=\"tight\", dpi=fig_dpi)\r\n print(\"saved: \", filename)\r\n\r\n # Forecasts\r\n elif i >= n_obs and precip_fct is not None:\r\n title = title_first_line + \"Forecast\"\r\n if timestamps_obs is not None:\r\n title += f\" valid for {timestamps_obs[-1].strftime('%Y-%m-%d %H:%M')}\"\r\n if timestep_min is not None:\r\n title += \" +%02d min\" % ((1 + i - n_obs) * timestep_min)\r\n else:\r\n title += \" +%02d\" % (1 + i - n_obs)\r\n\r\n plt.clf()\r\n if ptype == \"prob\":\r\n prob_field = st.postprocessing.ensemblestats.excprob(\r\n precip_fct[:, i - n_obs, :, :], prob_thr\r\n )\r\n ax = st.plt.plot_precip_field(\r\n prob_field,\r\n ptype=\"prob\",\r\n geodata=geodata,\r\n probthr=prob_thr,\r\n title=title,\r\n map_kwargs=map_kwargs,\r\n **precip_kwargs,\r\n )\r\n elif ptype == \"mean\":\r\n ens_mean = st.postprocessing.ensemblestats.mean(\r\n precip_fct[:, i - n_obs, :, :]\r\n )\r\n ax = st.plt.plot_precip_field(\r\n ens_mean,\r\n geodata=geodata,\r\n title=title,\r\n map_kwargs=map_kwargs,\r\n **precip_kwargs,\r\n )\r\n else:\r\n ax = st.plt.plot_precip_field(\r\n precip_fct[n, i - n_obs, ...],\r\n geodata=geodata,\r\n title=title,\r\n map_kwargs=map_kwargs,\r\n **precip_kwargs,\r\n )\r\n\r\n if motion_field is not None:\r\n if motion_plot == \"quiver\":\r\n st.plt.quiver(\r\n motion_field, ax=ax, geodata=geodata, **motion_kwargs\r\n )\r\n elif motion_plot == \"streamplot\":\r\n st.plt.streamplot(\r\n motion_field, ax=ax, geodata=geodata, **motion_kwargs\r\n )\r\n\r\n if ptype == \"ensemble\" and n_members > 1:\r\n plt.text(\r\n 0.01,\r\n 0.99,\r\n \"m %02d\" % (n + 1),\r\n transform=ax.transAxes,\r\n ha=\"left\",\r\n va=\"top\",\r\n )\r\n\r\n if savefig & (loop == 0):\r\n figtags = [reftime_str, ptype, f\"f{i:02d}\", f\"m{n + 1:02d}\"]\r\n figname = \"_\".join([tag for tag in figtags if tag])\r\n filename = os.path.join(path_outputs, f\"{figname}.{fig_format}\")\r\n plt.savefig(filename, bbox_inches=\"tight\", dpi=fig_dpi)\r\n print(\"saved: \", filename)\r\n\r\n if display_animation:\r\n plt.pause(time_wait)\r\n\r\n if display_animation:\r\n plt.pause(2 * time_wait)\r\n\r\n loop += 1\r\n\r\n plt.close()\r\n" }, { "path": "pysteps/visualization/basemaps.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\npysteps.visualization.basemaps\r\n==============================\r\n\r\nMethods for plotting geographical maps using Cartopy.\r\n\r\n.. autosummary::\r\n :toctree: ../generated/\r\n\r\n plot_geography\r\n plot_map_cartopy\r\n\"\"\"\r\n\r\nfrom matplotlib import gridspec\r\nimport matplotlib.pylab as plt\r\nimport numpy as np\r\nimport warnings\r\nfrom pysteps.exceptions import MissingOptionalDependency\r\n\r\ntry:\r\n import cartopy.crs as ccrs\r\n import cartopy.feature as cfeature\r\n from cartopy.mpl.geoaxes import GeoAxesSubplot\r\n\r\n CARTOPY_IMPORTED = True\r\nexcept ImportError:\r\n CARTOPY_IMPORTED = False\r\ntry:\r\n import pyproj\r\n\r\n PYPROJ_IMPORTED = True\r\nexcept ImportError:\r\n PYPROJ_IMPORTED = False\r\n\r\nfrom . import utils\r\n\r\nVALID_BASEMAPS = (\"cartopy\",)\r\n\r\n\r\n#########################\r\n# Basemap features zorder\r\n# - ocean: 0\r\n# - land: 0\r\n# - lakes: 0\r\n# - rivers_lake_centerlines: 0\r\n# - coastline: 15\r\n# - cultural: 15\r\n# - reefs: 15\r\n# - minor_islands: 15\r\n\r\n\r\ndef plot_geography(\r\n proj4str,\r\n extent,\r\n lw=0.5,\r\n drawlonlatlines=False,\r\n drawlonlatlabels=True,\r\n plot_map=\"cartopy\",\r\n scale=\"50m\",\r\n subplot=None,\r\n **kwargs,\r\n):\r\n \"\"\"\r\n Plot geographical map in a chosen projection using cartopy.\r\n\r\n .. _SubplotSpec: https://matplotlib.org/api/_as_gen/matplotlib.gridspec.SubplotSpec.html\r\n\r\n Parameters\r\n ----------\r\n proj4str: str\r\n The PROJ.4-compatible projection string.\r\n extent: scalars (left, right, bottom, top)\r\n The bounding box in proj4str coordinates.\r\n lw: float, optional`\r\n Linewidth of the map (administrative boundaries and coastlines).\r\n drawlonlatlines: bool, optional\r\n If set to True, draw longitude and latitude lines.\r\n drawlonlatlabels: bool, optional\r\n If set to True, draw longitude and latitude labels. Valid only if\r\n 'drawlonlatlines' is True.\r\n plot_map: {'cartopy', None}, optional\r\n The type of basemap, either 'cartopy' or None. If None, the figure\r\n axis is returned without any basemap drawn. Default `'cartopy'`.\r\n scale: {'10m', '50m', '110m'}, optional\r\n The scale (resolution). Applicable if 'plot_map' is 'cartopy'.\r\n The available options are '10m', '50m', and '110m'. Default ``'50m'``.\r\n subplot: tuple of int (nrows, ncols, index) or SubplotSpec_ instance, optional\r\n The subplot where to plot the basemap.\r\n By the default, the basemap will replace the current axis.\r\n\r\n Returns\r\n -------\r\n ax: fig Axes\r\n Cartopy axes.\r\n \"\"\"\r\n\r\n if len(kwargs) > 0:\r\n warnings.warn(\r\n \"plot_geography: The following keywords are ignored:\\n\"\r\n + str(kwargs)\r\n + \"\\nIn version 1.5, passing unsupported arguments will raise an error.\",\r\n DeprecationWarning,\r\n )\r\n\r\n if plot_map is None:\r\n return plt.gca()\r\n\r\n if plot_map not in VALID_BASEMAPS:\r\n raise ValueError(\r\n f\"unsupported plot_map method {plot_map}. Supported basemaps: \"\r\n f\"{VALID_BASEMAPS}\"\r\n )\r\n\r\n if plot_map == \"cartopy\" and not CARTOPY_IMPORTED:\r\n warnings.warn(\r\n \"The cartopy package is required to plot the geographical map but it is \"\r\n \"not installed. Ignoring the geographic information.\"\r\n )\r\n return plt.gca()\r\n\r\n if not PYPROJ_IMPORTED:\r\n warnings.warn(\r\n \"the pyproj package is required to plot the geographical map \"\r\n \"but it is not installed\"\r\n )\r\n return plt.gca()\r\n\r\n crs = utils.proj4_to_cartopy(proj4str)\r\n\r\n ax = plot_map_cartopy(\r\n crs,\r\n extent,\r\n scale,\r\n drawlonlatlines=drawlonlatlines,\r\n drawlonlatlabels=drawlonlatlabels,\r\n lw=lw,\r\n subplot=subplot,\r\n )\r\n\r\n return ax\r\n\r\n\r\ndef plot_map_cartopy(\r\n crs,\r\n extent,\r\n cartopy_scale,\r\n drawlonlatlines=False,\r\n drawlonlatlabels=True,\r\n lw=0.5,\r\n subplot=None,\r\n):\r\n \"\"\"\r\n Plot coastlines, countries, rivers and meridians/parallels using cartopy.\r\n\r\n .. _SubplotSpec: https://matplotlib.org/api/_as_gen/matplotlib.gridspec.SubplotSpec.html\r\n\r\n Parameters\r\n ----------\r\n crs: object\r\n Instance of a crs class defined in cartopy.crs.\r\n It can be created using utils.proj4_to_cartopy.\r\n extent: scalars (left, right, bottom, top)\r\n The coordinates of the bounding box.\r\n drawlonlatlines: bool\r\n Whether to plot longitudes and latitudes.\r\n drawlonlatlabels: bool, optional\r\n If set to True, draw longitude and latitude labels. Valid only if\r\n 'drawlonlatlines' is True.\r\n cartopy_scale: {'10m', '50m', '110m'}\r\n The scale (resolution) of the map. The available options are '10m',\r\n '50m', and '110m'.\r\n lw: float\r\n Line width.\r\n subplot: tuple of int (nrows, ncols, index) or SubplotSpec_ instance, optional\r\n The subplot where to place the basemap.\r\n By the default, the basemap will replace the current axis.\r\n\r\n Returns\r\n -------\r\n ax: axes\r\n Cartopy axes. Compatible with matplotlib.\r\n \"\"\"\r\n if not CARTOPY_IMPORTED:\r\n raise MissingOptionalDependency(\r\n \"the cartopy package is required to plot the geographical map\"\r\n \" but it is not installed\"\r\n )\r\n\r\n if subplot is None:\r\n ax = plt.gca()\r\n else:\r\n if isinstance(subplot, gridspec.SubplotSpec):\r\n ax = plt.subplot(subplot, projection=crs)\r\n else:\r\n ax = plt.subplot(*subplot, projection=crs)\r\n\r\n if not isinstance(ax, GeoAxesSubplot):\r\n ax = plt.subplot(ax.get_subplotspec(), projection=crs)\r\n # cax.clear()\r\n ax.set_axis_off()\r\n\r\n ax.add_feature(\r\n cfeature.NaturalEarthFeature(\r\n \"physical\",\r\n \"ocean\",\r\n scale=\"50m\" if cartopy_scale == \"10m\" else cartopy_scale,\r\n edgecolor=\"none\",\r\n facecolor=np.array([0.59375, 0.71484375, 0.8828125]),\r\n ),\r\n zorder=0,\r\n )\r\n ax.add_feature(\r\n cfeature.NaturalEarthFeature(\r\n \"physical\",\r\n \"land\",\r\n scale=cartopy_scale,\r\n edgecolor=\"none\",\r\n facecolor=np.array([0.9375, 0.9375, 0.859375]),\r\n ),\r\n zorder=0,\r\n )\r\n ax.add_feature(\r\n cfeature.NaturalEarthFeature(\r\n \"physical\",\r\n \"coastline\",\r\n scale=cartopy_scale,\r\n edgecolor=\"black\",\r\n facecolor=\"none\",\r\n linewidth=lw,\r\n ),\r\n zorder=15,\r\n )\r\n ax.add_feature(\r\n cfeature.NaturalEarthFeature(\r\n \"physical\",\r\n \"lakes\",\r\n scale=cartopy_scale,\r\n edgecolor=\"none\",\r\n facecolor=np.array([0.59375, 0.71484375, 0.8828125]),\r\n ),\r\n zorder=0,\r\n )\r\n ax.add_feature(\r\n cfeature.NaturalEarthFeature(\r\n \"physical\",\r\n \"rivers_lake_centerlines\",\r\n scale=cartopy_scale,\r\n edgecolor=np.array([0.59375, 0.71484375, 0.8828125]),\r\n facecolor=\"none\",\r\n ),\r\n zorder=0,\r\n )\r\n ax.add_feature(\r\n cfeature.NaturalEarthFeature(\r\n \"cultural\",\r\n \"admin_0_boundary_lines_land\",\r\n scale=cartopy_scale,\r\n edgecolor=\"black\",\r\n facecolor=\"none\",\r\n linewidth=lw,\r\n ),\r\n zorder=15,\r\n )\r\n if cartopy_scale in [\"10m\", \"50m\"]:\r\n ax.add_feature(\r\n cfeature.NaturalEarthFeature(\r\n \"physical\",\r\n \"reefs\",\r\n scale=\"10m\",\r\n edgecolor=\"black\",\r\n facecolor=\"none\",\r\n linewidth=lw,\r\n ),\r\n zorder=15,\r\n )\r\n ax.add_feature(\r\n cfeature.NaturalEarthFeature(\r\n \"physical\",\r\n \"minor_islands\",\r\n scale=\"10m\",\r\n edgecolor=\"black\",\r\n facecolor=\"none\",\r\n linewidth=lw,\r\n ),\r\n zorder=15,\r\n )\r\n\r\n if drawlonlatlines:\r\n grid_lines = ax.gridlines(\r\n crs=ccrs.PlateCarree(), draw_labels=drawlonlatlabels, dms=True\r\n )\r\n grid_lines.top_labels = grid_lines.right_labels = False\r\n grid_lines.y_inline = grid_lines.x_inline = False\r\n grid_lines.rotate_labels = False\r\n\r\n ax.set_extent(extent, crs)\r\n\r\n return ax\r\n" }, { "path": "pysteps/visualization/motionfields.py", "content": "# -- coding: utf-8 --\n\"\"\"\npysteps.visualization.motionfields\n==================================\n\nFunctions to plot motion fields.\n\n.. autosummary::\n :toctree: ../generated/\n\n motion_plot\n quiver\n streamplot\n\"\"\"\n\nfrom pysteps.visualization import utils\n\nVALID_PLOT_TYPES = (\"quiver\", \"streamplot\", \"stream\")\n\n\n#################################\n# Motion plots zorder definitions\n# - quiver: 20\n# - stream function: 30\n\n\ndef motion_plot(\n uv_motion_field,\n plot_type=\"quiver\",\n ax=None,\n geodata=None,\n axis=\"on\",\n plot_kwargs=None,\n map_kwargs=None,\n step=20,\n):\n \"\"\"\n Function to plot a motion field as arrows (quiver) or as stream lines (streamplot).\n\n .. _`quiver_doc`: https://matplotlib.org/api/_as_gen/matplotlib.pyplot.quiver.htm\n\n .. _`streamplot_doc`: https://matplotlib.org/api/_as_gen/matplotlib.pyplot.streamplot.html\n\n Parameters\n ----------\n uv_motion_field: array-like\n Array of shape (2,m,n) containing the input motion field.\n plot_type: str\n Plot type. \"quiver\" or \"streamplot\".\n ax: axis object\n Optional axis object to use for plotting.\n geodata: dictionary or None\n Optional dictionary containing geographical information about\n the field.\n\n If geodata is not None, it must contain the following key-value pairs:\n\n .. tabularcolumns:: |p{1.5cm}|L|\n\n +----------------+----------------------------------------------------+\n | Key | Value |\n +================+====================================================+\n | projection | PROJ.4-compatible projection definition |\n +----------------+----------------------------------------------------+\n | x1 | x-coordinate of the lower-left corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | y1 | y-coordinate of the lower-left corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | x2 | x-coordinate of the upper-right corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | y2 | y-coordinate of the upper-right corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | yorigin | a string specifying the location of the first |\n | | element in the data raster w.r.t. y-axis: |\n | | 'upper' = upper border, 'lower' = lower border |\n +----------------+----------------------------------------------------+\n axis: {'off','on'}, optional\n Whether to turn off or on the x and y axis.\n step: int\n Optional resample step to control the density of the arrows.\n plot_kwargs: dict, optional\n Optional dictionary containing keyword arguments passed to `quiver()` or\n `streamplot`.\n For more information, see the `quiver_doc`_ and `streamplot_doc`_ matplotlib's\n documentation.\n map_kwargs: dict\n Optional parameters that need to be passed to\n :py:func:`pysteps.visualization.basemaps.plot_geography`.\n\n Returns\n -------\n out: axis object\n Figure axes. Needed if one wants to add e.g. text inside the plot.\n \"\"\"\n if plot_type not in VALID_PLOT_TYPES:\n raise ValueError(\n f\"Invalid plot_type: {plot_type}.\\nSupported: {str(VALID_PLOT_TYPES)}\"\n )\n\n if plot_kwargs is None:\n plot_kwargs = {}\n if map_kwargs is None:\n map_kwargs = {}\n\n # Assumes the input dimensions are lat/lon\n _, nlat, nlon = uv_motion_field.shape\n\n x_grid, y_grid, extent, _, _ = utils.get_geogrid(nlat, nlon, geodata=geodata)\n\n ax = utils.get_basemap_axis(extent, ax=ax, geodata=geodata, map_kwargs=map_kwargs)\n\n ###########################################################\n # Undersample the number of grid points to use in the plots\n skip = (slice(None, None, step), slice(None, None, step))\n dx = uv_motion_field[0, :, :][skip]\n dy = uv_motion_field[1, :, :][skip].copy()\n x_grid = x_grid[skip]\n y_grid = y_grid[skip]\n\n # If we have yorigin\"=\"upper\" we flip the y axes for the motion field in the y axis.\n if geodata is None or geodata[\"yorigin\"] == \"upper\":\n dy *= -1\n\n if plot_type.lower() == \"quiver\":\n ax.quiver(x_grid, y_grid, dx, dy, angles=\"xy\", zorder=20, **plot_kwargs)\n else:\n ax.streamplot(x_grid, y_grid, dx, dy, zorder=30, **plot_kwargs)\n\n # Quiver sometimes do not produce tight axes\n ax.autoscale(enable=True, axis=\"both\", tight=True)\n\n if geodata is None or axis == \"off\":\n ax.xaxis.set_ticks([])\n ax.xaxis.set_ticklabels([])\n ax.yaxis.set_ticks([])\n ax.yaxis.set_ticklabels([])\n\n return ax\n\n\ndef quiver(\n uv_motion_field,\n ax=None,\n geodata=None,\n axis=\"on\",\n step=20,\n quiver_kwargs=None,\n map_kwargs=None,\n):\n \"\"\"Function to plot a motion field as arrows.\n Wrapper for :func:`pysteps.visualization.motionfields.motion_plot` passing\n `plot_type=\"quiver\"`.\n\n .. _`quiver_doc`: https://matplotlib.org/api/_as_gen/matplotlib.pyplot.quiver.html\n\n Parameters\n ----------\n uv_motion_field: array-like\n Array of shape (2, m,n) containing the input motion field.\n quiver_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the quiver method.\n This argument is passed to\n See the `quiver_doc`_ matplotlib's documentation.\n\n Other parameters\n ----------------\n See :py::func:`pysteps.visualization.motionfields.motion_plot`.\n\n Returns\n -------\n out: axis object0\n Figure axes. Needed if one wants to add e.g. text inside the plot.\n \"\"\"\n if quiver_kwargs is None:\n quiver_kwargs = dict()\n\n return motion_plot(\n uv_motion_field,\n plot_type=\"quiver\",\n ax=ax,\n geodata=geodata,\n axis=axis,\n step=step,\n plot_kwargs=quiver_kwargs,\n map_kwargs=map_kwargs,\n )\n\n\ndef streamplot(\n uv_motion_field,\n ax=None,\n geodata=None,\n axis=\"on\",\n streamplot_kwargs=None,\n map_kwargs=None,\n step=20,\n):\n \"\"\"\n Function to plot a motion field as streamlines.\n Wrapper for :func:`pysteps.visualization.motionfields.motion_plot` passing\n `plot_type=\"streamplot\"`.\n\n .. _`streamplot_doc`: https://matplotlib.org/api/_as_gen/matplotlib.pyplot.streamplot.html\n\n Parameters\n ----------\n uv_motion_field: array-like\n Array of shape (2, m,n) containing the input motion field.\n streamplot_kwargs: dict, optional\n Optional dictionary containing keyword arguments for the quiver method.\n This argument is passed to\n See the `streamplot_doc`_ matplotlib's documentation.\n\n Other parameters\n ----------------\n See :py:func:`pysteps.visualization.motionfields.motion_plot`.\n\n Returns\n -------\n out: axis object\n Figure axes. Needed if one wants to add e.g. text inside the plot.\n \"\"\"\n\n if streamplot_kwargs is None:\n streamplot_kwargs = dict()\n\n return motion_plot(\n uv_motion_field,\n plot_type=\"streamplot\",\n ax=ax,\n geodata=geodata,\n axis=axis,\n step=step,\n plot_kwargs=streamplot_kwargs,\n map_kwargs=map_kwargs,\n )\n" }, { "path": "pysteps/visualization/precipfields.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.visualization.precipfields\n==================================\n\nMethods for plotting precipitation fields.\n\n.. autosummary::\n :toctree: ../generated/\n\n plot_precip_field\n get_colormap\n\"\"\"\n\nimport copy\nimport warnings\n\nimport matplotlib.pylab as plt\nimport numpy as np\nfrom matplotlib import pyplot, colors\n\nfrom pysteps.visualization.utils import get_geogrid, get_basemap_axis\n\nPRECIP_VALID_TYPES = (\"intensity\", \"depth\", \"prob\")\nPRECIP_VALID_UNITS = (\"mm/h\", \"mm\", \"dBZ\")\n\n\n############################\n# precipitation plots zorder\n# - precipitation: 10\n\n\ndef plot_precip_field(\n precip,\n ptype=\"intensity\",\n ax=None,\n geodata=None,\n units=\"mm/h\",\n bbox=None,\n colorscale=\"pysteps\",\n probthr=None,\n title=None,\n colorbar=True,\n axis=\"on\",\n cax=None,\n map_kwargs=None,\n colormap_config=None,\n):\n \"\"\"\n Function to plot a precipitation intensity or probability field with a\n colorbar.\n\n .. _Axes: https://matplotlib.org/api/axes_api.html#matplotlib.axes.Axes\n\n .. _SubplotSpec: https://matplotlib.org/api/_as_gen/matplotlib.gridspec.SubplotSpec.html\n\n Parameters\n ----------\n precip: array-like\n Two-dimensional array containing the input precipitation field or an\n exceedance probability map.\n ptype: {'intensity', 'depth', 'prob'}, optional\n Type of the map to plot: 'intensity' = precipitation intensity field,\n 'depth' = precipitation depth (accumulation) field,\n 'prob' = exceedance probability field.\n geodata: dictionary or None, optional\n Optional dictionary containing geographical information about\n the field. Required is map is not None.\n\n If geodata is not None, it must contain the following key-value pairs:\n\n .. tabularcolumns:: |p{1.5cm}|L|\n\n +-----------------+---------------------------------------------------+\n | Key | Value |\n +=================+===================================================+\n | projection | PROJ.4-compatible projection definition |\n +-----------------+---------------------------------------------------+\n | x1 | x-coordinate of the lower-left corner of the data |\n | | raster |\n +-----------------+---------------------------------------------------+\n | y1 | y-coordinate of the lower-left corner of the data |\n | | raster |\n +-----------------+---------------------------------------------------+\n | x2 | x-coordinate of the upper-right corner of the |\n | | data raster |\n +-----------------+---------------------------------------------------+\n | y2 | y-coordinate of the upper-right corner of the |\n | | data raster |\n +-----------------+---------------------------------------------------+\n | yorigin | a string specifying the location of the first |\n | | element in the data raster w.r.t. y-axis: |\n | | 'upper' = upper border, 'lower' = lower border |\n +-----------------+---------------------------------------------------+\n units : {'mm/h', 'mm', 'dBZ'}, optional\n Units of the input array. If ptype is 'prob', this specifies the unit of\n the intensity threshold.\n bbox : tuple, optional\n Four-element tuple specifying the coordinates of the bounding box. Use\n this for plotting a subdomain inside the input grid. The coordinates are\n of the form (lower left x, lower left y ,upper right x, upper right y).\n If 'geodata' is not None, the bbox is in map coordinates, otherwise\n it represents image pixels.\n colorscale : {'pysteps', 'STEPS-BE', 'STEPS-NL', 'BOM-RF3'}, optional\n Which colorscale to use. Applicable if units is 'mm/h', 'mm' or 'dBZ'.\n probthr : float, optional\n Intensity threshold to show in the color bar of the exceedance\n probability map.\n Required if ptype is \"prob\" and colorbar is True.\n title : str, optional\n If not None, print the title on top of the plot.\n colorbar : bool, optional\n If set to True, add a colorbar on the right side of the plot.\n axis : {'off','on'}, optional\n Whether to turn off or on the x and y axis.\n cax : Axes_ object, optional\n Axes into which the colorbar will be drawn. If no axes is provided\n the colorbar axes are created next to the plot.\n colormap_config : ColormapConfig, optional\n Custom colormap configuration. If provided, this will override the\n colorscale parameter.\n The ColormapConfig class must have the following attributes: cmap,\n norm, clevs.\n\n Other parameters\n ----------------\n map_kwargs: dict\n Optional parameters that need to be passed to\n :py:func:`pysteps.visualization.basemaps.plot_geography`.\n\n Returns\n -------\n ax : fig Axes_\n Figure axes. Needed if one wants to add e.g. text inside the plot.\n \"\"\"\n\n if map_kwargs is None:\n map_kwargs = {}\n\n if ptype not in PRECIP_VALID_TYPES:\n raise ValueError(\n f\"Invalid precipitation type '{ptype}'.\"\n f\"Supported: {str(PRECIP_VALID_TYPES)}\"\n )\n\n if units not in PRECIP_VALID_UNITS:\n raise ValueError(\n f\"Invalid precipitation units '{units}.\"\n f\"Supported: {str(PRECIP_VALID_UNITS)}\"\n )\n\n if ptype == \"prob\" and colorbar and probthr is None:\n raise ValueError(\"ptype='prob' but probthr not specified\")\n\n if len(precip.shape) != 2:\n raise ValueError(\"The input is not two-dimensional array\")\n\n # Assumes the input dimensions are lat/lon\n nlat, nlon = precip.shape\n\n x_grid, y_grid, extent, regular_grid, origin = get_geogrid(\n nlat, nlon, geodata=geodata\n )\n\n ax = get_basemap_axis(extent, ax=ax, geodata=geodata, map_kwargs=map_kwargs)\n\n precip = np.ma.masked_invalid(precip)\n\n # Handle colormap configuration\n if colormap_config is None:\n cmap, norm, clevs, clevs_str = get_colormap(ptype, units, colorscale)\n else:\n cmap, norm, clevs = _validate_colormap_config(colormap_config, ptype)\n clevs_str = _dynamic_formatting_floats(clevs)\n\n # Plot the precipitation field\n if regular_grid:\n im = _plot_field(precip, ax, extent, cmap, norm, origin=origin)\n else:\n im = _plot_field(precip, ax, extent, cmap, norm, x_grid=x_grid, y_grid=y_grid)\n\n plt.title(title)\n\n # Add colorbar\n if colorbar:\n if ptype in [\"intensity\", \"depth\"]:\n extend = \"max\"\n else:\n extend = \"neither\"\n cbar = plt.colorbar(\n im, ticks=clevs, spacing=\"uniform\", extend=extend, shrink=0.8, cax=cax\n )\n if clevs_str is not None:\n cbar.ax.set_yticklabels(clevs_str)\n\n if ptype == \"intensity\":\n cbar.set_label(f\"Precipitation intensity [{units}]\")\n elif ptype == \"depth\":\n cbar.set_label(f\"Precipitation depth [{units}]\")\n else:\n cbar.set_label(f\"P(R > {probthr:.1f} {units})\")\n\n if geodata is None or axis == \"off\":\n ax.xaxis.set_ticks([])\n ax.xaxis.set_ticklabels([])\n ax.yaxis.set_ticks([])\n ax.yaxis.set_ticklabels([])\n\n if bbox is not None:\n ax.set_xlim(bbox[0], bbox[2])\n ax.set_ylim(bbox[1], bbox[3])\n\n return ax\n\n\ndef _plot_field(precip, ax, extent, cmap, norm, origin=None, x_grid=None, y_grid=None):\n precip = precip.copy()\n\n if (x_grid is None) or (y_grid is None):\n im = ax.imshow(\n precip,\n cmap=cmap,\n norm=norm,\n extent=extent,\n interpolation=\"nearest\",\n origin=origin,\n zorder=10,\n )\n else:\n im = ax.pcolormesh(\n x_grid,\n y_grid,\n precip,\n cmap=cmap,\n norm=norm,\n zorder=10,\n )\n\n return im\n\n\ndef get_colormap(ptype, units=\"mm/h\", colorscale=\"pysteps\"):\n \"\"\"\n Function to generate a colormap (cmap) and norm.\n\n Parameters\n ----------\n ptype : {'intensity', 'depth', 'prob'}, optional\n Type of the map to plot: 'intensity' = precipitation intensity field,\n 'depth' = precipitation depth (accumulation) field,\n 'prob' = exceedance probability field.\n units : {'mm/h', 'mm', 'dBZ'}, optional\n Units of the input array. If ptype is 'prob', this specifies the unit of\n the intensity threshold.\n colorscale : {'pysteps', 'STEPS-BE', 'STEPS-NL', 'BOM-RF3'}, optional\n Which colorscale to use. Applicable if units is 'mm/h', 'mm' or 'dBZ'.\n\n Returns\n -------\n cmap : Colormap instance\n colormap\n norm : colors.Normalize object\n Colors norm\n clevs: list(float)\n List of precipitation values defining the color limits.\n clevs_str: list(str)\n List of precipitation values defining the color limits (with correct\n number of decimals).\n \"\"\"\n if ptype in [\"intensity\", \"depth\"]:\n # Get list of colors\n color_list, clevs, clevs_str = _get_colorlist(units, colorscale)\n\n cmap = colors.LinearSegmentedColormap.from_list(\n \"cmap\", color_list, len(clevs) - 1\n )\n\n if colorscale == \"BOM-RF3\":\n cmap.set_over(\"black\", 1)\n if colorscale == \"pysteps\":\n cmap.set_over(\"darkred\", 1)\n if colorscale == \"STEPS-NL\":\n cmap.set_over(\"darkmagenta\", 1)\n if colorscale == \"STEPS-BE\":\n cmap.set_over(\"black\", 1)\n norm = colors.BoundaryNorm(clevs, cmap.N)\n\n cmap.set_bad(\"gray\", alpha=0.5)\n cmap.set_under(\"none\")\n\n return cmap, norm, clevs, clevs_str\n\n if ptype == \"prob\":\n cmap = copy.copy(plt.get_cmap(\"OrRd\", 10))\n cmap.set_bad(\"gray\", alpha=0.5)\n cmap.set_under(\"none\")\n clevs = np.linspace(0, 1, 11)\n clevs[0] = 1e-3 # to set zeros to transparent\n norm = colors.BoundaryNorm(clevs, cmap.N)\n clevs_str = [f\"{clev:.1f}\" for clev in clevs]\n return cmap, norm, clevs, clevs_str\n\n return pyplot.get_cmap(\"jet\"), colors.Normalize(), None, None\n\n\ndef _get_colorlist(units=\"mm/h\", colorscale=\"pysteps\"):\n \"\"\"\n Function to get a list of colors to generate the colormap.\n\n Parameters\n ----------\n units : str\n Units of the input array (mm/h, mm or dBZ)\n colorscale : str\n Which colorscale to use (BOM-RF3, pysteps, STEPS-BE, STEPS-NL)\n\n Returns\n -------\n color_list : list(str)\n List of color strings.\n\n clevs : list(float)\n List of precipitation values defining the color limits.\n\n clevs_str : list(str)\n List of precipitation values defining the color limits\n (with correct number of decimals).\n \"\"\"\n\n if colorscale == \"BOM-RF3\":\n color_list = np.array(\n [\n (255, 255, 255), # 0.0\n (245, 245, 255), # 0.2\n (180, 180, 255), # 0.5\n (120, 120, 255), # 1.5\n (20, 20, 255), # 2.5\n (0, 216, 195), # 4.0\n (0, 150, 144), # 6.0\n (0, 102, 102), # 10\n (255, 255, 0), # 15\n (255, 200, 0), # 20\n (255, 150, 0), # 30\n (255, 100, 0), # 40\n (255, 0, 0), # 50\n (200, 0, 0), # 60\n (120, 0, 0), # 75\n (40, 0, 0),\n ]\n ) # > 100\n color_list = color_list / 255.0\n if units == \"mm/h\":\n clevs = [\n 0.0,\n 0.2,\n 0.5,\n 1.5,\n 2.5,\n 4,\n 6,\n 10,\n 15,\n 20,\n 30,\n 40,\n 50,\n 60,\n 75,\n 100,\n 150,\n ]\n elif units == \"mm\":\n clevs = [\n 0.0,\n 0.2,\n 0.5,\n 1.5,\n 2.5,\n 4,\n 5,\n 7,\n 10,\n 15,\n 20,\n 25,\n 30,\n 35,\n 40,\n 45,\n 50,\n ]\n else:\n raise ValueError(\"Wrong units in get_colorlist: %s\" % units)\n elif colorscale == \"pysteps\":\n # pinkHex = '#%02x%02x%02x' % (232, 215, 242)\n redgrey_hex = \"#%02x%02x%02x\" % (156, 126, 148)\n color_list = [\n redgrey_hex,\n \"#640064\",\n \"#AF00AF\",\n \"#DC00DC\",\n \"#3232C8\",\n \"#0064FF\",\n \"#009696\",\n \"#00C832\",\n \"#64FF00\",\n \"#96FF00\",\n \"#C8FF00\",\n \"#FFFF00\",\n \"#FFC800\",\n \"#FFA000\",\n \"#FF7D00\",\n \"#E11900\",\n ]\n if units in [\"mm/h\", \"mm\"]:\n clevs = [\n 0.08,\n 0.16,\n 0.25,\n 0.40,\n 0.63,\n 1,\n 1.6,\n 2.5,\n 4,\n 6.3,\n 10,\n 16,\n 25,\n 40,\n 63,\n 100,\n 160,\n ]\n elif units == \"dBZ\":\n clevs = np.arange(10, 65, 5)\n else:\n raise ValueError(\"Wrong units in get_colorlist: %s\" % units)\n elif colorscale == \"STEPS-NL\":\n redgrey_hex = \"#%02x%02x%02x\" % (156, 126, 148)\n color_list = [\n \"lightgrey\",\n \"lightskyblue\",\n \"deepskyblue\",\n \"blue\",\n \"darkblue\",\n \"yellow\",\n \"gold\",\n \"darkorange\",\n \"red\",\n \"darkred\",\n ]\n if units in [\"mm/h\", \"mm\"]:\n clevs = [0.1, 0.5, 1.0, 1.6, 2.5, 4.0, 6.4, 10.0, 16.0, 25.0, 40.0]\n else:\n raise ValueError(\"Wrong units in get_colorlist: %s\" % units)\n elif colorscale == \"STEPS-BE\":\n color_list = [\n \"cyan\",\n \"deepskyblue\",\n \"dodgerblue\",\n \"blue\",\n \"chartreuse\",\n \"limegreen\",\n \"green\",\n \"darkgreen\",\n \"yellow\",\n \"gold\",\n \"orange\",\n \"red\",\n \"magenta\",\n \"darkmagenta\",\n ]\n if units in [\"mm/h\", \"mm\"]:\n clevs = [0.1, 0.25, 0.4, 0.63, 1, 1.6, 2.5, 4, 6.3, 10, 16, 25, 40, 63, 100]\n elif units == \"dBZ\":\n clevs = np.arange(10, 65, 5)\n else:\n raise ValueError(\"Wrong units in get_colorlist: %s\" % units)\n\n else:\n print(\"Invalid colorscale\", colorscale)\n raise ValueError(\"Invalid colorscale \" + colorscale)\n\n # Generate color level strings with correct amount of decimal places\n clevs_str = _dynamic_formatting_floats(clevs)\n\n return color_list, clevs, clevs_str\n\n\ndef _dynamic_formatting_floats(float_array, colorscale=\"pysteps\"):\n \"\"\"Function to format the floats defining the class limits of the colorbar.\"\"\"\n float_array = np.array(float_array, dtype=float)\n\n labels = []\n for label in float_array:\n if 0.1 <= label < 1:\n if colorscale == \"pysteps\":\n formatting = \",.2f\"\n else:\n formatting = \",.1f\"\n elif 0.01 <= label < 0.1:\n formatting = \",.2f\"\n elif 0.001 <= label < 0.01:\n formatting = \",.3f\"\n elif 0.0001 <= label < 0.001:\n formatting = \",.4f\"\n elif label >= 1 and label.is_integer():\n formatting = \"i\"\n else:\n formatting = \",.1f\"\n\n if formatting != \"i\":\n labels.append(format(label, formatting))\n else:\n labels.append(str(int(label)))\n\n return labels\n\n\ndef _validate_colormap_config(colormap_config, ptype):\n \"\"\"Validate the colormap configuration provided by the user.\"\"\"\n\n # Ensure colormap_config has the necessary attributes\n required_attrs = [\"cmap\", \"norm\", \"clevs\"]\n missing_attrs = [\n attr for attr in required_attrs if not hasattr(colormap_config, attr)\n ]\n if missing_attrs:\n raise ValueError(\n f\"colormap_config is missing required attributes: {', '.join(missing_attrs)}\"\n )\n\n # Ensure that ptype is appropriate when colormap_config is provided\n if ptype not in [\"intensity\", \"depth\"]:\n raise ValueError(\n \"colormap_config is only supported for ptype='intensity' or 'depth'\"\n )\n\n cmap = colormap_config.cmap\n clevs = colormap_config.clevs\n\n # Validate that the number of colors matches len(clevs)\n if isinstance(cmap, colors.ListedColormap):\n num_colors = len(cmap.colors)\n else:\n num_colors = cmap.N\n\n expected_colors = len(clevs)\n if num_colors != expected_colors:\n raise ValueError(\n f\"Number of colors in colormap (N={num_colors}) does not match len(clevs) (N={expected_colors}).\"\n )\n\n return colormap_config.cmap, colormap_config.norm, colormap_config.clevs\n" }, { "path": "pysteps/visualization/spectral.py", "content": "# -*- coding: utf-8 -*-\r\n\"\"\"\r\npysteps.visualization.spectral\r\n==============================\r\n\r\nMethods for plotting Fourier spectra.\r\n\r\n.. autosummary::\r\n :toctree: ../generated/\r\n\r\n plot_spectrum1d\r\n\"\"\"\r\n\r\nimport matplotlib.pylab as plt\r\nimport numpy as np\r\n\r\n\r\ndef plot_spectrum1d(\r\n fft_freq,\r\n fft_power,\r\n x_units=None,\r\n y_units=None,\r\n wavelength_ticks=None,\r\n color=\"k\",\r\n lw=1.0,\r\n label=None,\r\n ax=None,\r\n **kwargs,\r\n):\r\n \"\"\"\r\n Function to plot in log-log a radially averaged Fourier spectrum.\r\n\r\n Parameters\r\n ----------\r\n fft_freq: array-like\r\n 1d array containing the Fourier frequencies computed with the function\r\n :py:func:`pysteps.utils.spectral.rapsd`.\r\n fft_power: array-like\r\n 1d array containing the radially averaged Fourier power spectrum\r\n computed with the function :py:func:`pysteps.utils.spectral.rapsd`.\r\n x_units: str, optional\r\n Units of the X variable (distance, e.g. \"km\").\r\n y_units: str, optional\r\n Units of the Y variable (amplitude, e.g. \"dBR\").\r\n wavelength_ticks: array-like, optional\r\n List of wavelengths where to show xticklabels.\r\n color: str, optional\r\n Line color.\r\n lw: float, optional\r\n Line width.\r\n label: str, optional\r\n Label (for legend).\r\n ax: Axes, optional\r\n Plot axes.\r\n\r\n Returns\r\n -------\r\n ax: Axes\r\n Plot axes\r\n \"\"\"\r\n # Check input dimensions\r\n n_freq = len(fft_freq)\r\n n_pow = len(fft_power)\r\n if n_freq != n_pow:\r\n raise ValueError(\r\n f\"Dimensions of the 1d input arrays must be equal. {n_freq} vs {n_pow}\"\r\n )\r\n\r\n if ax is None:\r\n ax = plt.subplot(111)\r\n\r\n # Plot spectrum in log-log scale\r\n ax.plot(\r\n 10 * np.log10(fft_freq),\r\n 10 * np.log10(fft_power),\r\n color=color,\r\n linewidth=lw,\r\n label=label,\r\n **kwargs,\r\n )\r\n\r\n # X-axis\r\n if wavelength_ticks is not None:\r\n wavelength_ticks = np.array(wavelength_ticks)\r\n freq_ticks = 1 / wavelength_ticks\r\n ax.set_xticks(10 * np.log10(freq_ticks))\r\n ax.set_xticklabels(wavelength_ticks)\r\n if x_units is not None:\r\n ax.set_xlabel(f\"Wavelength [{x_units}]\")\r\n else:\r\n if x_units is not None:\r\n ax.set_xlabel(f\"Frequency [1/{x_units}]\")\r\n\r\n # Y-axis\r\n if y_units is not None:\r\n # { -> {{ with f-strings\r\n power_units = rf\"$10log_{{ 10 }}(\\frac{{ {y_units}^2 }}{{ {x_units} }})$\"\r\n ax.set_ylabel(f\"Power {power_units}\")\r\n\r\n return ax\r\n" }, { "path": "pysteps/visualization/thunderstorms.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\npysteps.visualization.tstorm\n============================\n\nMethods for plotting thunderstorm cells.\n\nCreated on Wed Nov 4 11:09:44 2020\n\n@author: mfeldman\n\n.. autosummary::\n :toctree: ../generated/\n\n plot_track\n plot_cart_contour\n\"\"\"\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n################################\n# track and contour plots zorder\n# - precipitation: 40\n\n\ndef plot_track(track_list, geodata=None, ref_shape=None):\n \"\"\"\n Plot storm tracks.\n\n .. _Axes: https://matplotlib.org/api/axes_api.html#matplotlib.axes.Axes\n\n Parameters\n ----------\n track_list: list\n List of tracks provided by DATing.\n geodata: dictionary or None, optional\n Optional dictionary containing geographical information about\n the field. If not None, plots the contours in a georeferenced frame.\n ref_shape: (vertical, horizontal)\n Shape of the 2D precipitation field used to find the cells' contours.\n This is only needed only if `geodata=None`.\n\n IMPORTANT: If `geodata=None` it is assumed that the y-origin of the reference\n precipitation fields is the upper-left corner (yorigin=\"upper\").\n\n Returns\n -------\n ax: fig Axes_\n Figure axes.\n \"\"\"\n ax = plt.gca()\n pix2coord = _pix2coord_factory(geodata, ref_shape)\n\n color = iter(plt.cm.spring(np.linspace(0, 1, len(track_list))))\n for track in track_list:\n cen_x, cen_y = pix2coord(track.cen_x, track.cen_y)\n ax.plot(cen_x, cen_y, c=next(color), zorder=40)\n return ax\n\n\ndef plot_cart_contour(contours, geodata=None, ref_shape=None):\n \"\"\"\n Plots input image with identified cell contours.\n Also, this function can be user to add points of interest to a plot.\n\n .. _Axes: https://matplotlib.org/api/axes_api.html#matplotlib.axes.Axes\n\n Parameters\n ----------\n contours: list or dataframe-element\n list of identified cell contours.\n geodata: dictionary or None, optional\n Optional dictionary containing geographical information about\n the field. If not None, plots the contours in a georeferenced frame.\n ref_shape: (vertical, horizontal)\n Shape of the 2D precipitation field used to find the cells' contours.\n This is only needed only if `geodata=None`.\n\n IMPORTANT: If `geodata=None` it is assumed that the y-origin of the reference\n precipitation fields is the upper-left corner (yorigin=\"upper\").\n\n Returns\n -------\n ax: fig Axes_\n Figure axes.\n \"\"\"\n ax = plt.gca()\n pix2coord = _pix2coord_factory(geodata, ref_shape)\n\n contours = list(contours)\n for contour in contours:\n for c in contour:\n x, y = pix2coord(c[:, 1], c[:, 0])\n ax.plot(x, y, color=\"black\", zorder=40)\n return ax\n\n\ndef _pix2coord_factory(geodata, ref_shape):\n \"\"\"\n Construct the pix2coord transformation function.\"\"\"\n if geodata is not None:\n\n def pix2coord(x_input, y_input):\n x = geodata[\"x1\"] + geodata[\"xpixelsize\"] * x_input\n if geodata[\"yorigin\"] == \"lower\":\n y = geodata[\"y1\"] + geodata[\"ypixelsize\"] * y_input\n else:\n y = geodata[\"y2\"] - geodata[\"ypixelsize\"] * y_input\n return x, y\n\n else:\n if ref_shape is None:\n raise ValueError(\"'ref_shape' can't be None when not geodata is available.\")\n\n # Default pix2coord function when no geographical information is present.\n def pix2coord(x_input, y_input):\n # yorigin is \"upper\" by default\n return x_input, ref_shape[0] - y_input\n\n return pix2coord\n" }, { "path": "pysteps/visualization/utils.py", "content": "\"\"\"\npysteps.visualization.utils\n===========================\n\nMiscellaneous utility functions for the visualization module.\n\n.. autosummary::\n :toctree: ../generated/\n\n parse_proj4_string\n proj4_to_cartopy\n reproject_geodata\n get_geogrid\n get_basemap_axis\n\"\"\"\n\nimport warnings\n\nimport matplotlib.pylab as plt\nimport numpy as np\n\nfrom pysteps.exceptions import MissingOptionalDependency\nfrom pysteps.visualization import basemaps\n\ntry:\n import cartopy.crs as ccrs\n from cartopy.mpl.geoaxes import GeoAxesSubplot\n\n PYPROJ_PROJECTION_TO_CARTOPY = dict(\n tmerc=ccrs.TransverseMercator,\n laea=ccrs.LambertAzimuthalEqualArea,\n lcc=ccrs.LambertConformal,\n merc=ccrs.Mercator,\n utm=ccrs.UTM,\n stere=ccrs.Stereographic,\n aea=ccrs.AlbersEqualArea,\n aeqd=ccrs.AzimuthalEquidistant,\n # Note: ccrs.epsg(2056) doesn't work because the projection\n # limits are too strict.\n # We'll use the Stereographic projection as an alternative.\n somerc=ccrs.Stereographic,\n geos=ccrs.Geostationary,\n )\n\n CARTOPY_IMPORTED = True\nexcept ImportError:\n CARTOPY_IMPORTED = False\n PYPROJ_PROJECTION_TO_CARTOPY = dict()\n GeoAxesSubplot = None\n ccrs = None\n\ntry:\n import pyproj\n\n PYPROJ_IMPORTED = True\nexcept ImportError:\n PYPROJ_IMPORTED = False\n\nPYPROJ_PROJ_KWRDS_TO_CARTOPY = {\n \"lon_0\": \"central_longitude\",\n \"lat_0\": \"central_latitude\",\n \"lat_ts\": \"true_scale_latitude\",\n \"x_0\": \"false_easting\",\n \"y_0\": \"false_northing\",\n \"k\": \"scale_factor\",\n \"zone\": \"zone\",\n}\n\nPYPROJ_GLOB_KWRDS_TO_CARTOPY = {\n \"a\": \"semimajor_axis\",\n \"b\": \"semiminor_axis\",\n \"datum\": \"datum\",\n \"ellps\": \"ellipse\",\n \"f\": \"flattening\",\n \"rf\": \"inverse_flattening\",\n}\n\n\ndef parse_proj4_string(proj4str):\n \"\"\"\n Construct a dictionary from a PROJ.4 projection string.\n\n Parameters\n ----------\n proj4str: str\n A PROJ.4-compatible projection string.\n\n Returns\n -------\n out: dict\n Dictionary, where keys and values are parsed from the projection\n parameter tokens beginning with '+'.\n \"\"\"\n\n if not PYPROJ_IMPORTED:\n raise MissingOptionalDependency(\n \"pyproj package is required for parse_proj4_string function utility \"\n \"but it is not installed\"\n )\n\n with warnings.catch_warnings():\n warnings.simplefilter(\"ignore\", category=UserWarning)\n # Ignore the warning raised by to_dict() about losing information.\n proj_dict = pyproj.Proj(proj4str).crs.to_dict()\n\n return proj_dict\n\n\ndef proj4_to_cartopy(proj4str):\n \"\"\"\n Convert a PROJ.4 projection string into a Cartopy coordinate reference\n system (crs) object.\n\n Parameters\n ----------\n proj4str: str\n A PROJ.4-compatible projection string.\n\n Returns\n -------\n out: object\n Instance of a crs class defined in cartopy.crs.\n \"\"\"\n if not CARTOPY_IMPORTED:\n raise MissingOptionalDependency(\n \"cartopy package is required for proj4_to_cartopy function \"\n \"utility but it is not installed\"\n )\n\n if not PYPROJ_IMPORTED:\n raise MissingOptionalDependency(\n \"pyproj package is required for proj4_to_cartopy function utility \"\n \"but it is not installed\"\n )\n\n proj = pyproj.Proj(proj4str)\n\n try:\n # pyproj >= 2.2.0\n is_geographic = proj.crs.is_geographic\n except AttributeError:\n # pyproj < 2.2.0\n is_geographic = proj.is_latlong()\n\n if is_geographic:\n return ccrs.PlateCarree()\n\n proj_dict = parse_proj4_string(proj4str)\n\n cartopy_crs_kwargs = dict()\n globe_kwargs = dict()\n cartopy_crs = None\n globe = None\n\n for key, value in proj_dict.items():\n if key == \"proj\":\n if value in PYPROJ_PROJECTION_TO_CARTOPY:\n cartopy_crs = PYPROJ_PROJECTION_TO_CARTOPY[value]\n else:\n raise ValueError(f\"Unsupported projection: {value}\")\n\n if key in PYPROJ_PROJ_KWRDS_TO_CARTOPY:\n cartopy_crs_kwargs[PYPROJ_PROJ_KWRDS_TO_CARTOPY[key]] = value\n\n if key in PYPROJ_GLOB_KWRDS_TO_CARTOPY:\n globe_kwargs[PYPROJ_GLOB_KWRDS_TO_CARTOPY[key]] = value\n\n # issubset: <=\n if {\"lat_1\", \"lat_2\"} <= proj_dict.keys():\n cartopy_crs_kwargs[\"standard_parallels\"] = (\n proj_dict[\"lat_1\"],\n proj_dict[\"lat_2\"],\n )\n if \"R\" in proj_dict.keys():\n globe_kwargs[\"semimajor_axis\"] = proj_dict[\"R\"]\n globe_kwargs[\"semiminor_axis\"] = proj_dict[\"R\"]\n\n if globe_kwargs:\n globe = ccrs.Globe(**globe_kwargs)\n\n if isinstance(cartopy_crs, ccrs.Mercator):\n cartopy_crs_kwargs.pop(\"false_easting\", None)\n cartopy_crs_kwargs.pop(\"false_northing\", None)\n\n return cartopy_crs(globe=globe, **cartopy_crs_kwargs)\n\n\ndef reproject_geodata(geodata, t_proj4str, return_grid=None):\n \"\"\"\n Reproject geodata and optionally create a grid in a new projection.\n\n Parameters\n ----------\n geodata: dictionary\n Dictionary containing geographical information about the field.\n It must contain the attributes projection, x1, x2, y1, y2, xpixelsize,\n ypixelsize, as defined in the documentation of pysteps.io.importers.\n t_proj4str: str\n The target PROJ.4-compatible projection string (fallback).\n return_grid: {None, 'coords', 'quadmesh'}, optional\n Whether to return the coordinates of the projected grid.\n The default return_grid=None does not compute the grid,\n return_grid='coords' returns the centers of projected grid points,\n return_grid='quadmesh' returns the coordinates of the quadrilaterals\n (e.g. to be used by pcolormesh).\n\n Returns\n -------\n geodata: dictionary\n Dictionary containing the reprojected geographical information\n and optionally the required X_grid and Y_grid.\n\n It also includes a fixed boolean attribute regular_grid=False to indicate\n that the reprojected grid has no regular spacing.\n \"\"\"\n if not PYPROJ_IMPORTED:\n raise MissingOptionalDependency(\n \"pyproj package is required for reproject_geodata function utility\"\n \" but it is not installed\"\n )\n\n geodata = geodata.copy()\n s_proj4str = geodata[\"projection\"]\n extent = (geodata[\"x1\"], geodata[\"x2\"], geodata[\"y1\"], geodata[\"y2\"])\n shape = (\n int((geodata[\"y2\"] - geodata[\"y1\"]) / geodata[\"ypixelsize\"]),\n int((geodata[\"x2\"] - geodata[\"x1\"]) / geodata[\"xpixelsize\"]),\n )\n\n s_srs = pyproj.Proj(s_proj4str)\n t_srs = pyproj.Proj(t_proj4str)\n\n x1 = extent[0]\n x2 = extent[1]\n y1 = extent[2]\n y2 = extent[3]\n\n # Reproject grid on fall-back projection\n if return_grid is not None:\n if return_grid == \"coords\":\n y_coord = (\n np.linspace(y1, y2, shape[0], endpoint=False)\n + geodata[\"ypixelsize\"] / 2.0\n )\n x_coord = (\n np.linspace(x1, x2, shape[1], endpoint=False)\n + geodata[\"xpixelsize\"] / 2.0\n )\n elif return_grid == \"quadmesh\":\n y_coord = np.linspace(y1, y2, shape[0] + 1)\n x_coord = np.linspace(x1, x2, shape[1] + 1)\n else:\n raise ValueError(\"unknown return_grid value %s\" % return_grid)\n\n x_grid, y_grid = np.meshgrid(x_coord, y_coord)\n\n x_grid, y_grid = pyproj.transform(\n s_srs, t_srs, x_grid.flatten(), y_grid.flatten()\n )\n x_grid = x_grid.reshape((y_coord.size, x_coord.size))\n y_grid = y_grid.reshape((y_coord.size, x_coord.size))\n geodata[\"X_grid\"] = x_grid\n geodata[\"Y_grid\"] = y_grid\n\n # Reproject extent on fall-back projection\n x1, y1 = pyproj.transform(s_srs, t_srs, x1, y1)\n x2, y2 = pyproj.transform(s_srs, t_srs, x2, y2)\n\n # update geodata\n geodata[\"projection\"] = t_proj4str\n geodata[\"x1\"] = x1\n geodata[\"x2\"] = x2\n geodata[\"y1\"] = y1\n geodata[\"y2\"] = y2\n geodata[\"regular_grid\"] = False\n geodata[\"xpixelsize\"] = None\n geodata[\"ypixelsize\"] = None\n\n return geodata\n\n\ndef get_geogrid(nlat, nlon, geodata=None):\n \"\"\"\n Get the geogrid data.\n If geodata is None, a regular grid is returned. In this case, it is assumed that\n the origin of the 2D input data is the upper left corner (\"upper\").\n\n Parameters\n ----------\n nlat: int\n Number of grid points along the latitude axis\n nlon: int\n Number of grid points along the longitude axis\n geodata:\n geodata: dictionary or None\n Optional dictionary containing geographical information about\n the field.\n\n If geodata is not None, it must contain the following key-value pairs:\n\n .. tabularcolumns:: |p{1.5cm}|L|\n\n +----------------+----------------------------------------------------+\n | Key | Value |\n +================+====================================================+\n | projection | PROJ.4-compatible projection definition |\n +----------------+----------------------------------------------------+\n | x1 | x-coordinate of the lower-left corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | y1 | y-coordinate of the lower-left corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | x2 | x-coordinate of the upper-right corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | y2 | y-coordinate of the upper-right corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | yorigin | a string specifying the location of the first |\n | | element in the data raster w.r.t. y-axis: |\n | | 'upper' = upper border, 'lower' = lower border |\n +----------------+----------------------------------------------------+\n\n Returns\n -------\n x_grid: 2D array\n X grid with dimensions of (nlat, nlon) with the same `y-origin` as the one\n specified in the geodata (or \"upper\" if geodata is None).\n y_grid: 2D array\n Y grid with dimensions of (nlat, nlon) with the same `y-origin` as the one\n specified in the geodata (or \"upper\" if geodata is None).\n extent: tuple\n Four-element tuple specifying the extent of the domain according to\n (lower left x, upper right x, lower left y, upper right y).\n regular_grid: bool\n True if the grid is regular. False otherwise.\n origin: str\n Place the [0, 0] index of the array to plot in the upper left or lower left\n corner of the axes.\n \"\"\"\n\n if geodata is not None:\n regular_grid = geodata.get(\"regular_grid\", True)\n\n x_lims = sorted((geodata[\"x1\"], geodata[\"x2\"]))\n x, xpixelsize = np.linspace(\n x_lims[0], x_lims[1], nlon, endpoint=False, retstep=True\n )\n x += xpixelsize / 2.0\n\n y_lims = sorted((geodata[\"y1\"], geodata[\"y2\"]))\n y, ypixelsize = np.linspace(\n y_lims[0], y_lims[1], nlat, endpoint=False, retstep=True\n )\n y += ypixelsize / 2.0\n\n extent = (geodata[\"x1\"], geodata[\"x2\"], geodata[\"y1\"], geodata[\"y2\"])\n\n x_grid, y_grid = np.meshgrid(x, y)\n\n if geodata[\"yorigin\"] == \"upper\":\n y_grid = np.flipud(y_grid)\n\n return x_grid, y_grid, extent, regular_grid, geodata[\"yorigin\"]\n\n # Default behavior: return a simple regular grid\n # Assume yorigin = upper\n x_grid, y_grid = np.meshgrid(np.arange(nlon), np.arange(nlat))\n y_grid = np.flipud(y_grid)\n extent = (0, nlon - 1, 0, nlat - 1)\n regular_grid = True\n return x_grid, y_grid, extent, regular_grid, \"upper\"\n\n\ndef get_basemap_axis(extent, geodata=None, ax=None, map_kwargs=None):\n \"\"\"\n Safely get a basemap axis. If ax is None, the current axis is returned.\n\n If geodata is not None and ax is not a cartopy axis already, it creates a basemap\n axis and return it.\n\n Parameters\n ----------\n extent: tuple\n Four-element tuple specifying the extent of the domain according to\n (lower left x, upper right x, lower left y, upper right y).\n geodata:\n geodata: dictionary or None\n Optional dictionary containing geographical information about\n the field.\n\n If geodata is not None, it must contain the following key-value pairs:\n\n .. tabularcolumns:: |p{1.5cm}|L|\n\n +----------------+----------------------------------------------------+\n | Key | Value |\n +================+====================================================+\n | projection | PROJ.4-compatible projection definition |\n +----------------+----------------------------------------------------+\n | x1 | x-coordinate of the lower-left corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | y1 | y-coordinate of the lower-left corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | x2 | x-coordinate of the upper-right corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | y2 | y-coordinate of the upper-right corner of the data |\n | | raster |\n +----------------+----------------------------------------------------+\n | yorigin | a string specifying the location of the first |\n | | element in the data raster w.r.t. y-axis: |\n | | 'upper' = upper border, 'lower' = lower border |\n +----------------+----------------------------------------------------+\n\n ax: axis object\n Optional axis object to use for plotting.\n map_kwargs: dict\n Optional parameters that need to be passed to\n :py:func:`pysteps.visualization.basemaps.plot_geography`.\n\n Returns\n -------\n ax: axis object\n \"\"\"\n if map_kwargs is None:\n map_kwargs = dict()\n\n if ax is None:\n # If no axes is passed, use the current axis.\n ax = plt.gca()\n\n # Create the cartopy axis if the axis is not a cartopy axis.\n if geodata is not None:\n if not CARTOPY_IMPORTED:\n warnings.warn(\n \"cartopy package is required for the get_geogrid function \"\n \"but it is not installed. Ignoring geographical information.\"\n )\n return ax\n\n if not PYPROJ_IMPORTED:\n warnings.warn(\n \"pyproj package is required for the get_geogrid function \"\n \"but it is not installed. Ignoring geographical information.\"\n )\n return ax\n\n if not isinstance(ax, GeoAxesSubplot):\n # Check `ax` is not a GeoAxesSubplot axis to avoid overwriting the map.\n ax = basemaps.plot_geography(geodata[\"projection\"], extent, **map_kwargs)\n\n return ax\n" }, { "path": "requirements.txt", "content": "numpy\nopencv-python\npillow\npyproj\nscipy\nmatplotlib\njsmin\njsonschema\nnetCDF4\n" }, { "path": "requirements_dev.txt", "content": "# Base dependencies\npython>=3.10\nnumpy\nopencv-python\npillow\npyproj\nscipy\nmatplotlib\njsmin\njsonschema\nnetCDF4\n\n# Optional dependencies\ndask\npyfftw\ncartopy>=0.18\nh5py\nscikit-image\nscikit-learn\npandas\nrasterio\n\n# Testing\npytest\n\n" }, { "path": "setup.py", "content": "# -*- coding: utf-8 -*-\n\nimport sys\n\nfrom setuptools import find_packages, setup\nfrom setuptools.extension import Extension\n\ntry:\n from Cython.Build import cythonize\nexcept ImportError:\n raise RuntimeError(\n \"Cython required for running the package installation\\n\"\n + \"Try installing it with:\\n\"\n + \"$> pip install cython\"\n )\n\ntry:\n import numpy\nexcept ImportError:\n raise RuntimeError(\n \"Numpy required for running the package installation\\n\"\n + \"Try installing it with:\\n\"\n + \"$> pip install numpy\"\n )\n\n# Define common arguments used to compile the extensions\ncommon_link_args = [\"-fopenmp\"]\ncommon_compile_args = [\"-fopenmp\", \"-O3\", \"-ffast-math\"]\ncommon_include = [numpy.get_include()]\n\nif sys.platform.startswith(\"darwin\"):\n common_link_args.append(\"-Wl,-rpath,/usr/local/opt/gcc/lib/gcc/9/\")\n\nextensions_data = {\n \"pysteps.motion._proesmans\": {\"sources\": [\"pysteps/motion/_proesmans.pyx\"]},\n \"pysteps.motion._vet\": {\"sources\": [\"pysteps/motion/_vet.pyx\"]},\n}\n\nextensions = []\n\nfor name, data in extensions_data.items():\n include = data.get(\"include\", common_include)\n\n extra_compile_args = data.get(\"extra_compile_args\", common_compile_args)\n\n extra_link_args = data.get(\"extra_link_args\", common_link_args)\n\n pysteps_extension = Extension(\n name,\n sources=data[\"sources\"],\n depends=data.get(\"depends\", []),\n include_dirs=include,\n language=data.get(\"language\", \"c\"),\n define_macros=data.get(\"macros\", []),\n extra_compile_args=extra_compile_args,\n extra_link_args=extra_link_args,\n )\n\n extensions.append(pysteps_extension)\n\nexternal_modules = cythonize(extensions, force=True, language_level=3)\n\nrequirements = [\n \"numpy\",\n \"jsmin\",\n \"scipy\",\n \"matplotlib\",\n \"jsonschema\",\n]\n\nsetup(\n name=\"pysteps\",\n version=\"1.20.0\",\n author=\"PySteps developers\",\n packages=find_packages(),\n license=\"LICENSE\",\n include_package_data=True,\n description=\"Python framework for short-term ensemble prediction systems\",\n long_description=open(\"README.rst\").read(),\n long_description_content_type=\"text/x-rst\",\n url=\"https://pysteps.github.io/\",\n project_urls={\n \"Source\": \"https://github.com/pySTEPS/pysteps\",\n \"Issues\": \"https://github.com/pySTEPS/pysteps/issues\",\n \"CI\": \"https://github.com/pySTEPS/pysteps/actions\",\n \"Changelog\": \"https://github.com/pySTEPS/pysteps/releases\",\n \"Documentation\": \"https://pysteps.readthedocs.io\",\n },\n classifiers=[\n \"Development Status :: 5 - Production/Stable\",\n \"Intended Audience :: Science/Research\",\n \"Topic :: Scientific/Engineering\",\n \"Topic :: Scientific/Engineering :: Atmospheric Science\",\n \"Topic :: Scientific/Engineering :: Hydrology\",\n \"License :: OSI Approved :: BSD License\",\n \"Programming Language :: Python :: 3 :: Only\",\n \"Programming Language :: Python :: 3.11\",\n \"Programming Language :: Python :: 3.12\",\n \"Programming Language :: Python :: 3.13\",\n \"Operating System :: OS Independent\",\n ],\n ext_modules=external_modules,\n setup_requires=requirements,\n install_requires=requirements,\n)\n" }, { "path": "tox.ini", "content": "# Tox configuration file for pysteps projects\n# Need conda, tox and tox-conda installed to run\n#\n# In conda run:\n# > conda install -c conda-forge tox tox-conda\n#\n# Alternatively, you can install them using pip:\n# > pip install tox tox-conda\n#\n# Then, to run the tests, from the repo’s root run:\n#\n# > tox # Run pytests\n# > tox -e install # Test package installation\n# > tox -e black # Test for black formatting warnings\n\n[tox]\nenvlist = py37, py38, py39\n\n[testenv]\ndescription = Run the pysteps's test suite\ndeps =\n -r{toxinidir}/requirements.txt\n cython\n dask\n toolz\n pillow\n pyfftw\n h5py\n PyWavelets\n\tscikit-learn\n gitpython\n pytest\n pytest-cov\n codecov\nconda_deps =\n netCDF4\n pyproj\n cartopy\n pygrib\n\trasterio\nconda_channels = conda-forge\n\nsetenv =\n PYSTEPS_DATA_PATH = {toxworkdir}/pysteps-data\n PYSTEPSRC = {toxworkdir}/pysteps-data/pystepsrc\n PACKAGE_ROOT = {toxinidir}\n PROJ_LIB={envdir}/share/proj\ncommands =\n python {toxinidir}/ci/fetch_pysteps_data.py\n pytest --pyargs pysteps --cov=pysteps -ra --disable-warnings\n\n[test_no_cov]\ncommands =\n python {toxinidir}/ci/fetch_pysteps_data.py\n pytest --pyargs pysteps --disable-warnings\n\n[testenv:install]\ndescription = Test the installation of the package in a clean environment and run minimal tests\ndeps = pytest\nconda_deps =\nchangedir = {homedir}\ncommands =\n pip install -U {toxinidir}/\n python -c \"import pysteps\"\n\n # Test the pysteps plugin support\n pip install cookiecutter\n cookiecutter -f --no-input https://github.com/pySTEPS/cookiecutter-pysteps-plugin -o {temp_dir}/\n # NB: this should match the default name for a cookiecutter-generated plugin!\n pip install {temp_dir}/pysteps-importer-institution-name\n python {toxinidir}/ci/test_plugin_support.py\n # Check the compiled modules\n python -c \"from pysteps import motion\"\n python -c \"from pysteps.motion import vet\"\n python -c \"from pysteps.motion import proesmans\"\n\n\n[testenv:install_full]\ndescription = Test the installation of the package in an environment with all the dependencies\nchangedir = {homedir}\ncommands =\n {[testenv:install]commands}\n {[test_no_cov]commands}\n\n[testenv:pypi]\ndescription = Test the installation of the package from the PyPI in a clean environment\ndeps = pytest\nconda_deps =\nchangedir = {homedir}\ncommands =\n pip install --no-cache-dir pysteps\n python -c \"import pysteps\"\n {[test_no_cov]commands}\n\n[testenv:pypi_test]\ndescription = Test the installation of the package from the test-PyPI in a clean environment\ndeps = pytest\nconda_deps =\nchangedir = {homedir}\ncommands =\n pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url=https://pypi.org/simple/ pysteps\n python -c \"import pysteps\"\n {[test_no_cov]commands}\n\n[testenv:pypi_test_full]\ndescription = Test the installation of the package from the test-PyPI in an environment with all the dependencies\nchangedir = {homedir}\ncommands = {[testenv:pypi_test]commands}\n\n\n[testenv:docs]\ndescription = Build the html documentation using sphinx\nusedevelop = True\ndeps =\n -r{toxinidir}/requirements.txt\n -r{toxinidir}/doc/requirements.txt\n cython\nconda_channels =\n conda-forge\n default\nchangedir = doc\nsetenv =\n PYSTEPS_DATA_PATH = {toxworkdir}/pysteps-data\n PYSTEPSRC = {toxworkdir}/pysteps-data/pystepsrc\ncommands =\n python {toxinidir}/ci/fetch_pysteps_data.py\n sphinx-build -b html source _build\n\n[testenv:black]\ndeps = black\ncommands = black --check pysteps\n" } ]