Repository: ContinuumIO/intake Branch: master Commit: a04cec104a9b Files: 237 Total size: 1.1 MB Directory structure: gitextract_g6quolyo/ ├── .ci-coveragerc ├── .coveragerc ├── .gitattributes ├── .github/ │ └── workflows/ │ ├── main.yaml │ ├── pre-commit.yml │ └── pypipublish.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── MANIFEST.in ├── README.md ├── README_refactor.md ├── docs/ │ ├── Makefile │ ├── README.md │ ├── environment.yml │ ├── make.bat │ ├── make_api.py │ ├── plugins.py │ ├── plugins.yaml │ ├── requirements.txt │ └── source/ │ ├── _static/ │ │ ├── .keep │ │ ├── css/ │ │ │ └── custom.css │ │ └── images/ │ │ └── plotting_example.html │ ├── api.rst │ ├── api2.rst │ ├── api_base.rst │ ├── api_other.rst │ ├── api_user.rst │ ├── catalog.rst │ ├── changelog.rst │ ├── code-of-conduct.rst │ ├── community.rst │ ├── conf.py │ ├── contributing.rst │ ├── data-packages.rst │ ├── deployments.rst │ ├── examples.rst │ ├── glossary.rst │ ├── gui.rst │ ├── guide.rst │ ├── index.rst │ ├── index_v1.rst │ ├── making-plugins.rst │ ├── overview.rst │ ├── persisting.rst │ ├── plotting.rst │ ├── plugin-directory.rst │ ├── quickstart.rst │ ├── reference.rst │ ├── roadmap.rst │ ├── scope2.rst │ ├── start.rst │ ├── tools.rst │ ├── tour2.rst │ ├── transforms.rst │ ├── use_cases.rst │ ├── user2.rst │ └── walkthrough2.rst ├── examples/ │ └── Take2.ipynb ├── intake/ │ ├── __init__.py │ ├── catalog/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── default.py │ │ ├── entry.py │ │ ├── exceptions.py │ │ ├── gui.py │ │ ├── local.py │ │ ├── tests/ │ │ │ ├── __init__.py │ │ │ ├── cache_data/ │ │ │ │ └── states.csv │ │ │ ├── catalog.yml │ │ │ ├── catalog1.yml │ │ │ ├── catalog_alias.yml │ │ │ ├── catalog_caching.yml │ │ │ ├── catalog_dup_parameters.yml │ │ │ ├── catalog_dup_sources.yml │ │ │ ├── catalog_hierarchy.yml │ │ │ ├── catalog_named.yml │ │ │ ├── catalog_non_dict.yml │ │ │ ├── catalog_search/ │ │ │ │ ├── example_packages/ │ │ │ │ │ ├── ep/ │ │ │ │ │ │ └── __init__.py │ │ │ │ │ └── ep-0.1.dist-info/ │ │ │ │ │ └── entry_points.txt │ │ │ │ └── yaml.yml │ │ │ ├── catalog_union_1.yml │ │ │ ├── catalog_union_2.yml │ │ │ ├── conftest.py │ │ │ ├── data_source_missing.yml │ │ │ ├── data_source_name_non_string.yml │ │ │ ├── data_source_non_dict.yml │ │ │ ├── data_source_value_non_dict.yml │ │ │ ├── dot-nest.yaml │ │ │ ├── entry1_1.csv │ │ │ ├── entry1_2.csv │ │ │ ├── example1_source.py │ │ │ ├── example_plugin_dir/ │ │ │ │ └── example2_source.py │ │ │ ├── multi_plugins.yaml │ │ │ ├── multi_plugins2.yaml │ │ │ ├── obsolete_data_source_list.yml │ │ │ ├── obsolete_params_list.yml │ │ │ ├── params_missing_required.yml │ │ │ ├── params_name_non_string.yml │ │ │ ├── params_non_dict.yml │ │ │ ├── params_value_bad_choice.yml │ │ │ ├── params_value_bad_type.yml │ │ │ ├── params_value_non_dict.yml │ │ │ ├── plugins_non_dict.yml │ │ │ ├── plugins_source_missing.yml │ │ │ ├── plugins_source_missing_key.yml │ │ │ ├── plugins_source_non_dict.yml │ │ │ ├── plugins_source_non_list.yml │ │ │ ├── plugins_source_non_string.yml │ │ │ ├── small.npy │ │ │ ├── test_alias.py │ │ │ ├── test_catalog_save.py │ │ │ ├── test_core.py │ │ │ ├── test_default.py │ │ │ ├── test_discovery.py │ │ │ ├── test_gui.py │ │ │ ├── test_local.py │ │ │ ├── test_parameters.py │ │ │ ├── test_reload_integration.py │ │ │ ├── test_utils.py │ │ │ ├── test_zarr.py │ │ │ └── util.py │ │ ├── utils.py │ │ └── zarr.py │ ├── config.py │ ├── conftest.py │ ├── container/ │ │ ├── __init__.py │ │ └── base.py │ ├── interface/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── catalog/ │ │ │ ├── __init__.py │ │ │ ├── add.py │ │ │ └── search.py │ │ ├── gui.py │ │ └── source/ │ │ ├── __init__.py │ │ └── defined_plots.py │ ├── readers/ │ │ ├── __init__.py │ │ ├── catalogs.py │ │ ├── convert.py │ │ ├── datatypes.py │ │ ├── entry.py │ │ ├── examples.py │ │ ├── importlist.py │ │ ├── metadata.py │ │ ├── mixins.py │ │ ├── namespaces.py │ │ ├── output.py │ │ ├── readers.py │ │ ├── search.py │ │ ├── tests/ │ │ │ ├── __init__.py │ │ │ ├── cats/ │ │ │ │ ├── __init__.py │ │ │ │ ├── stac_data/ │ │ │ │ │ ├── 1.0.0/ │ │ │ │ │ │ ├── catalog/ │ │ │ │ │ │ │ ├── catalog.json │ │ │ │ │ │ │ └── child-catalog.json │ │ │ │ │ │ ├── collection/ │ │ │ │ │ │ │ ├── collection.json │ │ │ │ │ │ │ ├── simple-item.json │ │ │ │ │ │ │ └── zarr-collection.json │ │ │ │ │ │ ├── item/ │ │ │ │ │ │ │ └── zarr-item.json │ │ │ │ │ │ └── itemcollection/ │ │ │ │ │ │ └── example-search.json │ │ │ │ │ └── 1.0.0beta2/ │ │ │ │ │ └── earthsearch/ │ │ │ │ │ ├── readme.md │ │ │ │ │ └── single-file-stac.json │ │ │ │ ├── test_sql.py │ │ │ │ ├── test_stac.py │ │ │ │ ├── test_thredds.py │ │ │ │ └── test_tiled.py │ │ │ ├── test_basic.py │ │ │ ├── test_consistency.py │ │ │ ├── test_dict.py │ │ │ ├── test_errors.py │ │ │ ├── test_reader.py │ │ │ ├── test_search.py │ │ │ ├── test_up.py │ │ │ ├── test_utils.py │ │ │ └── test_workflows.py │ │ ├── transform.py │ │ ├── user_parameters.py │ │ └── utils.py │ ├── source/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── csv.py │ │ ├── derived.py │ │ ├── discovery.py │ │ ├── jsonfiles.py │ │ ├── npy.py │ │ ├── tests/ │ │ │ ├── __init__.py │ │ │ ├── alias.yaml │ │ │ ├── cached.yaml │ │ │ ├── data.zarr/ │ │ │ │ ├── .zarray │ │ │ │ └── 0 │ │ │ ├── der.yaml │ │ │ ├── footer_csvs/ │ │ │ │ ├── sample_fewfooters.csv │ │ │ │ ├── sample_manyfooters.csv │ │ │ │ └── sample_nofooters.csv │ │ │ ├── pipeline.yaml │ │ │ ├── plugin_searchpath/ │ │ │ │ ├── collision_foo/ │ │ │ │ │ └── __init__.py │ │ │ │ ├── collision_foo2/ │ │ │ │ │ └── __init__.py │ │ │ │ ├── driver_with_entrypoints/ │ │ │ │ │ └── __init__.py │ │ │ │ ├── driver_with_entrypoints-0.1.dist-info/ │ │ │ │ │ └── entry_points.txt │ │ │ │ ├── intake_foo/ │ │ │ │ │ └── __init__.py │ │ │ │ └── not_intake_foo/ │ │ │ │ └── __init__.py │ │ │ ├── sample1.csv │ │ │ ├── sample2_1.csv │ │ │ ├── sample2_2.csv │ │ │ ├── sample3_2.csv │ │ │ ├── sources.yaml │ │ │ ├── test_base.py │ │ │ ├── test_csv.py │ │ │ ├── test_derived.py │ │ │ ├── test_discovery.py │ │ │ ├── test_json.py │ │ │ ├── test_npy.py │ │ │ ├── test_text.py │ │ │ ├── test_tiled.py │ │ │ ├── test_utils.py │ │ │ └── util.py │ │ ├── textfiles.py │ │ ├── tiled.py │ │ ├── utils.py │ │ └── zarr.py │ ├── tests/ │ │ ├── __init__.py │ │ ├── catalog1.yml │ │ ├── catalog2.yml │ │ ├── catalog_inherit_params.yml │ │ ├── catalog_nested.yml │ │ ├── catalog_nested_sub.yml │ │ ├── test_config.py │ │ ├── test_top_level.py │ │ └── test_utils.py │ ├── util_tests.py │ └── utils.py ├── pyproject.toml ├── readthedocs.yml └── scripts/ └── ci/ ├── environment-pip.yml ├── environment-py310.yml ├── environment-py311.yml ├── environment-py312.yml ├── environment-py313.yml └── environment-py314.yml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .ci-coveragerc ================================================ [run] omit = *tests/*, */_version.py ================================================ FILE: .coveragerc ================================================ [run] omit = */tests/* */test_*.py *_version.py source = intake [report] show_missing = True ================================================ FILE: .gitattributes ================================================ intake/_version.py export-subst ================================================ FILE: .github/workflows/main.yaml ================================================ name: CI on: push: branches: "*" pull_request: branches: master jobs: test: name: ${{ matrix.OS }}-${{ matrix.CONDA_ENV }}-pytest runs-on: ${{ matrix.OS }} strategy: fail-fast: false matrix: OS: [ubuntu-latest, windows-latest] CONDA_ENV: [py310, py311, py312, py313, py314, pip] steps: - name: Checkout uses: actions/checkout@v4 - name: Setup conda uses: conda-incubator/setup-miniconda@v3 with: environment-file: scripts/ci/environment-${{ matrix.CONDA_ENV }}.yml - name: pip-install shell: bash -l {0} run: | pip install . --no-deps - name: Run Tests shell: bash -l {0} run: | pytest -v intake/readers ================================================ FILE: .github/workflows/pre-commit.yml ================================================ name: pre-commit on: pull_request: branches: - '*' push: branches: [master] workflow_dispatch: jobs: pre-commit: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "3.11" - uses: pre-commit/action@v3.0.0 ================================================ FILE: .github/workflows/pypipublish.yaml ================================================ name: Upload Python Package on: release: types: [created] jobs: deploy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: python-version: "3.x" - name: Install dependencies run: | python -m pip install --upgrade pip pip install setuptools setuptools-scm wheel twine - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python setup.py sdist bdist_wheel twine upload dist/* ================================================ FILE: .gitignore ================================================ .DS_Store # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class _version.py # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache .pytest_cache nosetests.xml coverage.xml *.cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ docs/source/plugin-list.html # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # dotenv .env # virtualenv .venv venv/ ENV/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ # jetbrains/pycharm .idea/ ================================================ FILE: .pre-commit-config.yaml ================================================ # This is the configuration for pre-commit, a local framework for managing pre-commit hooks # Check out the docs at: https://pre-commit.com/ default_stages: [pre-commit] repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: check-builtin-literals - id: check-case-conflict - id: check-docstring-first - id: check-executables-have-shebangs - id: check-toml - id: detect-private-key - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/ambv/black rev: 23.1.0 hooks: - id: black - repo: https://github.com/charliermarsh/ruff-pre-commit rev: v0.0.249 hooks: - id: ruff # See 'setup.cfg' for args args: [intake] files: intake/ - repo: https://github.com/hoxbro/clean_notebook rev: 0.1.5 hooks: - id: clean-notebook ci: autofix_prs: false autoupdate_schedule: quarterly ================================================ FILE: LICENSE ================================================ Copyright (c) 2017, Anaconda, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: MANIFEST.in ================================================ prune .github prune docs prune examples prune scripts global-exclude test*.py *.yml *.yaml *.csv calvert* *.png *.json ================================================ FILE: README.md ================================================ # Intake: Take 2 **A general python package for describing, loading and processing data** ![Logo](https://github.com/intake/intake/raw/master/logo-small.png) [![Build Status](https://github.com/intake/intake/workflows/CI/badge.svg)](https://github.com/intake/intake/actions) [![Documentation Status](https://readthedocs.org/projects/intake/badge/?version=latest)](http://intake.readthedocs.io/en/latest/?badge=latest) *Taking the pain out of data access and distribution* Intake is an open-source package to: - describe your data declaratively - gather data sets into catalogs - search catalogs and services to find the right data you need - load, transform and output data in many formats - work with third party remote storage and compute platforms Documentation is available at [Read the Docs](http://intake.readthedocs.io/en/latest). Please report issues at https://github.com/intake/intake/issues Install ------- Recommended method using conda: ```bash conda install -c conda-forge intake ``` You can also install using `pip`, in which case you have a choice as to how many of the optional dependencies you install, with the simplest having least requirements ```bash pip install intake ``` Note that you may well need specific drivers and other plugins, which usually have additional dependencies of their own. Development ----------- * Create development Python environment with the required dependencies, ideally with `conda`. The requirements can be found in the yml files in the `scripts/ci/` directory of this repo. * e.g. `conda env create -f scripts/ci/environment-py311.yml` and then `conda activate test_env` * Install intake using `pip install -e .` * Use `pytest` to run tests. * Create a fork on github to be able to submit PRs. * We respect, but do not enforce, pep8 standards; all new code should be covered by tests. Support ------- Work on this repository is supported in part by: "Anaconda, Inc. - Advancing AI through open source." .. raw:: html anaconda logo ================================================ FILE: README_refactor.md ================================================ ## Intake Take2 Intake has been extensively rewritten to produce Intake Take2, https://github.com/intake/intake/pull/737 . This will now become the version of the ``main`` branch and be released as v2.0.0. The main documentation will move to describing V2, and V1 will not be further developed. Existing users of the legacy version ("v1") may find their code breaks and will need a version pin, although we aim to support most legacy workflows via backward compatibility. To install, you would do the following ```shell > pip install intake or > conda install intake ``` To get v1: ```shell > pip install "intake<2" or > conda install "intake<2" ``` This README is being kept to describe why the rewrite was done and considerations that went into it. ### Motivation for the rewrite. The main way to get the most out of Intake v1 has been by editing YAML files. This is how the documentation is structured. Yes, you could use intake.open_* to seed them, but then you will find a strong discontinuity between the documentation of the driver and the third party library that actually does the reading. This made is very unlikely to convert a novice data-oriented python user into someone that can create even the simplest catalogs. They will certainly never use more advanced features like parametrisation or derived datasets. The new model eases users in and lends itself to being overlaid with graphical/wizard interfaces (i.e., in jupyterlab or in preparation for use with [anaconda.cloud](https://docs.anaconda.com/free/anaconda-notebooks/notebook-data-catalog/)). ### Main changes This is a total rewrite. Backward compatibility is desired and some v1 sources have been rewritten to use the v2 readers. #### Simplification We are dropping features that added complexity but were only rarely used. - the server; the Intake server was never production-ready, and most use-cases can be provided by [tiled](https://blueskyproject.io/tiled/) - the caching/persist stuff; files can be persisted by fsspec, and we maintain the ability to write to various formats - explicit dependence on dask; dask is just one of many possible compute engines and an we should not be tied to one - less added functionality in the readers (like file pattern stuff) - explicit dependence on hvplot (but you can still choose to use it) - the CLI #### New structure Many new classes have appeared. From an intake-savy point of view, the biggest change is the splitting of "drivers" into "data" and "reader". I view them as the objective description of what the dataset is (e.g., "this is CSV at this URL") versus how you might load it ("call pandas with these arguments"). This strongly implies that you might want to read the same data in different ways. Crucially, it makes the readers much easier to write. Here is the Awkward reader for parquet files. Particularly for files, often all you need to do is specify which function will do the read and what keyword accepts the URL. ```python class AwkwardParquet(Awkward): implements = {datatypes.Parquet} imports = {"awkward", "pyarrow"} func = "awkward:from_parquet" url_arg = "path" ``` The imports are declared and deferred until needed, so there is no need to make all those intake-* repos with their own dependencies. (Of course, you might still want to declare packages and requirements; considering whether catalogs should have requirements, but this is better suited for something like conda-project). The arguments accepted are the same as for the target function, and the method `.doc()` will show this. ### New features - recommendation system to try to guess the right data type from a URL or existing function call, and readers that can use that type (and for each, tells you the instance it makes and provides docs). Can be extended to "I have this type but I want this other type, what set of steps get me there" - embracing any compute engines as first-class (e.g., duck, dask, ray, spark) or none - no constraints on the types of data that can/should be returned - pipeline building tools, including explicit conversion, types operations, generalised getattr and getitem (like dask delayed) and apply. Most of these available as "transform" attributes, including new namespaces like "reader.np.max(..)" will call numpy on whatever the reader makes, but lazily. - output functions, as a special type of "conversion", returning a new data description for further manipulation. This is effectively caching (would like to add conditions to the pipeline, only load and convert if converted version doesn't already exist). - generalised derived datasets, including functions of multiple intake inputs. A data or any reader output might be the input of any other reader, forming a graph. Picking a specific output from those possible gives you the pipeline, ready for execution. Any such pipelines could be encoded in a catalog. - user parameters are similar to before, but are also plugable; a few types are provided. Some helper methods have been made to walk data/reader kwargs and extract default values as parameters, replacing their original value with a reference to the parameter. The parameters are hierarchical catalog->data->reader Some examples of each of these exist in the current state of the code. There are many many more to write, but the functions themselves are really simple. This is aiming for composition and easy crowd sourcing, high bus factor. ### Work to follow - thorough search capability, which will need some thoughts in this context - compatibility with remaining existing intake plugins - the catalog serialisation currently uses custom YAML tags, but this should not be necessary - add those magic methods that make pipelines work on descriptions on catalogs, not just materialised readers. - metadata conventions, to persist basic dataset properties (e.g., based on frictionlessdata spec) and validation as a pipeline operation you can do to any data entry using any available reader that can produce the info - probably much more - I will need help! ### Unanswered questions - actual functions and classes are now embedded into any YAML serialised catalog as strings. These are imported/instantiated when the reader is instantiated from its description. So arbitrary code execution is possible, but not at catalog parse time. We only have a loose permissions config story around this - this implementation maintains the distinction between "descriptions" (which have templated values and user parameters) and readers (which only have concrete values and real instances). Is this a major confusion we somehow want to eliminate in V2? ================================================ FILE: docs/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build SPHINXPROJ = intake SOURCEDIR = source BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Run custom script to build HTML table of plugins html: Makefile python plugins.py @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/README.md ================================================ # Building Documentation An environment with several prerequisites is needed to build the documentation. Create this with: ## First option for environment ```bash conda create -n intake-docs python=3.8 pandas dask python-snappy appdirs -c conda-forge -y conda activate intake-docs ``` Additional pip packages are listed in `./requirements.txt` are required to build the docs: ```bash pip install -r requirements.txt ``` ## Second option for environment A conda environment with pip packages included is in `environment.yml` of the current directory, and you may create it with: ```bash conda env create conda activate intake-docs ``` ## Build docs To make HTML documentation: ```bash make html ``` Outputs to `build/html/index.html` ================================================ FILE: docs/environment.yml ================================================ name: intake-docs channels: - conda-forge dependencies: - appdirs - python=3.12 - dask - numpy - pandas - msgpack-python - msgpack-numpy - requests - tornado - jinja2 - python-snappy - pyyaml - hvplot - platformdirs - panel - bokeh - docutils - sphinx - sphinx_rtd_theme - numpydoc - entrypoints - aiohttp ================================================ FILE: docs/make.bat ================================================ @ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=source set BUILDDIR=build set SPHINXPROJ=intake if "%1" == "" goto help %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% :end popd ================================================ FILE: docs/make_api.py ================================================ import os import sys import intake def run(path): fn = os.path.join(path, "source", "api2.rst") with open(fn, "w") as f: print( f""" API Reference ============= User Functions -------------- .. autosummary:: intake.config.Config intake.readers.datatypes.recommend intake.readers.convert.auto_pipeline intake.readers.entry.Catalog intake.readers.entry.DataDescription intake.readers.entry.ReaderDescription intake.readers.readers.recommend intake.readers.readers.reader_from_call .. autoclass:: intake.config.Config :members: .. autofunction:: intake.readers.datatypes.recommend .. autofunction:: intake.readers.convert.auto_pipeline .. autoclass:: intake.readers.entry.Catalog :members: .. autoclass:: intake.readers.entry.DataDescription :members: .. autoclass:: intake.readers.entry.ReaderDescription :members: .. autofunction:: intake.readers.readers.recommend .. autofunction:: intake.readers.readers.reader_from_call Base Classes ------------ These may be subclassed by developers .. autosummary::""", file=f, ) bases = ( "intake.readers.datatypes.BaseData", "intake.readers.readers.BaseReader", "intake.readers.convert.BaseConverter", "intake.readers.namespaces.Namespace", "intake.readers.search.SearchBase", "intake.readers.user_parameters.BaseUserParameter", ) for base in bases: print(" ", base, file=f) print(file=f) for base in bases: print( f""".. autoclass:: {base} :members: """, file=f, ) print( """ Data Classes ------------ .. autosummary::""", file=f, ) for cls in sorted(intake.readers.subclasses(intake.BaseData), key=lambda c: c.qname()): print(" ", cls.qname().replace(":", "."), file=f) print( """ Reader Classes -------------- Includes readers, transformers, converters and output classes. .. autosummary::""", file=f, ) for cls in sorted(intake.readers.subclasses(intake.BaseReader), key=lambda c: c.qname()): print(" ", cls.qname().replace(":", "."), file=f) if __name__ == "__main__": here = os.path.abspath(os.path.dirname(sys.argv[0])) run(here) else: here = os.path.abspath(os.path.dirname(__file__)) ================================================ FILE: docs/plugins.py ================================================ import asyncio import aiohttp import pandas as pd import yaml def format_package_links(package_name, repo_link): return f'{package_name}' def format_repo_link(repo_link): if "http" not in repo_link: return f"https://github.com/{repo_link}/" return repo_link def format_badge_html(badge, link): return f'' async def check_ok(client, url): async with client.get(url) as r: if "anaconda.org" in url: body = await r.text() if "requires authentication" in body: return False return r.ok async def check_all_ok(urls): async with aiohttp.client.ClientSession() as c: coroutines = [check_ok(c, url) for url in urls] return await asyncio.gather(*coroutines) def generate_plugin_table(): plugins = yaml.safe_load(open("plugins.yaml", "rb")) plugin_df = pd.DataFrame(plugins) plugin_df = plugin_df.rename(columns={"description": "Description", "drivers": "Drivers"}) plugin_df["short_name"] = plugin_df["name"].apply(lambda x: x.split("/")[-1]) plugin_df["repo_links"] = plugin_df["repo"].apply(format_repo_link) plugin_df["conda_package"] = plugin_df["conda_package"].fillna(plugin_df["short_name"]) plugin_df["ci_yaml"] = plugin_df["ci_yaml"].fillna("main.yaml") # CI badges plugin_df["ci_badges"] = plugin_df[["repo_links", "ci_yaml"]].apply( lambda x: f"{x[0]}/actions/workflows/{x[1]}/badge.svg", axis=1 ) plugin_df["ci_links"] = plugin_df["repo_links"].apply(lambda x: f"{x}/actions") ci_badges_ok = asyncio.run(check_all_ok(plugin_df["ci_badges"])) # Docs badges plugin_df["docs_badges"] = plugin_df["short_name"].apply( lambda x: f"https://readthedocs.org/projects/{x}/badge/?version=latest" ) plugin_df["docs_links"] = plugin_df["short_name"].apply( lambda x: f"https://{x}.readthedocs.io/en/latest/?badge=latest" ) docs_badges_ok = asyncio.run(check_all_ok(plugin_df["docs_links"])) # PyPi badges plugin_df["pypi_badges"] = plugin_df["short_name"].apply( lambda x: f"https://img.shields.io/pypi/v/{x}.svg?maxAge=3600" ) plugin_df["pypi_links"] = plugin_df["short_name"].apply( lambda x: f"https://pypi.org/project/{x}" ) pypi_badges_ok = asyncio.run(check_all_ok(plugin_df["pypi_links"])) # Conda badges plugin_df["conda_badges"] = plugin_df[["conda_channel", "conda_package"]].apply( lambda x: f"https://img.shields.io/conda/vn/{x[0]}/{x[1]}.svg?colorB=4488ff&label={x[0]}&style=flat", axis=1, ) plugin_df["conda_links"] = plugin_df[["conda_channel", "conda_package"]].apply( lambda x: f"https://anaconda.org/{x[0]}/{x[1]}", axis=1 ) conda_badges_ok = asyncio.run(check_all_ok(plugin_df["conda_links"])) # Conda defaults badges plugin_df["conda_defaults_links"] = plugin_df["conda_package"].apply( lambda x: f"https://anaconda.org/anaconda/{x}" ) plugin_df["conda_defaults_badges"] = plugin_df["conda_package"].apply( lambda x: f"https://img.shields.io/conda/vn/anaconda/{x}.svg?colorB=4488ff&label=defaults&style=flat" ) conda_defaults_badges_ok = asyncio.run(check_all_ok(plugin_df["conda_defaults_links"])) # Conda forge badges plugin_df["conda_forge_links"] = plugin_df["conda_package"].apply( lambda x: f"https://anaconda.org/conda-forge/{x}" ) plugin_df["conda_forge_badges"] = plugin_df["conda_package"].apply( lambda x: f"https://img.shields.io/conda/vn/conda-forge/{x}.svg?colorB=4488ff&style=flat" ) conda_forge_badges_ok = asyncio.run(check_all_ok(plugin_df["conda_forge_links"])) plugin_df["Package Name"] = plugin_df[["name", "repo_links"]].apply( lambda x: format_package_links(*x), axis=1 ) plugin_df["CI"] = plugin_df[["ci_badges", "ci_links"]][ci_badges_ok].apply( lambda x: format_badge_html(*x), axis=1 ) plugin_df["Docs"] = plugin_df[["docs_badges", "docs_links"]][docs_badges_ok].apply( lambda x: format_badge_html(*x), axis=1 ) plugin_df["PyPi"] = plugin_df[["pypi_badges", "pypi_links"]][pypi_badges_ok].apply( lambda x: format_badge_html(*x), axis=1 ) plugin_df["conda"] = plugin_df[["conda_badges", "conda_links"]][conda_badges_ok].apply( lambda x: format_badge_html(*x), axis=1 ) plugin_df["conda_forge"] = plugin_df[["conda_forge_badges", "conda_forge_links"]][ conda_forge_badges_ok ].apply(lambda x: format_badge_html(*x), axis=1) plugin_df["conda_defaults"] = plugin_df[["conda_defaults_badges", "conda_defaults_links"]][ conda_defaults_badges_ok ].apply(lambda x: format_badge_html(*x), axis=1) plugin_df = plugin_df.fillna("") # Concat conda badges plugin_df["Conda"] = plugin_df["conda"] + plugin_df["conda_forge"] + plugin_df["conda_defaults"] plugin_df.to_html( "source/plugin-list.html", escape=False, justify="left", index=False, border=0, classes="table_wrapper", columns=["Package Name", "Description", "Drivers", "CI", "Docs", "PyPi", "Conda"], col_space=["auto", "auto", "auto", "90px", "90px", "90px", "90px"], ) if __name__ == "__main__": print("Generating custom plugin table... ", end="") generate_plugin_table() print("done") ================================================ FILE: docs/plugins.yaml ================================================ - name: intake repo: intake/intake description: Builtin to Intake drivers: catalog, csv, intake_remote, ndzarr, numpy, textfiles, yaml_file_cat, yaml_files_cat, zarr_cat, json, jsonl - name: intake-astro repo: intake/intake-astro description: Table and array loading of FITS astronomical data drivers: fits_array, fits_table conda_channel: intake - name: intake-accumulo repo: intake/intake-accumulo description: Apache Accumulo clustered data storage drivers: accumulo - name: intake-avro repo: intake/intake-avro description: Apache Avro data serialization format drivers: avro_table, avro_sequence - name: intake-bluesky repo: nsls-ii/intake-bluesky description: Search and retrieve data in the bluesky data model - name: intake-dcat repo: CityOfLosAngeles/intake-dcat description: Browse and load data from DCAT catalogs drivers: dcat - name: intake-dremio repo: intake/intake-dremio description: Scan tables and send SQL queries to a Dremio server drivers: dremio - name: intake-duckdb repo: intake/intake-duckdb description: Load DuckDB tables and build catalogs from DuckDB backends drivers: duckdb, duckdb_cat - name: intake-dynamodb repo: informatics-lab/intake-dynamodb> description: Link to Amazon DynamoDB drivers: dynamodb conda_channel: informaticslab conda_package: intake_dynamodb - name: intake-elasticsearch repo: intake/intake-elasticsearch description: Elasticsearch search and analytics engine drivers: elasticsearch_seq, elasticsearch_table - name: intake-esm repo: NCAR/intake-esm description: Plugin for building and loading intake catalogs for earth system data sets holdings, such as CMIP (Coupled Model Intercomparison Project) and CESM Large Ensemble datasets - name: intake-geopandas repo: informatics-lab/intake_geopandas description: Load from ESRI Shape Files, GeoJSON, and geospatial databases with geopandas drivers: geojson, postgis, shapefile, spatialite, regionmask conda_channel: informaticslab - name: intake-google-analytics repo: intake/intake-google-analytics description: Run Google Analytics queries and load data as a DataFrame drivers: google_analytics_query - name: intake-hbase repo: intake/intake-hbase description: Apache HBase database drivers: hbase conda_channel: intake - name: intake-iris repo: informatics-lab/intake-iris description: Load netCDF and GRIB files with IRIS drivers: grib, netcdf conda_channel: informaticslab conda_package: intake_iris - name: intake-metabase repo: continuumio/intake-metabase description: Generate catalogs and load tables as DataFrames from Metabase drivers: metabase_catalog, metabase_table - name: intake-mongo repo: intake/intake-mongo description: MongoDB noSQL query drivers: mongo conda_channel: intake - name: intake-nested-yaml-catalog repo: zillow/intake-nested-yaml-catalog description: Plugin supporting a single YAML hierarchical catalog to organize datasets and avoid a data swamp drivers: nested_yaml_cat - name: intake-netflow repo: intake/intake-netflow description: Netflow packet format drivers: netflow conda_channel: intake - name: intake-notebook repo: informatics-lab/intake-notebook description: Experimental plugin to access parameterised notebooks through intake and executed via papermill drivers: ipynb conda_channel: informaticslab - name: intake-odbc repo: intake/intake-odbc description: ODBC database drivers: odbc conda_channel: intake - name: intake-parquet repo: intake/intake-parquet description: Apache Parquet file format drivers: parquet - name: intake-pattern-catalog repo: DTN-Public/intake-pattern-catalog description: Plugin for specifying a file-path pattern which can represent a number of different entries drivers: pattern_cat - name: intake-pcap repo: intake/intake-pcap description: PCAP network packet format drivers: pcap - name: intake-postgres repo: intake/intake-postgres description: PostgreSQL database drivers: postgres conda_channel: intake - name: intake-s3-manifests repo: informatics-lab/intake-s3-manifests drivers: s3_manifest conda_channel: informaticslab conda_package: intake_s3_manifests - name: intake-salesforce repo: sophiamyang/intake-salesforce description: Generate catalogs and load tables as DataFrames from Salesforce drivers: salesforce_catalog, salesforce_table - name: intake-sdmx repo: dr-leo/intake_sdmx description: Plugin for SDMX-compliant data sources such as BIS, ECB, ESTAT, INSEE, ILO, UN, UNICEF, World Bank and more drivers: sdmx_dataset - name: intake-sklearn repo: AlbertDeFusco/intake-sklearn description: Load scikit-learn models from Pickle files drivers: sklearn - name: intake-solr repo: intake/intake-solr description: Apache Solr search platform drivers: solr conda_channel: intake - name: intake-stac repo: intake/intake-stac description: Intake Driver for SpatioTemporal Asset Catalogs (STAC) - name: intake-stripe repo: sophiamyang/intake-stripe description: Generate catalogs and load tables as DataFrames from Stripe drivers: stripe_catalog, stripe_table - name: intake-spark repo: intake/intake-spark description: Data processed by Apache Spark drivers: spark_cat, spark_rdd, spark_dataframe - name: intake-sql repo: intake/intake-sql description: Generic SQL queries via SQLAlchemy drivers: sql_cat, sql, sql_auto, sql_manual - name: intake-sqlite repo: catalyst-cooperative/intake-sqlite description: Local caching of remote SQLite DBs and queries via SQLAlchemy drivers: sqlite_cat, sqlite, sqlite_auto, sqlite_manual - name: intake-splunk repo: intake/intake-splunk description: Splunk machine data query drivers: splunk conda_channel: intake - name: intake-streamz repo: intake/intake-streamz description: Real-time event processing using Streamz drivers: streamz - name: intake-thredds repo: NCAR/intake-thredds ci_yaml: ci.yaml description: Intake interface to THREDDS data catalogs drivers: thredds_cat, thredds_merged_source - name: intake-xarray repo: intake/intake-xarray description: Load netCDF, Zarr and other multi-dimensional data drivers: xarray_image, netcdf, grib, opendap, rasterio, remote-xarray, zarr - name: intake-dataframe-catalog repo: ACCESS-NRI/intake-dataframe-catalog ci_yaml: ci.yml description: A searchable table of intake sources and associated metadata drivers: df_catalog conda_channel: accessnri ================================================ FILE: docs/requirements.txt ================================================ sphinx sphinx_rtd_theme numpydoc panel hvplot entrypoints ================================================ FILE: docs/source/_static/.keep ================================================ ================================================ FILE: docs/source/_static/css/custom.css ================================================ div.prompt { display: none } div.logo-block img { display: none !important } .table_wrapper{ display: block; overflow-x: auto; } .table_wrapper td, th { padding: 2px; } .table_wrapper tr:nth-child(even) { background: #E0E0E0; } ================================================ FILE: docs/source/_static/images/plotting_example.html ================================================ HoloPlot Plot
================================================ FILE: docs/source/api.rst ================================================ API === Auto-generated reference .. toctree:: :maxdepth: 1 api_user.rst api_base.rst api_other.rst .. raw:: html ================================================ FILE: docs/source/api2.rst ================================================ .. _api2: API Reference ============= User Functions -------------- .. autosummary:: intake.config.Config intake.readers.datatypes.recommend intake.readers.convert.auto_pipeline intake.readers.convert.path intake.readers.entry.Catalog intake.readers.entry.DataDescription intake.readers.entry.ReaderDescription intake.readers.readers.recommend intake.readers.readers.reader_from_call .. autoclass:: intake.config.Config :members: .. autofunction:: intake.readers.datatypes.recommend .. autofunction:: intake.readers.convert.auto_pipeline .. autoclass:: intake.readers.entry.Catalog :members: .. autoclass:: intake.readers.entry.DataDescription :members: .. autoclass:: intake.readers.entry.ReaderDescription :members: .. autofunction:: intake.readers.readers.recommend .. autofunction:: intake.readers.readers.reader_from_call .. _base: Base Classes ------------ These may be subclassed by developers .. autosummary:: intake.readers.datatypes.BaseData intake.readers.readers.BaseReader intake.readers.convert.BaseConverter intake.readers.namespaces.Namespace intake.readers.search.SearchBase intake.readers.user_parameters.BaseUserParameter .. autoclass:: intake.readers.datatypes.BaseData :members: .. autoclass:: intake.readers.readers.BaseReader :members: .. autoclass:: intake.readers.convert.BaseConverter :members: .. autoclass:: intake.readers.namespaces.Namespace :members: .. autoclass:: intake.readers.search.SearchBase :members: .. autoclass:: intake.readers.user_parameters.BaseUserParameter :members: .. _data: Data Classes ------------ .. autosummary:: intake.readers.datatypes.ASDF intake.readers.datatypes.AVRO intake.readers.datatypes.CSV intake.readers.datatypes.Catalog intake.readers.datatypes.CatalogAPI intake.readers.datatypes.CatalogFile intake.readers.datatypes.DICOM intake.readers.datatypes.DeltalakeTable intake.readers.datatypes.Excel intake.readers.datatypes.FITS intake.readers.datatypes.Feather1 intake.readers.datatypes.Feather2 intake.readers.datatypes.FileData intake.readers.datatypes.GDALRasterFile intake.readers.datatypes.GDALVectorFile intake.readers.datatypes.GRIB2 intake.readers.datatypes.GeoJSON intake.readers.datatypes.GeoPackage intake.readers.datatypes.HDF5 intake.readers.datatypes.Handle intake.readers.datatypes.HuggingfaceDataset intake.readers.datatypes.IcebergDataset intake.readers.datatypes.JPEG intake.readers.datatypes.JSONFile intake.readers.datatypes.KerasModel intake.readers.datatypes.Literal intake.readers.datatypes.MatlabArray intake.readers.datatypes.MatrixMarket intake.readers.datatypes.NetCDF3 intake.readers.datatypes.Nifti intake.readers.datatypes.NumpyFile intake.readers.datatypes.ORC intake.readers.datatypes.OpenDAP intake.readers.datatypes.PNG intake.readers.datatypes.Parquet intake.readers.datatypes.PickleFile intake.readers.datatypes.Prometheus intake.readers.datatypes.PythonSourceCode intake.readers.datatypes.RawBuffer intake.readers.datatypes.SKLearnPickleModel intake.readers.datatypes.SQLQuery intake.readers.datatypes.SQLite intake.readers.datatypes.STACJSON intake.readers.datatypes.Service intake.readers.datatypes.Shapefile intake.readers.datatypes.TFRecord intake.readers.datatypes.THREDDSCatalog intake.readers.datatypes.TIFF intake.readers.datatypes.Text intake.readers.datatypes.TileDB intake.readers.datatypes.TiledDataset intake.readers.datatypes.TiledService intake.readers.datatypes.WAV intake.readers.datatypes.XML intake.readers.datatypes.YAMLFile intake.readers.datatypes.Zarr .. _reader: Reader Classes -------------- Includes readers, transformers, converters and output classes. .. autosummary:: intake.readers.catalogs.EarthdataCatalogReader intake.readers.catalogs.EarthdataReader intake.readers.catalogs.HuggingfaceHubCatalog intake.readers.catalogs.SKLearnExamplesCatalog intake.readers.catalogs.SQLAlchemyCatalog intake.readers.catalogs.STACIndex intake.readers.catalogs.StacCatalogReader intake.readers.catalogs.StacSearch intake.readers.catalogs.StackBands intake.readers.catalogs.THREDDSCatalogReader intake.readers.catalogs.TensorFlowDatasetsCatalog intake.readers.catalogs.TiledCatalogReader intake.readers.catalogs.TorchDatasetsCatalog intake.readers.convert.ASDFToNumpy intake.readers.convert.BaseConverter intake.readers.convert.DaskArrayToTileDB intake.readers.convert.DaskDFToPandas intake.readers.convert.DaskToRay intake.readers.convert.DeltaQueryToDask intake.readers.convert.DeltaQueryToDaskGeopandas intake.readers.convert.DicomToNumpy intake.readers.convert.DuckToPandas intake.readers.convert.FITSToNumpy intake.readers.convert.GenericFunc intake.readers.convert.HuggingfaceToRay intake.readers.convert.NibabelToNumpy intake.readers.convert.NumpyToTileDB intake.readers.convert.PandasToGeopandas intake.readers.convert.PandasToMetagraph intake.readers.convert.PandasToPolars intake.readers.convert.PandasToRay intake.readers.convert.Pipeline intake.readers.convert.PolarsEager intake.readers.convert.PolarsLazy intake.readers.convert.PolarsToPandas intake.readers.convert.RayToDask intake.readers.convert.RayToPandas intake.readers.convert.RayToSpark intake.readers.convert.SparkDFToRay intake.readers.convert.TileDBToNumpy intake.readers.convert.TileDBToPandas intake.readers.convert.TiledNodeToCatalog intake.readers.convert.TiledSearch intake.readers.convert.ToHvPlot intake.readers.convert.TorchToRay intake.readers.output.CatalogToJson intake.readers.output.DaskArrayToZarr intake.readers.output.GeopandasToFile intake.readers.output.MatplotlibToPNG intake.readers.output.NumpyToNumpyFile intake.readers.output.PandasToCSV intake.readers.output.PandasToFeather intake.readers.output.PandasToHDF5 intake.readers.output.PandasToParquet intake.readers.output.Repr intake.readers.output.ToMatplotlib intake.readers.output.XarrayToNetCDF intake.readers.output.XarrayToZarr intake.readers.readers.ASDFReader intake.readers.readers.Awkward intake.readers.readers.AwkwardAVRO intake.readers.readers.AwkwardJSON intake.readers.readers.AwkwardParquet intake.readers.readers.Condition intake.readers.readers.CupyNumpyReader intake.readers.readers.CupyTextReader intake.readers.readers.DaskAwkwardJSON intake.readers.readers.DaskAwkwardParquet intake.readers.readers.DaskCSV intake.readers.readers.DaskDF intake.readers.readers.DaskDeltaLake intake.readers.readers.DaskHDF intake.readers.readers.DaskJSON intake.readers.readers.DaskNPYStack intake.readers.readers.DaskParquet intake.readers.readers.DaskSQL intake.readers.readers.DaskZarr intake.readers.readers.DeltaReader intake.readers.readers.DicomReader intake.readers.readers.DuckCSV intake.readers.readers.DuckDB intake.readers.readers.DuckJSON intake.readers.readers.DuckParquet intake.readers.readers.DuckSQL intake.readers.readers.FITSReader intake.readers.readers.FileByteReader intake.readers.readers.FileExistsReader intake.readers.readers.FileReader intake.readers.readers.GeoPandasReader intake.readers.readers.GeoPandasTabular intake.readers.readers.HandleToUrlReader intake.readers.readers.HuggingfaceReader intake.readers.readers.KerasAudio intake.readers.readers.KerasImageReader intake.readers.readers.KerasModelReader intake.readers.readers.KerasText intake.readers.readers.NibabelNiftiReader intake.readers.readers.NumpyReader intake.readers.readers.NumpyText intake.readers.readers.NumpyZarr intake.readers.readers.Pandas intake.readers.readers.PandasCSV intake.readers.readers.PandasExcel intake.readers.readers.PandasFeather intake.readers.readers.PandasHDF5 intake.readers.readers.PandasORC intake.readers.readers.PandasParquet intake.readers.readers.PandasSQLAlchemy intake.readers.readers.Polars intake.readers.readers.PolarsAvro intake.readers.readers.PolarsCSV intake.readers.readers.PolarsDeltaLake intake.readers.readers.PolarsExcel intake.readers.readers.PolarsFeather intake.readers.readers.PolarsIceberg intake.readers.readers.PolarsJSON intake.readers.readers.PolarsParquet intake.readers.readers.PrometheusMetricReader intake.readers.readers.PythonModule intake.readers.readers.RasterIOXarrayReader intake.readers.readers.Ray intake.readers.readers.RayBinary intake.readers.readers.RayCSV intake.readers.readers.RayDeltaLake intake.readers.readers.RayJSON intake.readers.readers.RayParquet intake.readers.readers.RayText intake.readers.readers.Retry intake.readers.readers.SKImageReader intake.readers.readers.SKLearnExampleReader intake.readers.readers.SKLearnModelReader intake.readers.readers.ScipyMatlabReader intake.readers.readers.ScipyMatrixMarketReader intake.readers.readers.SparkCSV intake.readers.readers.SparkDataFrame intake.readers.readers.SparkDeltaLake intake.readers.readers.SparkParquet intake.readers.readers.SparkText intake.readers.readers.TFORC intake.readers.readers.TFPublicDataset intake.readers.readers.TFRecordReader intake.readers.readers.TFSQL intake.readers.readers.TFTextreader intake.readers.readers.TileDBDaskReader intake.readers.readers.TileDBReader intake.readers.readers.TiledClient intake.readers.readers.TiledNode intake.readers.readers.TorchDataset intake.readers.readers.XArrayDatasetReader intake.readers.readers.YAMLCatalogReader intake.readers.transform.DataFrameColumns intake.readers.transform.GetItem intake.readers.transform.Method intake.readers.transform.PysparkColumns intake.readers.transform.THREDDSCatToMergedDataset intake.readers.transform.XarraySel ================================================ FILE: docs/source/api_base.rst ================================================ Base Classes ------------ This is a reference API class listing, useful mainly for developers. .. autosummary:: intake.source.base.DataSourceBase intake.source.base.DataSource intake.catalog.Catalog intake.catalog.entry.CatalogEntry intake.catalog.local.UserParameter intake.source.derived.AliasSource intake.source.base.Schema .. autoclass:: intake.source.base.DataSource :members: .. attribute:: plot Accessor for HVPlot methods. See :doc:`plotting` for more details. .. autoclass:: intake.catalog.Catalog :members: .. autoclass:: intake.catalog.entry.CatalogEntry :members: .. autoclass:: intake.catalog.local.UserParameter :members: .. autoclass:: intake.source.derived.AliasSource :members: .. autoclass:: intake.source.base.Schema :members: .. raw:: html ================================================ FILE: docs/source/api_other.rst ================================================ Other Classes ============= GUI --- .. autosummary:: intake.interface.base.Base intake.interface.base.BaseSelector intake.interface.base.BaseView intake.interface.catalog.add.FileSelector intake.interface.catalog.add.URLSelector intake.interface.catalog.add.CatAdder intake.interface.catalog.search.Search intake.interface.source.defined_plots.Plots .. autoclass:: intake.interface.base.Base :members: .. autoclass:: intake.interface.base.BaseSelector :members: .. autoclass:: intake.interface.base.BaseView :members: .. autoclass:: intake.interface.catalog.add.FileSelector :members: .. autoclass:: intake.interface.catalog.add.URLSelector :members: .. autoclass:: intake.interface.catalog.add.CatAdder :members: .. autoclass:: intake.interface.catalog.search.Search :members: .. autoclass:: intake.interface.source.defined_plots.Plots :members: .. raw:: html ================================================ FILE: docs/source/api_user.rst ================================================ End User -------- These are reference class and function definitions likely to be useful to everyone. .. autosummary:: intake.open_catalog intake.registry intake.register_driver intake.unregister_driver intake.source.csv.CSVSource intake.source.textfiles.TextFilesSource intake.source.jsonfiles.JSONFileSource intake.source.jsonfiles.JSONLinesFileSource intake.source.npy.NPySource intake.source.zarr.ZarrArraySource intake.catalog.local.YAMLFileCatalog intake.catalog.local.YAMLFilesCatalog intake.catalog.zarr.ZarrGroupCatalog .. autofunction:: intake.open_catalog .. attribute:: intake.registry Mapping from plugin names to the DataSource classes that implement them. These are the names that should appear in the ``driver:`` key of each source definition in a catalog. See :doc:`plugin-directory` for more details. .. attribute:: intake.open_ Set of functions, one for each plugin, for direct opening of a data source. The names are derived from the names of the plugins in the registry at import time. Source classes '''''''''''''' .. autoclass:: intake.source.csv.CSVSource :members: __init__, discover, read_partition, read, to_dask .. autoclass:: intake.source.zarr.ZarrArraySource :members: __init__, discover, read_partition, read, to_dask .. autoclass:: intake.source.textfiles.TextFilesSource :members: __init__, discover, read_partition, read, to_dask .. autoclass:: intake.source.jsonfiles.JSONFileSource :members: __init__, discover, read .. autoclass:: intake.source.jsonfiles.JSONLinesFileSource :members: __init__, discover, read, head .. autoclass:: intake.source.npy.NPySource :members: __init__, discover, read_partition, read, to_dask .. autoclass:: intake.catalog.local.YAMLFileCatalog :members: __init__, reload, search, walk .. autoclass:: intake.catalog.local.YAMLFilesCatalog :members: __init__, reload, search, walk .. autoclass:: intake.catalog.zarr.ZarrGroupCatalog :members: __init__, reload, search, walk, to_zarr .. raw:: html ================================================ FILE: docs/source/catalog.rst ================================================ Catalogs ======== Data catalogs provide an abstraction that allows you to externally define, and optionally share, descriptions of datasets, called *catalog entries*. A catalog entry for a dataset includes information like: * The name of the Intake driver that can load the data * Arguments to the ``__init__()`` method of the driver * Metadata provided by the catalog author (such as field descriptions and types, or data provenance) In addition, Intake allows the arguments to data sources to be templated, with the variables explicitly expressed as "user parameters". The given arguments are rendered using ``jinja2``, the values of named user parameters, and any overrides. The parameters are also offer validation of the allowed types and values, for both the template values and the final arguments passed to the data source. The parameters are named and described, to indicate to the user what they are for. This kind of structure can be used to, for example, choose between two parts of a given data source, like "latest" and "stable", see the `entry1_part` entry in the example below. The user of the catalog can always override any template or argument value at the time that they access a give source. The Catalog class ----------------- In Intake, a ``Catalog`` instance is an object with one or more named entries. The entries might be read from a static file (e.g., YAML, see the next section), from an Intake server or from any other data service that has a driver. Drivers which create catalogs are ordinary DataSource classes, except that they have the container type "catalog", and do not return data products via the ``read()`` method. For example, you might choose to instantiate the base class and fill in some entries explicitly in your code .. code-block:: python from intake.catalog import Catalog from intake.catalog.local import LocalCatalogEntry mycat = Catalog.from_dict({ 'source1': LocalCatalogEntry(name, description, driver, args=...), ... }) Alternatively, subclasses of ``Catalog`` can define how entries are created from whichever file format or service they interact with, examples including ``RemoteCatalog`` and `SQLCatalog`_. These generate entries based on their respective targets; some provide advanced search capabilities executed on the server. .. _SQLCatalog: https://intake-sql.readthedocs.io/en/latest/api.html#intake_sql.SQLCatalog YAML Format ----------- Intake catalogs can most simply be described with YAML files. This is very common in the tutorials and this documentation, because it simple to understand, but demonstrate the many features of Intake. Note that YAML files are also the easiest way to share a catalog, simply by copying to a publicly-available location such as a cloud storage bucket. Here is an example: .. code-block:: yaml metadata: version: 1 parameters: file_name: type: str description: default file name for child entries default: example_file_name sources: example: description: test driver: random args: {} entry1_full: description: entry1 full metadata: foo: 'bar' bar: [1, 2, 3] driver: csv args: # passed to the open() method urlpath: '{{ CATALOG_DIR }}/entry1_*.csv' entry1_part: description: entry1 part parameters: # User parameters part: description: section of the data type: str default: "stable" allowed: ["latest", "stable"] driver: csv args: urlpath: '{{ CATALOG_DIR }}/entry1_{{ part }}.csv' entry2: description: entry2 driver: csv args: # file_name parameter will be inherited from file-level parameters, so will # default to "example_file_name" urlpath: '{{ CATALOG_DIR }}/entry2/{{ file_name }}.csv` Metadata '''''''' Arbitrary extra descriptive information can go into the metadata section. Some fields will be claimed for internal use and some fields may be restricted to local reading; but for now the only field that is expected is ``version``, which will be updated when a breaking change is made to the file format. Any catalog will have ``.metadata`` and ``.version`` attributes available. Note that each source also has its own metadata. The metadata section an also contain ``parameters`` which will be inherited by the sources in the file (note that these sources can augment these parameters, or override them with their own parameters). Extra drivers ''''''''''''' The ``driver:`` entry of a data source specification can be a driver name, as has been shown in the examples so far. It can also be an absolute class path to use for the data source, in which case there will be no ambiguity about how to load the data. That is the the preferred way to be explicit, when the driver name alone is not enough (see `Driver Selection`_, below). .. code-block:: yaml plugins: source: - module: intake.catalog.tests.example1_source sources: ... However, you do not, in general, need to do this, since the ``driver:`` field of each source can also explicitly refer to the plugin class. Sources ''''''' The majority of a catalog file is composed of data sources, which are named data sets that can be loaded for the user. Catalog authors describe the contents of data set, how to load it, and optionally offer some customization of the returned data. Each data source has several attributes: - ``name``: The canonical name of the source. Best practice is to compose source names from valid Python identifiers. This allows Intake to support things like tab completion of data source names on catalog objects. For example, ``monthly_downloads`` is a good source name. - ``description``: Human readable description of the source. To help catalog browsing tools, the description should be Markdown. - ``driver``: Name of the Intake :term:`Driver` to use with this source. Must either already be installed in the current Python environment (i.e. with conda or pip) or loaded in the ``plugin`` section of the file. Can be a simple driver name like "csv" or the full path to the implementation class like "package.module.Class". - ``args``: Keyword arguments to the init method of the driver. Arguments may use template expansion. - ``metadata``: Any metadata keys that should be attached to the data source when opened. These will be supplemented by additional metadata provided by the driver. Catalog authors can use whatever key names they would like, with the exception that keys starting with a leading underscore are reserved for future internal use by Intake. - ``direct_access``: Control whether the data is directly accessed by the client, or proxied through a catalog server. See :ref:`remote-catalogs` for more details. - ``parameters``: A dictionary of data source parameters. See below for more details. Caching Source Files Locally '''''''''''''''''''''''''''' *This method of defining the cache with a dedicated block is deprecated, see the Remote Access section, below* To enable caching on the first read of remote data source files, add the ``cache`` section with the following attributes: - ``argkey``: The args section key which contains the URL(s) of the data to be cached. - ``type``: One of the keys in the cache registry [`intake.source.cache.registry`], referring to an implementation of caching behaviour. The default is "file" for the caching of one or more files. Example: .. code-block:: yaml test_cache: description: cache a csv file from the local filesystem driver: csv cache: - argkey: urlpath type: file args: urlpath: '{{ CATALOG_DIR }}/cache_data/states.csv' The ``cache_dir`` defaults to ``~/.intake/cache``, and can be specified in the intake configuration file or ``INTAKE_CACHE_DIR`` environment variable, or at runtime using the ``"cache_dir"`` key of the configuration. The special value ``"catdir"`` implies that cached files will appear in the same directory as the catalog file in which the data source is defined, within a directory named "intake_cache". These will not appear in the cache usage reported by the CLI. Optionally, the cache section can have a ``regex`` attribute, that modifies the path of the cache on the disk. By default, the cache path is made by concatenating ``cache_dir``, dataset name, hash of the url, and the url itself (without the protocol). ``regex`` attribute allows to remove part of the url (the matching part). Caching can be disabled at runtime for all sources regardless of the catalog specification:: from intake.config import conf conf['cache_disabled'] = True By default, progress bars are shown during downloads if the package ``tqdm`` is available, but this can be disabled (e.g., for consoles that don't support complex text) with conf['cache_download_progress'] = False or, equivalently, the environment parameter ``INTAKE_CACHE_PROGRESS``. The "types" of caching are that supported are listed in ``intake.source.cache.registry``, see the docstrings of each for specific parameters that should appear in the cache block. It is possible to work with compressed source files by setting ``type: compression`` in the cache specification. By default the compression type is inferred from the file extension, otherwise it can be set by assigning the ``decomp`` variable to any of the options listed in ``intake.source.decompress.decomp``. This will extract all the file(s) in the compressed file referenced by urlpath and store them in the cache directory. In cases where miscellaneous files are present in the compressed file, a ``regex_filter`` parameter can be used. Only the extracted filenames that match the pattern will be loaded. The cache path is appended to the filename so it is necessary to include a wildcard to the beginning of the pattern. Example: .. code-block:: yaml test_compressed: driver: csv args: urlpath: 'compressed_file.tar.gz' cache: - type: compressed decomp: tgz argkey: urlpath regex_filter: '.*data.csv' Templating ---------- Intake catalog files support Jinja2 templating for driver arguments. Any occurrence of a substring like ``{{field}}`` will be replaced by the value of the user parameters with that same name, or the value explicitly provided by the user. For how to specify these user parameters, see the next section. Some additional values are available for templating. The following is always available: ``CATALOG_DIR``, the full path to the directory containing the YAML catalog file. This is especially useful for constructing paths relative to the catalog directory to locate data files and custom drivers. For example, the search for CSV files for the two "entry1" blocks, above, will happen in the same directory as where the catalog file was found. The following functions `may` be available. Since these execute code, the user of a catalog may decide whether they trust those functions or not. - ``env("USER")``: look in the set environment variables for the named variable - ``client_env("USER")``: exactly the same, except that when using a client-server topology, the value will come from the environment of the client. - ``shell("get_login thisuser -t")``: execute the command, and use the output as the value. The output will be trimmed of any trailing whitespace. - ``client_shell("get_login thisuser -t")``: exactly the same, except that when using a client-server topology, the value will come from the system of the client. The reason for the "client" versions of the functions is to prevent leakage of potentially sensitive information between client and server by controlling where lookups happen. When working without a server, only the ones without "client" are used. An example: .. code-block:: yaml sources: personal_source: description: This source needs your username args: url: "http://server:port/user/{{env(USER)}}" Here, if the user is named "blogs", the ``url`` argument will resolve to ``"http://server:port/user/blogs"``; if the environment variable is not defined, it will resolve to ``"http://server:port/user/"`` .. _paramdefs: Parameter Definition -------------------- Source parameters ''''''''''''''''' A source definition can contain a "parameters" block. Expressed in YAML, a parameter may look as follows: .. code-block:: yaml parameters: name: description: name to use # human-readable text for what this parameter means type: str # one of bool, str, int, float, list[str | int | float], datetime, mlist default: normal # optional, value to assume if user does not override allowed: ["normal", "strange"] # optional, list of values that are OK, for validation min: "n" # optional, minimum allowed, for validation max: "t" # optional, maximum allowed, for validation A parameter, not to be confused with an :term:`argument`, can have one of two uses: - to provide values for variables to be used in templating the arguments. *If* the pattern "{{name}}" exists in any of the source arguments, it will be replaced by the value of the parameter. If the user provides a value (e.g., ``source = cat.entry(name='something")``), that will be used, otherwise the default value. If there is no user input or default, the empty value appropriate for type is used. The ``default`` field allows for the same function expansion as listed for arguments, above. - *If* an argument with the same name as the parameter exists, its value, after any templating, will be coerced to the given type of the parameter and validated against the allowed/max/min. It is therefore possible to use the string templating system (e.g., to get a value from the environment), but pass the final value as, for example, an integer. It makes no sense to provide a default for this case (the argument already has a value), but providing a default will not raise an exception. - the "mlist" type is special: it means that the input must be a list, whose values are chosen from the allowed list. This is the only type where the parameter value is not the same type as the allowed list's values, e.g., if a list of str is set for ``allowed``, a list of str must also be the final value. Note: the ``datetime`` type accepts multiple values: Python datetime, ISO8601 string, Unix timestamp int, "now" and "today". Catalog parameters '''''''''''''''''' You can also define user parameters at the catalog level. This applies the parameter to all entries within that catalog, without having to define it for each and every entry. Furthermore, catalogs dested within the catalog will also inherit the parameter(s). For example, with the following spec .. code-block:: yaml metadata: version: 1 parameters: bucket: type: str description: description default: test_bucket sources: param_source: driver: parquet description: description args: urlpath: s3://{{bucket}}/file.parquet subcat: driver: yaml_file path: "{{CATALOG_DIR}}/other.yaml" If ``cat`` is the corresponsing catalog instance, the URL of source ``cat.param_source`` will evaluate to "s3://test_bucket/file.parquet" by default, but the parameter can be overridden with ``cat.param_source(bucket="other_bucket")``. Also, any entries of ``subcat``, another catalog referenced from here, would also have the "bucket"-named parameter attached to all sources. Of course, those sources do no need to make use of the parameter. To change the default, we can gerenate a new instance .. code-block:: python cat2 = cat(bucket="production") # sets default value of "bucket" for cat2 subcat = cat.subcat(bucket="production") # sets default only for the nested catalog Of course, in these situations you can still override the value of the parameter for any source, or pass explicit values for the arguments of the source, as normal. For cases where the catalog is not defined in a YAML spec, the argument ``user_parameters`` to the constructor takes the same form as ``parameters`` above: a dict of user parameters, either as ``UserParameter`` instances or as a dictionary spec for each one. Templating parameters ''''''''''''''''''''' Template functions can also be used in parameters (see `Templating`_, above), but you can use the available functions directly without the extra `{{...}}`. For example, this catalog entry uses the ``env("HOME")`` functionality as described to set a default based on the user's home directory. .. code-block:: yaml sources: variabledefault: description: "This entry leads to an example csv file in the user's home directory by default, but the user can pass root="somepath" to override that." driver: csv args: path: "{{root}}/example.csv" parameters: root: description: "root path" type: str default: "env(HOME)" Driver Selection ---------------- In some cases, it may be possible that multiple backends are capable of loading from the same data format or service. Sometimes, this may mean two drivers with unique names, or a single driver with a parameter to choose between the different backends. However, it is possible that multiple drivers for reading a particular type of data also share the same driver name: for example, both the intake-iris and the intake-xarray packages contain drivers with the name ``"netcdf"``, which are capable of reading the same files, but with different backends. Here we will describe the various possibilities of coping with this situation. Intake's plugin system makes it easy to encode such choices. It may be acceptable to use any driver which claims to handle that data type, or to give the option of which driver to use to the user, or it may be necessary to specify which precise driver(s) are appropriate for that particular data. Intake allows all of these possibilities, even if the backend drivers require extra arguments. Specifying a single driver explicitly, rather than using a generic name, would look like this: .. code-block:: yaml sources: example: description: test driver: package.module.PluginClass args: {} It is also possible to describe a list of drivers with the same syntax. The first one found will be the one used. Note that the class imports will only happen at data source instantiation, i.e., when the entry is selected from the catalog. .. code-block:: yaml sources: example: description: test driver: - package.module.PluginClass - another_package.PluginClass2 args: {} These alternative plugins can also be given data-source specific names, allowing the user to choose at load time with `driver=` as a parameter. Additional arguments may also be required for each option (which, as usual, may include user parameters); however, the same global arguments will be passed to all of the drivers listed. .. code-block:: yaml sources: example: description: test driver: first: class: package.module.PluginClass args: specific_thing: 9 second: class: another_package.PluginClass2 args: {} Remote Access ------------- (see also :ref:`remote_data` for the implementation details) Many drivers support reading directly from remote data sources such as HTTP, S3 or GCS. In these cases, the path to read from is usually given with a protocol prefix such as ``gcs://``. Additional dependencies will typically be required (``requests``, ``s3fs``, ``gcsfs``, etc.), any data package should specify these. Further parameters may be necessary for communicating with the storage backend and, by convention, the driver should take a parameter ``storage_options`` containing arguments to pass to the backend. Some remote backends may also make use of environment variables or config files to determine thier default behaviour. The special template variable "CATALOG_DIR" may be used to construct relative URLs in the arguments to a source. In such cases, if the filesystem used to load that catalog contained arguments, then the ``storage_options`` of that file system will be extracted and passed to the source. Therefore, all sources which can accept general URLs (beyond just local paths) must make sure to accept this argument. As an example of using ``storage_options``, the following two sources would allow for reading CSV data from S3 and GCS backends without authentication (anonymous access), respectively .. code-block:: yaml sources: s3_csv: driver: csv description: "Publicly accessible CSV data on S3; requires s3fs" args: urlpath: s3://bucket/path/*.csv storage_options: anon: true gcs_csv: driver: csv description: "Publicly accessible CSV data on GCS; requires gcsfs" args: urlpath: gcs://bucket/path/*.csv storage_options: token: "anon" .. _caching: **Using S3 Profiles** An AWS profile may be specified as an argument under ``storage_options`` via the following format: .. code-block:: yaml args: urlpath: s3://bucket/path/*.csv storage_options: profile: aws-profile-name Caching ''''''' URLs interpreted by ``fsspec`` offer `automatic caching`_. For example, to enable file-based caching for the first source above, you can do: .. code-block:: yaml sources: s3_csv: driver: csv description: "Publicly accessible CSV data on S3; requires s3fs" args: urlpath: simplecache::s3://bucket/path/*.csv storage_options: s3: anon: true Here we have added the "simplecache" to the URL (this caching backend does not store any metadata about the cached file) and specified that the "anon" parameter is meant as an argument to s3, not to the caching mechanism. As each file in s3 is accessed, it will first be downloaded and then the local version used instead. .. _automatic caching: https://filesystem-spec.readthedocs.io/en/latest/features.html#caching-files-locally You can tailor how the caching works. In particular the location of the local storage can be set with the ``cache_storage`` parameter (under the "simplecache" group of storage_options, of course) - otherwise they are stored in a temporary location only for the duration of the current python session. The cache location is particularly useful in conjunction with an environment variable, or relative to "{{CATALOG_DIR}}", wherever the catalog was loaded from. Please see the ``fsspec`` documentation for the full set of cache types and their various options. Local Catalogs -------------- A Catalog can be loaded from a YAML file on the local filesystem by creating a Catalog object: .. code-block:: python from intake import open_catalog cat = open_catalog('catalog.yaml') Then sources can be listed: .. code-block:: python list(cat) and data sources are loaded via their name: .. code-block:: python data = cat.entry_part1 and you can optionally configure new instances of the source to define user parameters or override arguments by calling either of: .. code-block:: python data = cat.entry_part1.configure_new(part='1') data = cat.entry_part1(part='1') # this is a convenience shorthand Intake also supports loading a catalog from all of the files ending in ``.yml`` and ``.yaml`` in a directory, or by using an explicit glob-string. Note that the URL provided may refer to a remote storage systems by passing a protocol specifier such as ``s3://``, ``gcs://``.: .. code-block:: python cat = open_catalog('/research/my_project/catalog.d/') Intake Catalog objects will automatically reload changes or new additions to catalog files and directories on disk. These changes will not affect already-opened data sources. Catalog Nesting --------------- A catalog is just another type of data source for Intake. For example, you can print a YAML specification corresponding to a catalog as follows: .. code-block:: python cat = intake.open_catalog('cat.yaml') print(cat.yaml()) results in: .. code-block:: yaml sources: cat: args: path: cat.yaml description: '' driver: intake.catalog.local.YAMLFileCatalog metadata: {} The `point` here, is that this can be included in another catalog. (It would, of course, be better to include a description and the full path of the catalog file here.) If the entry above were saved to another file, "root.yaml", and the original catalog contained an entry, ``data``, you could access it as: .. code-block:: python root = intake.open_catalog('root.yaml') root.cat.data It is, therefore, possible to build up a hierarchy of catalogs referencing each other. These can, of course, include remote URLs and indeed catalog sources other than simple files (all the tables on a SQL server, for instance). Plus, since the argument and parameter system also applies to entries such as the example above, it would be possible to give the user a runtime choice of multiple catalogs to pick between, or have this decision depend on an environment variable. .. _remote-catalogs: Server Catalogs --------------- Intake also includes a server which can share an Intake catalog over HTTP (or HTTPS with the help of a TLS-enabled reverse proxy). From the user perspective, remote catalogs function identically to local catalogs: .. code-block:: python cat = open_catalog('intake://catalog1:5000') list(cat) The difference is that operations on the catalog translate to requests sent to the catalog server. Catalog servers provide access to data sources in one of two modes: * Direct access: In this mode, the catalog server tells the client how to load the data, but the client uses its local drivers to make the connection. This requires the client has the required driver already installed *and* has direct access to the files or data servers that the driver will connect to. * Proxied access: In this mode, the catalog server uses its local drivers to open the data source and stream the data over the network to the client. The client does not need *any* special drivers to read the data, and can read data from files and data servers that it cannot access, as long as the catalog server has the required access. Whether a particular catalog entry supports direct or proxied access is determined by the ``direct_access`` option: - ``forbid`` (default): Force all clients to proxy data through the catalog server - ``allow``: If the client has the required driver, access the source directly, otherwise proxy the data through the catalog server. - ``force``: Force all clients to access the data directly. If they do not have the required driver, an exception will be raised. Note that when the client is loading a data source via direct access, the catalog server will need to send the driver arguments to the client. Do not include sensitive credentials in a data source that allows direct access. Client Authorization Plugins '''''''''''''''''''''''''''' Intake servers can check if clients are authorized to access the catalog as a whole, or individual catalog entries. Typically a matched pair of server-side plugin (called an "auth plugin") and a client-side plugin (called a "client auth plugin) need to be enabled for authorization checks to work. This feature is still in early development, but see module ``intake.auth.secret`` for a demonstration pair of server and client classes implementation auth via a shared secret. .. raw:: html ================================================ FILE: docs/source/changelog.rst ================================================ Changelog ========= 2.0.4 ----- Released March 19, 2024 - re-enable v1 entrypoint sources - expose recommend functions higher up (e.g,. intake.recommend) - add more geo types and readers, including pmtiles - add migration guide 2.0.3 ----- Released February 29, 2024 - fix v1 caches - more docs 2.0.0 ----- Released Jan 31, 2024 - complete rewrite of the package, see main docs page 0.7.0 ----- Released May 29, 2023 - be able to override arguments when using a source defined in an entry-point - make sources usable without explicit dependence on dask: zarr, textfiles, csv - removed some explicit usage (but not all) of dask throughout the codebase - new dataframe pipeline transform source .. _v0.6.8: 0.6.8 ----- Released March 11, 2023 - user parameter parsed as string before conversion to given type - numpy source becomes first to have read() path avoid dask - when registering drivers dynamically, corresponding open_* functions will be created automatically (plus refactor/cleanup of the discovery code) - docs config and style updates; the list of plugins to automatically pull in status badges - catalog .gui attribute will make top-level GUI instance instead of cut down one-catalog version - pre-commit checks added and consistent code style applied .. _v0.6.7: 0.6.7 ----- Released February 13, 2023 - server fix for upstream dask change giving newlined in report - editable plots, based on hvPlot's "explorer" - remove "text" input to YAMLFileCatalog - GUI bug fixes - allow catalog TTL as None .. _v0.6.6: 0.6.6 ----- Released on August 26, 2022. - Fixed bug in json and jsonl driver. - Ensure description is retained in the catalog. - Fix cache issue when running inside a notebook. - Add templating parameters. - Plotting api keeps hold of hvplot calls to allow other plots to be made. - docs updates - fix urljoin for server via proxy .. _v0.6.5: 0.6.5 ----- Released on January 9, 2022. - Added link to intake-google-analytics. - Add tiled driver. - Add json and jsonl drivers. - Allow parameters to be passed through catalog. - Add mlist type which allows inputs from a known list of values. .. raw:: html ================================================ FILE: docs/source/code-of-conduct.rst ================================================ Code of Conduct =============== All participants in the fsspec community are expected to adhere to a Code of Conduct. As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities. We are committed to making participation in this project a harassment-free experience for everyone, treating everyone as unique humans deserving of respect. Examples of unacceptable behaviour by participants include: - The use of sexualized language or imagery - Personal attacks - Trolling or insulting/derogatory comments - Public or private harassment - Publishing other's private information, such as physical or electronic addresses, without explicit permission - Other unethical or unprofessional conduct Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviours that they deem inappropriate, threatening, offensive, or harmful. By adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently applying these principles to every aspect of managing this project. Project maintainers who do not follow or enforce the Code of Conduct may be permanently removed from the project team. This code of conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. If you feel the code of conduct has been violated, please report the incident to the fsspec core team. Reporting --------- If you believe someone is violating theCode of Conduct we ask that you report it to the Project by emailing community@anaconda.com. All reports will be kept confidential. In some cases we may determine that a public statement will need to be made. If that's the case, the identities of all victims and reporters will remain confidential unless those individuals instruct us otherwise. If you believe anyone is in physical danger, please notify appropriate law enforcement first. In your report please include: - Your contact info - Names (real, nicknames, or pseudonyms) of any individuals involved. If there were other witnesses besides you, please try to include them as well. - When and where the incident occurred. Please be as specific as possible. - Your account of what occurred. If there is a publicly available record please include a link. - Any extra context you believe existed for the incident. - If you believe this incident is ongoing. - If you believe any member of the core team has a conflict of interest in adjudicating the incident. - What, if any, corrective response you believe would be appropriate. - Any other information you believe we should have. Core team members are obligated to maintain confidentiality with regard to the reporter and details of an incident. What happens next? ~~~~~~~~~~~~~~~~~~ You will receive an email acknowledging receipt of your complaint. The core team will immediately meet to review the incident and determine: - What happened. - Whether this event constitutes a code of conduct violation. - Who the bad actor was. - Whether this is an ongoing situation, or if there is a threat to anyone's physical safety. - If this is determined to be an ongoing incident or a threat to physical safety, the working groups' immediate priority will be to protect everyone involved. If a member of the core team is one of the named parties, they will not be included in any discussions, and will not be provided with any confidential details from the reporter. If anyone on the core team believes they have a conflict of interest in adjudicating on a reported issue, they will inform the other core team members, and exempt themselves from any discussion about the issue. Following this declaration, they will not be provided with any confidential details from the reporter. Once the working group has a complete account of the events they will make a decision as to how to response. Responses may include: - Nothing (if we determine no violation occurred). - A private reprimand from the working group to the individual(s) involved. - A public reprimand. - An imposed vacation - A permanent or temporary ban from some or all spaces (GitHub repositories, etc.) - A request for a public or private apology. We'll respond within one week to the person who filed the report with either a resolution or an explanation of why the situation is not yet resolved. Once we've determined our final action, we'll contact the original reporter to let them know what action (if any) we'll be taking. We'll take into account feedback from the reporter on the appropriateness of our response, but we don't guarantee we'll act on it. Acknowledgement --------------- This CoC is modified from the one by `BeeWare`_, which in turn refers to the `Contributor Covenant`_ and the `Django`_ project. .. _BeeWare: https://beeware.org/community/behavior/code-of-conduct/ .. _Contributor Covenant: https://www.contributor-covenant.org/version/1/3/0/code-of-conduct/ .. _Django: https://www.djangoproject.com/conduct/reporting/ .. raw:: html ================================================ FILE: docs/source/community.rst ================================================ Community ========= Intake is used and developed by individuals at a variety of institutions. It is open source (`license `_) and sits within the broader Python numeric ecosystem commonly referred to as PyData or SciPy. Discussion ---------- Conversation happens in the following places: 1. **Usage questions** are directed to `Stack Overflow with the #intake tag`_. Intake developers monitor this tag. 2. **Bug reports and feature requests** are managed on the `GitHub issue tracker`_. Individual intake plugins are managed in separate repositories each with its own issue tracker. Please consult the :doc:`plugin-directory` for a list of available plugins. 3. **Chat** occurs on at `gitter.im/ContinuumIO/intake `_. Note that because gitter chat is not searchable by future users we discourage usage questions and bug reports on gitter and instead ask people to use Stack Overflow or GitHub. 4. **Monthly community meeting** happens the first Thursday of the month at 9:00 US Central Time. See ``_, with a reminder sent out on the gitter channel. Strictly informal chatter. .. _`Stack Overflow with the #intake tag`: https://stackoverflow.com/questions/tagged/intake .. _`GitHub issue tracker`: https://github.com/intake/intake/issues/ Asking for help --------------- We welcome usage questions and bug reports from all users, even those who are new to using the project. There are a few things you can do to improve the likelihood of quickly getting a good answer. 1. **Ask questions in the right place**: We strongly prefer the use of Stack Overflow or GitHub issues over Gitter chat. GitHub and Stack Overflow are more easily searchable by future users, and therefore is more efficient for everyone's time. Gitter chat is strictly reserved for developer and community discussion. If you have a general question about how something should work or want best practices then use Stack Overflow. If you think you have found a bug then use GitHub 2. **Ask only in one place**: Please restrict yourself to posting your question in only one place (likely Stack Overflow or GitHub) and don't post in both 3. **Create a minimal example**: It is ideal to create `minimal, complete, verifiable examples `_. This significantly reduces the time that answerers spend understanding your situation, resulting in higher quality answers more quickly. .. raw:: html ================================================ FILE: docs/source/conf.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # # intake documentation build configuration file, created by # sphinx-quickstart on Jan 8 09:15:00 2018. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys sys.path.insert(0, os.path.abspath("../..")) import intake # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ "sphinx.ext.intersphinx", "sphinx.ext.autodoc", "sphinx.ext.autosummary", "numpydoc", ] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = ".rst" # The master toctree document. master_doc = "index" # General information about the project. project = "intake" copyright = "2018, Anaconda" author = "Anaconda" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = intake.__version__.split("+")[0] # The full version, including alpha/beta/rc tags. release = intake.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = ["**.ipynb_checkpoints"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = "sphinx_rtd_theme" html_favicon = "_static/images/favicon.ico" # on_rtd is whether we are on readthedocs.org on_rtd = os.environ.get("READTHEDOCS", None) == "True" if not on_rtd: # only import and set the theme if we're building docs locally # otherwise, readthedocs.org uses their theme by default, so no need to specify it import sphinx_rtd_theme html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Default title is " v documentation" html_title = "Intake documentation" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] html_css_files = [ "css/custom.css", ] # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = "intakedoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, "intake.tex", "Intake Documentation", "Anaconda", "manual"), ] # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [(master_doc, "intake", "Intake Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ( master_doc, "intake", "Intake Documentation", author, "intake", "Fast data ingestion for Python.", "Miscellaneous", ), ] # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {"python": ("https://docs.python.org/", None)} # Config numpydoc numpydoc_show_class_members = False numpydoc_show_inherited_class_members = False numpydoc_class_members_toctree = False ================================================ FILE: docs/source/contributing.rst ================================================ Contributor guide ================= ``intake`` is an open-source project (see the LICENSE). We welcome contributions from the public, for code, including fixes and new features, documentation and anything else that will help make this repo better. Even posting issues can be very useful, as they will help others to make the necessary changes to alleviate the issue. Development process ------------------- Development of ``intake`` happens on `github`_. There you will find options for creating issues and commenting on existing issues and pull-requests (PRs). You may wish to "watch" the repo, to be notified of changes as they occur. You must have an account on github to be able to interact here, but this is free. By default, you will be notified of changes (e.g., new comments) on any issue or PR you have interacted with. In order to propose changes to the repo itself, you will need to create a PR. This is done by following these steps: 1. clone the repo. There are many ways to do this, but most common is the following command, which will create a local directory ``intake/`` containing the code, metadata, docs and version control information. .. code-block:: shell $ git clone https://github.com/intake/intake 2. create a fork or the repo using the github web interface. Your fork will probably live in your private github namespace. Set this as a remote inside your local copy of the repo .. code-block:: shell $ git remote add fork https://github.com//intake 3. make changes locally in a new branch. First you create the branch, and then add commits to that branch. Here are suggested ways to do this. Note that git is _very_ flexible and there are many ways to achieve each step. .. code-block:: shell $ git checkout -b $ git commit -a 4. When your branch is an a suitable state, `push` your work to your branch. github will prompt you with a URL to create the PR, or navigate to your fork and branch in the web interface to create the PR there. .. code-block:: shell $ git push fork 5. After review from a maintainer, you may wish to push more commits to your branch as required, and your PR may be accepted ("merged") or rejected ("closed"). .. _github: https://github.com/intake/intake Guidelines ---------- To make contributing as smooth as possible, we recommend the following. 1. Always follow the project's Code of Conduct when interacting with other humans. 2. Please describe as clearly as possible what your intent is. In the case of issues, this might include pasting the whole traceback your have seen following an error, listing the versions of ``intake`` and its dependencies that you have installed, describing the circumstances when you saw a problem or would like better behaviour. Ideally, you would include code that allows maintainers to fully reproduce your steps. 3. When submitting changes, make sure that you describe what the changes achieve and how. Ideally, all code should be covered by tests included in the same PR, and that run to completion as part of CI (see below). 4. New functions and classes should include reasonable `style`, e.g., appropriate labels and hierarchy, indentation and other code formatting matching the rest of the docs, and docstrings and comments as appropriate. A "precommit" set of linters is available to run against your code, and runs as part of CI to enforce a minimal set of style rules. To run these locally on every commit, you can run this in the repo root: .. code-block:: shell $ pre-commit install 5. Additions to the prose documentation (under docs/source/) should be included for new or altered features. After the initial full release, we will be maintaining a changelog. Testing ------- This repo uses ``pytest`` for testing. You can install test dependencies, for example with this command run in the repo root. There are many optional dependencies for specific tests, and we recommend that you use ``pytest.importorskip`` to tests that need these or additional packages, so that they will not fail for developers without those dependencies. **Do**, however, edit one or more files in scripts/ci/, to ensure that added tests will execute in at least one of the CI runs. The easiest way to boostrap a development environment in order to run tests as they will in CI is to use conda-env, e.g.: .. code-block:: shell $ conda env create -y -f scripts/ci/environment-py313.yml $ conda activate test_env To run the tests: .. code-block:: shell $ pytest -v Note that ensuring coverage is optional, but recommended. Adding docs ----------- Docstrings, prose text and examples/tutorials are eagerly accepted! We, as coders, often are late to fully document our work, and all contributions are welcome. Separate instructions can be found in the docs/README.md file. In addition, full notebook examples may be added in the examples/ directory, but you should be sure to add instructions on the appropriate environment or other preparation required to run them. .. raw:: html ================================================ FILE: docs/source/data-packages.rst ================================================ Making Data Packages ==================== Intake can used to create :term:`Data packages`, so that you can easily distribute your catalogs - others can just "install data". Since you may also want to distribute custom catalogues, perhaps with visualisations, and driver code, packaging these things together is a great convenience. Indeed, packaging gives you the opportunity to version-tag your distribution and to declare the requirements needed to be able to use the data. This is a common pattern for distributing code for python and other languages, but not commonly seen for data artifacts. The current version of Intake allows making data packages using standard python tools (to be installed, for example, using ``pip``). The previous, now deprecated, technique is still described below, under :ref:`condapack` and is specific to the `conda` packaging system. Python packaging solution ------------------------- Intake allows you to register data artifacts (catalogs and data sources) in the metadata of a python package. This means, that when you install that package, intake will automatically know of the registered items, and they will appear within the "builtin" catalog ``intake.cat``. Here we assume that you understand what is meant by a python package (i.e., a folder containing ``__init__.py`` and other code, config and data files). Furthermore, you should familiarise yourself with what is required for bundling such a package into a *distributable* package (one with a ``setup.py``) by reading the `official packaging documentation`_ .. _official packaging documentation: https://packaging.python.org/tutorials/packaging-projects/ The `intake examples`_ contains a full tutorial for packaging and distributing intake data and/or catalogs for ``pip`` and ``conda``, see the directory "data_package/". .. _intake examples: https://github.com/intake/intake-examples Entry points definition ''''''''''''''''''''''' Intake uses the concept of `entry points` to define the entries that are defined by a given package. Entry points provide a mechanism to register metadata about a package at install time, so that it can easily be found by other packages such as Intake. Entry points was originally a `separate package`_, but is included in the standard library as of python 3.8 (you will not need to install it, as Intake requires it). All you need to do to register an entry in ``intake.cat`` is: - define a data source somewhere in your package. This object can be of any ttype that makes sense to Intake, including Catalogs, and sources that have drivers defined in the very same package. Obviously, if you can have catalogs, you can populate these however you wish, including with more catalogs. You need not be restricted to simply loading in YAML files. - include a block in your call to ``setp`` in ``setup.py`` with code something like .. code-block:: python entry_points={ 'intake.catalogs': [ 'sea_cat = intake_example_package:cat', 'sea_data = intake_example_package:data' ] } Here only the lines with "sea_cat" and "sea_data" are specific to the example package, the rest is required boilerplate. Each of those two lines defines a name for the data entry (before the "=" sign) and the location to load from, in module:object format. - install the package using ``pip``, ``python setup.py``, or package it for ``conda`` .. _separate package: https://github.com/takluyver/entrypoints Intake's process '''''''''''''''' When Intake is imported, it investigates all registered entry points with the ``"intake.catalogs"`` group. It will go through and assign each name to the given location of the final object. In the above example, ``intake.cat.sea_cat`` would be associated with the ``cat`` object in the ``intake_example_package`` package, and so on. Note that Intake does **not** immediately import the given package or module, because imports can sometimes be expensive, and if you have a lot of data packages, it might cause a slow-down every time that Intake is imported. Instead, a placeholder entry is created, and whenever the entry is accessed, that's when the particular package will be imported. .. code-block:: python In [1]: import intake In [2]: intake.cat.sea_cat # does not import yet Out[2]: In [3]: cat = intake.cat.sea_cat() # imports now In [4]: cat # this data source happens to be a catalog Out[4]: (note here the parentheses - this explicitly initialises the source, and normally you don't have to do this) .. _condapack: Pure conda solution ------------------- This packaging method is deprecated, but still available. Combined with the `Conda Package Manger `_, Intake makes it possible to create :term:`Data packages` which can be installed and upgraded just like software packages. This offers several advantages: * Distributing Catalogs and Drivers becomes as easy as ``conda install`` * Data packages can be versioned, improving reproducibility in some cases * Data packages can depend on the libraries required for reading * Data packages can be self-describing using Intake catalog files * Applications that need certain Catalogs can include data packages in their dependency list In this tutorial, we give a walk-through to enable you to distribute any Catalogs to others, so that they can access the data using Intake without worrying about where it resides or how it should be loaded. Implementation '''''''''''''' The function ``intake.catalog.default.load_combo_catalog`` searches for YAML catalog files in a number of place at import. All entries in these catalogs are flattened and placed in the "builtin" ``intake.cat``. The places searched are: * a platform-specific user directory as given by the `appdirs`_ package * in the environment's "/share/intake" data directory, where the location of the current environment is found from virtualenv or conda environment variables * in directories listed in the "INTAKE_PATH" environment variable or "catalog_path" config parameter .. _appdirs: https://github.com/ActiveState/appdirs Defining a Package '''''''''''''''''' The steps involved in creating a data package are: 1. Identifying a dataset, which can be accessed via a URL or included directly as one or more files in the package. 2. Creating a package containing: * an intake catalog file * a ``meta.yaml`` file (description of the data, version, requirements, etc.) * a script to copy the data 3. Building the package using the command ``conda build``. 4. Uploading the package to a package repository such as `Anaconda Cloud `_ or your own private repository. Data packages are standard conda packages that install an Intake catalog file into the user's conda environment (``$CONDA_PREFIX/share/intake``). A data package does not necessarily imply there are data files inside the package. A data package could describe remote data sources (such as files in S3) and take up very little space on disk. These packages are considered ``noarch`` packages, so that one package can be installed on any platform, with any version of Python (or no Python at all). The easiest way to create such a package is using a `conda build `_ recipe. Conda-build recipes are stored in a directory that contains a files like: * ``meta.yaml`` - description of package metadata * ``build.sh`` - script for building/installing package contents (on Linux/macOS) * other files needed by the package (catalog files and data files for data packages) An example that packages up data from a Github repository would look like this: .. code-block:: yaml # meta.yaml package: version: '1.0.0' name: 'data-us-states' source: git_rev: v1.0.0 git_url: https://github.com/CivilServiceUSA/us-states build: number: 0 noarch: generic requirements: run: - intake build: [] about: description: Data about US states from CivilServices (https://civil.services/) license: MIT license_family: MIT summary: Data about US states from CivilServices The key parts of a data package recipe (different from typical conda recipes) is the ``build`` section: .. code-block:: yaml build: number: 0 noarch: generic This will create a package that can be installed on any platform, regardless of the platform where the package is built. If you need to rebuild a package, the build number can be incremented to ensure users get the latest version when they conda update. The corresponding ``build.sh`` file in the recipe looks like this: .. code-block:: bash #!/bin/bash mkdir -p $CONDA_PREFIX/share/intake/civilservices cp $SRC_DIR/data/states.csv $PREFIX/share/intake/civilservices cp $RECIPE_DIR/us_states.yaml $PREFIX/share/intake/ The ``$SRC_DIR`` variable refers to any source tree checked out (from Github or other service), and the ``$RECIPE_DIR`` refers to the directory where the ``meta.yaml`` is located. Finishing out this example, the catalog file for this data source looks like this: .. code-block:: yaml sources: states: description: US state information from [CivilServices](https://civil.services/) driver: csv args: urlpath: '{{ CATALOG_DIR }}/civilservices/states.csv' metadata: origin_url: 'https://github.com/CivilServiceUSA/us-states/blob/v1.0.0/data/states.csv' The ``{{ CATALOG_DIR }}`` Jinja2 variable is used to construct a path relative to where the catalog file was installed. To build the package, you must have conda-build installed: .. code-block:: bash conda install conda-build Building the package requires no special arguments: .. code-block:: bash conda build my_recipe_dir Conda-build will display the path of the built package, which you will need to upload it. If you want your data package to be publicly available on `Anaconda Cloud `_, you can install the anaconda-client utility: .. code-block:: bash conda install anaconda-client Then you can register your Anaconda Cloud credentials and upload the package: .. code-block:: bash anaconda login anaconda upload /Users/intake_user/anaconda/conda-bld/noarch/data-us-states-1.0.0-0.tar.bz2 Best Practices -------------- Versioning '''''''''' * Versions for data packages should be used to indicate changes in the data values or schema. This allows applications to easily pin to the specific data version they depend on. * Putting data files into a package ensures reproducibility by allowing a version number to be associated with files on disk. This can consume quite a bit of disk space for the user, however. Large data files are not generally included in pip or conda packages so, if possible, you should reference the data assets in an external place where they can be loaded. Packaging ''''''''' * Packages that refer to remote data sources (such as databases and REST APIs) need to think about authentication. Do not include authentication credentials inside a data package. They should be obtained from the environment. * Data packages should depend on the Intake plugins required to read the data, or Intake itself. * You may well want to break any driver code code out into a separate package so that it can be updated independent of the data. The data package would then depend on the driver package. Nested catalogs ''''''''''''''' As noted above, entries will appear in the users' builtin catalog as ``intake.cat.*``. In the case that the catalog has multiple entries, it may be desirable to put the entries below a namespace as ``intake.cat.data_package.*``. This can be achieved by having one catalog containing the (several) data sources, with only a single top-level entry pointing to it. This catalog could be defined in a YAML file, created using any other catalog driver, or constructed in the code, e.g.: .. code-block:: python from intake.catalog import Catalog from intake.catalog.local import LocalCatalogEntry as Entry cat = intake.catalog.Catalog() cat._entries = {name: Entry(name, descr, driver='package.module.driver', args={"urlpath": url}) for name, url in my_input_list} If your package contains many sources of different types, you may even nest the catalogs, i.e., have a top-level whose contents are also catalogs. .. code-block:: python e = Entry('first_cat', 'sample', driver='catalog') e._default_source = cat top_level = Catalog() top_level._entries = {'fist_cat': e, ...} where your entry point might look something like: ``"my_cat = my_package:top_level"``. You could achieve the same with multiple YAML files. .. raw:: html ================================================ FILE: docs/source/deployments.rst ================================================ Deployment Scenarios -------------------- In the following sections, we will describe some of the ways in which Intake is used in real production systems. These go well beyond the typical YAML files presented in the quickstart and examples sections, which are necessarily short and simple, and do not demonstrate the full power of Intake. Sharing YAML files ~~~~~~~~~~~~~~~~~~ This is the simplest scenario, and amply described in these documents. The primary advantage is simplicity: it is enough to put a file in an accessible place (even a gist or repo), in order for someone else to be able to discover and load that data. Furthermore, such files can easily refer to one-another, to build up a full tree of data assets with minimum pain Since YAML files are text, this also lends itself to working well with version control systems. Furthermore, all sources can describe themselves as YAML, and the ``export`` and ``upload`` commands can produce an efficient format (possibly remote) together with YAML definition in a single step. Pangeo ~~~~~~ The `Pangeo`_ collaboration uses Intake to catalog their data holdings, which are generally in various forms of netCDF-compliant formats, massive multi-dimensional arrays with data relating to earth and climate science and meteorology. On their cloud-based platform, containers start up jupyter-lab sessions which have Intake installed, and therefore can simply pick and load the data that each researcher needs - often requiring large Dask clusters to actually do the processing. A `static `__ rendering of the catalog contents is available, so that users can browse the holdings without even starting a python session. This rendering is produced by CI on the `repo `__ whenever new definitions are added, and it also checks (using Intake) that each definition is indeed loadable. Pangeo also developed intake-stac, which can talk to STAC servers to make real-time queries and parse the results into Intake data sources. This is a standard for spaceo-temporal data assets, and indexes massive amounts of cloud-stored data. .. _Pangeo: http://pangeo.io/ Anaconda Enterprise ~~~~~~~~~~~~~~~~~~~ Intake will be the basis of the data access and cataloging service within `Anaconda Enterprise`_, running as a micro-service in a container, and offering data source definitions to users. The access control, who gets to see which data-set, and serving of credentials to be able to read from the various data storage services, will all be handled by the platform and be fully configurable by admins. .. _Anaconda Enterprise: https://www.anaconda.com/enterprise/ National Center for Atmospheric Research ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ NCAR has developed `intake-esm`_, a mechanism for creating file-based Intake catalogs for climate data from project efforts such as the `Coupled Model Intercomparison Project (CMIP)`_ and the `Community Earth System Model (CESM) Large Ensemble Project`_. These projects produce a huge of amount climate data persisted on tape, disk storage components across multiple (of the order ~300,000) netCDF files. Finding, investigating, loading these files into data array containers such as `xarray` can be a daunting task due to the large number of files a user may be interested in. ``Intake-esm`` addresses this issue in three steps: - `Datasets Catalog Curation`_ in form of YAML files. These YAML files provide information about data locations, access pattern, directory structure, etc. ``intake-esm`` uses these YAML files in conjunction with file name templates to construct a local database. Each row in this database consists of a set of metadata such as ``experiment``, ``modeling realm``, ``frequency`` corresponding to data contained in one netCDF file. .. code-block:: python cat = intake.open_esm_metadatastore(catalog_input_definition="GLADE-CMIP5") - Search and Discovery: once the database is built, ``intake-esm`` can be used for searching and discovering of climate datasets by eliminating the need for the user to know specific locations (file path) of their data set of interest: .. code-block:: python sub_cat = cat.search(variable=['hfls'], frequency='mon', modeling_realm='atmos', institute=['CCCma', 'CNRM-CERFACS']) - Access: when the user is satisfied with the results of their query, they can ask ``intake-esm`` to load the actual netCDF files into xarray datasets: .. code-block:: python dsets = cat.to_xarray(decode_times=True, chunks={'time': 50}) .. _intake-esm: https://github.com/NCAR/intake-esm .. _Datasets Catalog Curation: https://github.com/NCAR/intake-esm-datastore .. _Coupled Model Intercomparison Project (CMIP): https://www.wcrp-climate.org/wgcm-cmip .. _Community Earth System Model (CESM) Large Ensemble Project: http://www.cesm.ucar.edu/projects/community-projects/LENS/ Brookhaven Archive ~~~~~~~~~~~~~~~~~~ The `Bluesky`_ project uses Intake to dynamically query a MongoDB instance, which holds the details of experimental and simulation data catalogs, to return a custom Catalog for every query. Data-sets can then be loaded into python, or the original raw data can be accessed ... .. _Bluesky: https://github.com/bluesky/intake-bluesky Zillow ~~~~~~ Zillow is developing Intake to meet the needs of their datalake access layer (DAL), to encapsulate the highly hierarchical nature of their data. Of particular importance, is the ability to provide different version (testing/production, and different storage formats) of the same logical dataset, depending on whether it is being read on a laptop versus the production infrastructure ... .. raw:: html ================================================ FILE: docs/source/examples.rst ================================================ Examples ======== Here we list links to notebooks and other code demonstrating the use of Intake in various scenarios. The first section is of general interest to various users, and the sections that follow tend to be more specific about particular features and workflows. Many of the entries here include a link to Binder, which a service that lest you execute code live in a notebook environment. This is a great way to experience using Intake. It can take a while, sometimes, for Binder to come up; please have patience. See also the `examples`_ repository, containing data-sets which can be built and installed as conda packages. .. _examples: https://github.com/intake/intake-examples/ General ------- - Basic Data scientist workflow: using Intake [`Static `__] [`Executable `__]. - Workflow for creating catalogs: a Data Engineer's approach to Intake [`Static `__] [`Executable `__] Developer --------- Tutorials delving deeper into the Internals of Intake, for those who wish to contribute - How you would go about writing a new plugin [`Static `__] [`Executable `__] Features -------- More specific examples of Intake functionality - Caching: - New-style data package creation [`Static `__] - Using automatically cached data-files [`Static `__] [`Executable `__] - Earth science demonstration of cached dataset [`Static `__] [`Executable `__] - File-name pattern parsing: - Satellite imagery, science workflow [`Static `__] [`Executable `__] - How to set up pattern parsing [`Static `__] [`Executable `__] - Custom catalogs: - A custom intake plugin that adapts DCAT catalogs [`Static `__] [`Executable `__] Data ---- - `Anaconda package data`_, originally announced in `this blog`_ - `Planet Four Catalog`_, originally from https://www.planetfour.org/results - The official Intake `examples`_ .. _Anaconda package data: https://github.com/ContinuumIO/anaconda-package-data .. _this blog: https://www.anaconda.com/announcing-public-anaconda-package-download-data/ .. _Planet Four Catalog: https://github.com/michaelaye/p4catalog Blogs ----- These are Intake-related articles that may be of interest. - `Discovering and Exploring Data in a Graphical Interface`_ - `Taking the Pain out of Data Access`_ - `Caching Data on First Read Makes Future Analysis Faster`_ - `Parsing Data from Filenames and Paths`_ - `Intake for cataloguing Spark`_ - `Intake released on Conda-Forge`_ .. _Discovering and Exploring Data in a Graphical Interface: https://www.anaconda.com/intake-discovering-and-exploring-data-in-a-graphical-interface/ .. _Intake for cataloguing Spark: https://www.anaconda.com/intake-for-cataloging-spark/ .. _Taking the Pain out of Data Access: https://www.anaconda.com/intake-taking-the-pain-out-of-data-access/ .. _Caching Data on First Read Makes Future Analysis Faster: https://www.anaconda.com/intake-caching-data-on-first-read-makes-future-analysis-faster/ .. _Parsing Data from Filenames and Paths: https://www.anaconda.com/intake-parsing-data-from-filenames-and-paths/ .. _Intake released on Conda-Forge: https://www.anaconda.com/intake-released-on-conda-forge/ Talks ----- - `__init__ podcast interview (May 2019)`_ - `AnacondaCon (March 2019)`_ - `PyData DC (November 2018)`_ - `PyData NYC (October 2018)`_ - `ESIP tech dive (November 2018)`_ .. _\__init__ podcast interview (May 2019): https://www.pythonpodcast.com/intake-data-catalog-episode-213/ .. _ESIP tech dive (November 2018): https://www.youtube.com/watch?v=PSD7r3JFml0&feature=youtu.be .. _PyData DC (November 2018): https://www.youtube.com/watch?v=OvZFtePHKXw .. _PyData NYC (October 2018): https://www.youtube.com/watch?v=pjkMmJQfTb8 .. _AnacondaCon (March 2019): https://www.youtube.com/watch?v=oyZJrROQzUs News ---- - See out `Wiki`_ page .. _Wiki: https://github.com/intake/intake/wiki/Community-News .. raw:: html ================================================ FILE: docs/source/glossary.rst ================================================ Glossary ======== .. glossary:: Argument One of a set of values passed to a function or class. In the Intake sense, this usually is the set of key-value pairs defined in the "args" section of a source definition; unless the user overrides, these will be used for instantiating the source. Cache Local copies of remote files. Intake allows for download-on-first-use for data-sources, so that subsequent access is much faster. The format of the files is unchanged in this case, but may be decompressed. Catalog An inventory of entries, each of which corresponds to a specific :term:`Data-set`. Within these docs, a catalog is most commonly defined in a :term:`YAML` file, for simplicity, but there are other possibilities, such as connecting to an Intake server or another third-party data service, like a SQL database. Thus, catalogs form a hierarchy: any catalog can contain other, nested catalogs. Catalog file A :term:`YAML` specification file which contains a list of named entries describing how to load data sources. :doc:`catalog`. Conda A package and environment management package for the python ecosystem, see the `conda website`_. Conda ensures dependencies and correct versions are installed for you, provides precompiled, binary-compatible software, and extends to many languages beyond python, such as R, javascript and C. Conda package A single installable item which the :term:`Conda` application can install. A package may include a :term:`Catalog`, data-files and maybe some additional code. It will also include a specification of the dependencies that it requires (e.g., Intake and any additional :term:`Driver`), so that Conda can install those automatically. Packages can be created locally, or can be found on `anaconda.org`_ or other package repositories. Container One of the supported data formats. Each :term:`Driver` outputs its data in one of these. The containers correspond to familiar data structures for end-analysis, such as list-of-dicts, Numpy nd-array or Pandas data-frame. Data-set A specific assemblage of data. The type of data (tabular, multi-dimensional or something else) and the format (file type, data service type) are all attributes of the data-set. In addition, in the context of Intake, data-sets are usually entries within a :term:`Catalog` with additional descriptive text and metadata and a specification of *how* to load the data. Data Source An Intake specification for a specific :term:`Data-set`. In most cases, the two terms are synonymous. Data User A person who uses data to produce models and other inferences/conclusions. This person generally uses standard python analysis packages like Numpy, Pandas, SKLearn and may produce graphical output. They will want to be able to find the right data for a given job, and for the data to be available in a standard format as quickly and easily as possible. In many organisations, the appropriate job title may be Data Scientist, but research scientists and BI/analysts also fit this description. Data packages Data packages are standard conda packages that install an Intake catalog file into the user’s conda environment ($CONDA_PREFIX/share/intake). A data package does not necessarily imply there are data files inside the package. A data package could describe remote data sources (such as files in S3) and take up very little space on disk. Data Provider A person whose main objective is to curate data sources, get them into appropriate formats, describe the contents, and disseminate the data to those that need to use them. Such a person may care about the specifics of the storage format and backing store, the right number of fields to keep and removing bad data. They may have a good idea of the best way to visualise any give data-set. In an organisation, this job may be known as Data Engineer, but it could as easily be done by a member of the IT team. These people are the most likely to author :term:`Catalogs`. Developer A person who writes or fixes code. In the context of Intake, a developer may make new format :term:`Drivers`, create authentication systems or add functionality to Intake itself. They can take existing code for loading data in other projects, and use Intake to add extra functionality to it, for instance, remote data access, parallel processing, or file-name parsing. Driver The thing that does the work of reading the data for a catalog entry is known as a driver, often referred to using a simple name such as "csv". Intake has a plugin architecture, and new drivers can be created or installed, and specific catalogs/data-sets may require particular drivers for their contained data-sets. If installed as :term:`Conda` packages, then these requirements will be automatically installed for you. The driver's output will be a :term:`Container`, and often the code is a simpler layer over existing functionality in a third-party package. GUI A Graphical User Interface. Intake comes with a GUI for finding and selecting data-sets, see :doc:`gui`. IT The Information Technology team for an organisation. Such a team may have control of the computing infrastructure and security (sys-ops), and may well act as gate-keepers when exposing data for use by other colleagues. Commonly, IT has stronger policy enforcement requirements that other groups, for instance requiring all data-set copy actions to be logged centrally. Persist A process of making a local version of a data-source. One canonical format is used for each of the container types, optimised for quick and parallel access. This is particularly useful if the data takes a long time to acquire, perhaps because it is the result of a complex query on a remote service. The resultant output can be set to expire and be automatically refreshed, see :doc:`persisting`. Not to be confused with the :term:`cache`. Plugin Modular extra functionality for Intake, provided by a package that is installed separately. The most common type of plugin will be for a :term:`Driver` to load some particular data format; but other parts of Intake are pluggable, such as authentication mechanisms for the server. Server A remote source for Intake catalogs. The server will provide data source specifications (i.e., a remote :term:`Catalog`), and may also provide the raw data, in situations where the client is not able or not allowed to access it directly. As such, the server can act as a gatekeeper of the data for security and monitoring purposes. The implementation of the server in Intake is accessible as the ``intake-server`` command, and acts as a reference: other implementations can easily be created for specific circumstances. TTL Time-to-live, how long before the given entity is considered to have expired. Usually in seconds. User Parameter A data source definition can contain a "parameters" section, which can act as explicit decision indicators for the user, or as validation and type coersion for the definition's :term:`Argument` s. See :ref:`paramdefs`. YAML A text-based format for expressing data with a dictionary (key-value) and list structure, with a limited number of data-types, such as strings and numbers. YAML uses indentations to nest objects, making it easy to read and write for humans, compared to JSON. Intake's catalogs and config are usually expressed in YAML files. .. _conda website: https://conda.io/docs/ .. _anaconda.org: http://anaconda.org .. raw:: html ================================================ FILE: docs/source/gui.rst ================================================ GUI === Using the GUI ------------- **Note**: the GUI requires ``panel`` and ``bokeh`` to be available in the current environment. The Intake top-level singleton ``intake.gui`` gives access to a graphical data browser within the Jupyter notebook. To expose it, simply enter it into a code cell (Jupyter automatically display the last object in a code cell). .. image:: _static/images/gui_builtin.png New instances of the GUI are also available by instantiating ``intake.interface.gui.GUI``, where you can specify a list of catalogs to initially include. The GUI contains three main areas: - a **list of catalogs**. The "builtin" catalog, displayed by default, includes data-sets installed in the system, the same as ``intake.cat``. - a **list of sources** within the currently selected catalog. - a **description** of the currently selected source. Catalogs -------- Selecting a catalog from the list will display nested catalogs below the parent and display source entries from the catalog in the **list of sources**. Below the **lists of catalogs** is a row of buttons that are used for adding, removing and searching-within catalogs: - **Add**: opens a sub-panel for adding catalogs to the interface, by either browsing for a local YAML file or by entering a URL for a catalog, which can be a remote file or Intake server - **Remove**: deletes the currently selected catalog from the list - **Search**: opens a sub-panel for finding entries in the currently selected catalog (and its sub-catalogs) Add Catalogs ~~~~~~~~~~~~ The Add button (+) exposes a sub-panel with two main ways to add catalogs to the interface: .. image:: _static/images/gui_add.png This panel has a tab to load files from **local**; from that you can navigate around the filesystem using the arrow or by editing the path directly. Use the home button to get back to the starting place. Select the catalog file you need. Use the "Add Catalog" button to add the catalog to the list above. .. image:: _static/images/gui_add_local.png Another tab loads a catalog from **remote**. Any URL is valid here, including cloud locations, ``"gcs://bucket/..."``, and intake servers, ``"intake://server:port"``. Without a protocol specifier, this can be a local path. Again, use the "Add Catalog" button to add the catalog to the list above. .. image:: _static/images/gui_add_remote.png Finally, you can add catalogs to the interface in code, using the ``.add()`` method, which can take filenames, remote URLs or existing ``Catalog`` instances. Remove Catalogs ~~~~~~~~~~~~~~~ The Remove button (-) deletes the currently selected catalog from the list. It is important to note that this action does not have any impact on files, it only affects what shows up in the list. .. image:: _static/images/gui_remove.png Search ~~~~~~ The sub-panel opened by the Search button (🔍) allows the user to search within the selected catalog .. image:: _static/images/gui_search.png From the Search sub-panel the user enters for free-form text. Since some catalogs contain nested sub-catalogs, the Depth selector allows the search to be limited to the stated number of nesting levels. This may be necessary, since, in theory, catalogs can contain circular references, and therefore allow for infinite recursion. .. image:: _static/images/gui_search_inputs.png Upon execution of the search, the currently selected catalog will be searched. Entries will be considered to match if any of the entered words is found in the description of the entry (this is case-insensitive). If any matches are found, a new entry will be made in the catalog list, with the suffix "_search". .. image:: _static/images/gui_search_cat.png Sources ------- Selecting a source from the list updates the description text on the left-side of the gui. Below the **list of sources** is a row of buttons for inspecting the selected data source: - **Plot**: opens a sub-panel for viewing the pre-defined (specified in the yaml) plots for the selected source. Plot ~~~~ The Plot button (📊) opens a sub-panel with an area for viewing pre-defined plots. .. image:: _static/images/gui_plot.png These plots are specified in the catalog yaml and that yaml can be displayed by checking the box next to "show yaml". .. image:: _static/images/gui_plot_yaml.png The holoviews object can be retrieved from the gui using ``intake.interface.source.plot.pane.object``, and you can then use it in Python or export it to a file. Interactive Visualization ''''''''''''''''''''''''' If you have installed the optional extra packages `dfviz`_ and `xrviz`_, you can interactively plot your dataframe or array data, respectively. .. image:: _static/images/custom_button.png .. _dfviz: https://dfviz.readthedocs.io/ .. _xrviz: https://xrviz.readthedocs.io/ The button "customize" will be available for data sources of the appropriate type. Click this to open the interactive interface. If you have not selected a predefined plot (or there are none), then the interface will start without any prefilled values, but if you do first select a plot, then the interface will have its options pre-filled from the options For specific instructions on how to use the interfaces (which can also be used independently of the Intake GUI), please navigate to the linked documentation. Note that the final parameters that are sent to ``hvPlot`` to produce the output each time a plot if updated, are explicitly available in YAML format, so that you can save the state as a "predefined plot" in the catalog. The same set of parameters can also be used in code, with ``datasource.plot(...)``. .. image:: _static/images/YAMLtab.png Using the Selection ------------------- Once catalogs are loaded and the desired sources has been identified and selected, the selected sources will be available at the ``.sources`` attribute (``intake.gui.sources``). Each source entry has informational methods available and can be opened as a data source, as with any catalog entry: .. code-block:: python In [ ]: source_entry = intake.gui.sources[0] source_entry Out : name: sea_ice_origin container: dataframe plugin: ['csv'] description: Arctic/Antarctic Sea Ice direct_access: forbid user_parameters: [] metadata: args: urlpath: https://timeseries.weebly.com/uploads/2/1/0/8/21086414/sea_ice.csv In [ ]: data_source = source_entry() # may specify parameters here data_source.read() Out : < some data > In [ ]: source_entry.plot() # or skip data source step Out : < graphics> .. raw:: html ================================================ FILE: docs/source/guide.rst ================================================ User Guide ---------- More detailed information about specific parts of Intake, such as how to author catalogs, how to use the graphical interface, plotting, etc. .. toctree:: :maxdepth: 1 gui.rst catalog.rst tools.rst persisting.rst plotting.rst plugin-directory.rst transforms.rst .. raw:: html ================================================ FILE: docs/source/index.rst ================================================ .. raw:: html Intake Logo .. _take2: Intake Take2 ============ *Taking the pain out of data access and distribution* Intake is an open-source package to: - describe your data declaratively - gather data sets into catalogs - search catalogs and services to find the right data you need - load, transform and output data in many formats - work with third party remote storage and compute platforms This is the start of the documentation for the alpha version of Intake: Take2, a rewrite of Intake (henceforth referred to as legacy or V1). We will give an introduction to the ideas of Intake in general and specifically how to use this release. Go directly to the walkthrough and examples, or read the following motivation and declarations of scope. .. note:: We are making Take2 as a full release. It is still "beta" in the sense that we will be adding many data types, readers and transformers, and are prepared to revisit the API in general. The reason not to use a pre-release or RC, is that users never see these. .. warning:: Looking for :ref:`v1` documentation? You may have just installed Intake and found that Take2 broke things for you, so you might wish to pin to an older version. Or stick around and find out why you might wish to update your code. All old "sources", whether still working or not, should be considered deprecated. .. toctree:: :maxdepth: 2 scope2.rst user2.rst walkthrough2.rst tour2.rst api2.rst code-of-conduct.rst contributing.rst index_v1.rst Install ------- To install Intake Take2: .. code-block:: bash pip install -c conda-forge intake or pip install intake Please leave issues and discussions on our `repo page`_. .. _repo page: https://github.com/intake/intake Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` These docs pages collect anonymous tracking data using goatcounter, and the dashboard is available to the public: https://intake.goatcounter.com/ . .. raw:: html ================================================ FILE: docs/source/index_v1.rst ================================================ .. raw:: html Intake Logo .. _v1: Intake Legacy ============= *Taking the pain out of data access and distribution* Intake is a lightweight package for finding, investigating, loading and disseminating data. It will appeal to different groups for some of the reasons below, but is useful for all and acts as a common platform that everyone can use to smooth the progression of data from developers and providers to users. .. warning:: This is the Legacy documentation for Intake pre-v2. To install, please pin your versions to "<2". You should expect old catalogs and sources to continue working, but a lot has changed, so we encourage all comers to read the new documentation and adapt their catalogs and code if possible. Looking for :ref:`take2` ? Intake contains the following main components. You *do not* need to use them all! The library is modular, only use the parts you need: * A set of **data loaders** (:term:`Drivers`) with a common interface, so that you can investigate or load anything, from local or remote, with the exact same call, and turning into data structures that you already know how to manipulate, such as arrays and data-frames. * A **Cataloging system** (:term:`Catalogs`) for listing data sources, their metadata and parameters, and referencing which of the Drivers should load each. The catalogs for a hierarchical, searchable structure, which can be backed by files, Intake servers or third-party data services * Sets of **convenience functions** to apply to various data sources, such as data-set persistence, automatic concatenation and metadata inference and the ability to distribute catalogs and data sources using simple packaging abstractions. * A **GUI layer** accessible in the Jupyter notebook or as a standalone webserver, which allows you to find and navigate catalogs, investigate data sources, and plot either predefined visualisations or interactively find the right view yourself * A **client-server protocol** to allow for arbitrary data cataloging services or to serve the data itself, with a pluggable auth model. :term:`Data User` ----------------- .. raw:: html Line graph * Intake loads the data for a range of formats and types (see :ref:`plugin-directory`) into containers you already use, like Pandas dataframes, Python lists, NumPy arrays, and more * Intake loads, then gets out of your way * GUI search and introspect data-sets in :term:`Catalogs`: quickly find what you need to do your work * Install data-sets and automatically get requirements * Leverage cloud resources and distributed computing. See the executable tutorial: .. image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/intake/intake-examples/master?filepath=tutorial%2Fdata_scientist.ipynb :term:`Data Provider` --------------------- .. raw:: html Grid * Simple spec to define data sources * Single point of truth, no more copy&paste * Distribute data using packages, shared files or a server * Update definitions in-place * Parametrise user options * Make use of additional functionality like filename parsing and caching. See the executable tutorial: .. image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/intake/intake-examples/master?filepath=tutorial%2Fdata_engineer.ipynb :term:`IT` ---------- .. raw:: html FA-terminal * Create catalogs out of established departmental practices * Provide data access credentials via Intake parameters * Use server-client architecture as gatekeeper: * add authentication methods * add monitoring point; track the data-sets being accessed. * Hook Intake into proprietary data access systems. :term:`Developer` ----------------- .. raw:: html Python code * Turn boilerplate code into a reusable :term:`Driver` * Pluggable architecture of Intake allows for many points to add and improve * Open, simple code-base -- come and get involved on `github`_! .. _github: https://github.com/intake/intake See the executable tutorial: .. image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/intake/intake-examples/master?filepath=tutorial%2Fdev.ipynb First steps =========== The :doc:`start` document contains the sections that all users new to Intake should read through. :ref:`usecases` shows specific problems that Intake solves. For a brief demonstration, which you can execute locally, go to :doc:`quickstart`. For a general description of all of the components of Intake and how they fit together, go to :doc:`overview`. Finally, for some notebooks using Intake and articles about Intake, go to :doc:`examples` and `intake-examples`_. These and other documentation pages will make reference to concepts that are defined in the :doc:`glossary`. .. _intake-examples: https://github.com/intake/intake-examples | | .. toctree:: :maxdepth: 1 :hidden: start.rst guide.rst reference.rst roadmap.rst glossary.rst community.rst Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` .. raw:: html ================================================ FILE: docs/source/making-plugins.rst ================================================ Making Drivers ============== The goal of the Intake plugin system is to make it very simple to implement a :term:`Driver` for a new data source, without any special knowledge of Dask or the Intake catalog system. Assumptions ----------- Although Intake is very flexible about data, there are some basic assumptions that a driver must satisfy. Data Model '''''''''' Intake currently supports 3 kinds of containers, represented the most common data models used in Python: * dataframe * ndarray * python (list of Python objects, usually dictionaries) Although a driver can load *any* type of data into any container, and new container types can be added to the list above, it is reasonable to expect that the number of container types remains small. Declaring a container type is only informational for the user when read locally, but streaming of data from a server requires that the container type be known to both server and client. A given driver must only return one kind of container. If a file format (such as HDF5) could reasonably be interpreted as two different data models depending on usage (such as a dataframe or an ndarray), then two different drivers need to be created with different names. If a driver returns the ``python`` container, it should document what Python objects will appear in the list. The source of data should be essentially permanent and immutable. That is, loading the data should not destroy or modify the data, nor should closing the data source destroy the data either. When a data source is serialized and sent to another host, it will need to be reopened at the destination, which may cause queries to be re-executed and files to be reopened. Data sources that treat readers as "consumers" and remove data once read will cause erratic behavior, so Intake is not suitable for accessing things like FIFO message queues. Schema '''''' The schema of a data source is a detailed description of the data, which can be known by loading only metadata or by loading only some small representative portion of the data. It is information to present to the user about the data that they are considering loading, and may be important in the case of server-client communication. In the latter context, the contents of the schema must be serializable by ``msgpack`` (i.e., numbers, strings, lists and dictionaries only). There may be unknown parts of the schema before the whole data is read. drivers may require this unknown information in the `__init__()` method (or the catalog spec), or do some kind of partial data inspection to determine the schema; or more simply, may be given as unknown ``None`` values. Regardless of method used, the time spent figuring out the schema ahead of time should be short and not scale with the size of the data. Typical fields in a schema dictionary are ``npartitions``, ``dtype``, ``shape``, etc., which will be more appropriate for some drivers/data-types than others. Partitioning '''''''''''' Data sources are assumed to be *partitionable*. A data partition is a randomly accessible fragment of the data. In the case of sequential and data-frame sources, partitions are numbered, starting from zero, and correspond to contiguous chunks of data divided along the first dimension of the data structure. In general, any partitioning scheme is conceivable, such as a tuple-of-ints to index the chunks of a large numerical array. Not all data sources can be partitioned. For example, file formats without sufficient indexing often can only be read from beginning to end. In these cases, the DataSource object should report that there is only 1 partition. However, it often makes sense for a data source to be able to represent a directory of files, in which case each file will correspond to one partition. Metadata '''''''' Once opened, a DataSource object can have arbitrary metadata associated with it. The metadata for a data source should be a dictionary that can be serialized as JSON. This metadata comes from the following sources: 1. A data catalog entry can associate fixed metadata with the data source. This is helpful for data formats that do not have any support for metadata within the file format. 2. The driver handling the data source may have some general metadata associated with the state of the system at the time of access, available even before loading any data-specific information. 2. A driver can add additional metadata when the schema is loaded for the data source. This allows metadata embedded in the data source to be exported. From the user perspective, all of the metadata should be loaded once the data source has loaded the rest of the schema (after ``discover()``, ``read()``, ``to_dask()``, etc have been called). Subclassing ``intake.source.base.DataSourceBase`` ------------------------------------------------- Every Intake driver class should be a subclass of ``intake.source.base.DataSource``. The class should have the following attributes to identify itself: - ``name``: The short name of the driver. This should be a valid python identifier. You should not include the word ``intake`` in the driver name. - ``version``: A version string for the driver. This may be reported to the user by tools based on Intake, but has no semantic importance. - ``container``: The container type of data sources created by this object, e.g., ``dataframe``, ``ndarray``, or ``python``, one of the keys of ``intake.container.container_map``. For simplicity, a driver many only return one typed of container. If a particular source of data could be used in multiple ways (such as HDF5 files interpreted as dataframes or as ndarrays), two drivers must be created. These two drivers can be part of the same Python package. - ``partition_access``: Do the data sources returned by this driver have multiple partitions? This may help tools in the future make more optimal decisions about how to present data. If in doubt (or the answer depends on init arguments), ``True`` will always result in correct behavior, even if the data source has only one partition. The ``__init()__`` method should always accept a keyword argument ``metadata``, a dictionary of metadata from the catalog to associate with the source. This dictionary must be serializable as JSON. The `DataSourceBase` class has a small number of methods which should be overridden. Here is an example producing a data-frame:: class FooSource(intake.source.base.DataSource): container = 'dataframe' name = 'foo' version = '0.0.1' partition_access = True def __init__(self, a, b, metadata=None): # Do init here with a and b super(FooSource, self).__init__( metadata=metadata ) def _get_schema(self): return intake.source.base.Schema( datashape=None, dtype={'x': "int64", 'y': "int64"}, shape=(None, 2), npartitions=2, extra_metadata=dict(c=3, d=4) ) def _get_partition(self, i): # Return the appropriate container of data here return pd.DataFrame({'x': [1, 2, 3], 'y': [10, 20, 30]}) def read(self): self._load_metadata() return pd.concat([self.read_partition(i) for i in range(self.npartitions)]) def _close(self): # close any files, sockets, etc pass Most of the work typically happens in the following methods: - ``__init__()``: Should be very lightweight and fast. No files or network resources should be opened, and no significant memory should be allocated yet. Data sources might be serialized immediately. The default implementation of the pickle protocol in the base class will record all the arguments to ``__init__()`` and recreate the object with those arguments when unpickled, assuming the class has no side effects. - ``_get_schema()``: May open files and network resources and return as much of the schema as possible in small amount of *approximately* constant time. Typically, imports of packages needed by the source only happen here. The ``npartitions`` and ``extra_metadata`` attributes must be correct when ``_get_schema`` returns. Further keys such as ``dtype``, ``shape``, etc., should reflect the container type of the data-source, and can be ``None`` if not easily knowable, or include ``None`` for some elements. File-based sources should use fsspec to open a local or remote URL, and pass ``storage_options`` to it. This ensures compatibility and extra features such as caching. If the backend can only deal with local files, you may still want to use ``fsspec.open_local`` to allow for caching. - ``_get_partition(self, i)``: Should return all of the data from partition id ``i``, where ``i`` is typically an integer, but may be something more complex. The base class will automatically verify that ``i`` is in the range ``[0, npartitions)``, so no range checking is required in the typical case. - ``_close(self)``: Close any network or file handles and deallocate any significant memory. Note that these resources may be need to be reopened/reallocated if a read is called again later. The full set of user methods of interest are as follows: - ``discover(self)``: Read the source attributes, like ``npartitions``, etc. As with ``_get_schema()`` above, this method is assumed to be fast, and make a best effort to set attributes. The output should be serializable, if the source is to be used on a server; the details contained will be used for creating a remote-source on the client. - ``read(self)``: Return all the data in memory in one in-memory container. - ``read_chunked(self)``: Return an iterator that returns contiguous chunks of the data. The chunking is generally assumed to be at the partition level, but could be finer grained if desired. - ``read_partition(self, i)``: Returns the data for a given partition id. It is assumed that reading a given partition does not require reading the data that precedes it. If ``i`` is out of range, an ``IndexError`` should be raised. - ``to_dask(self)``: Return a (lazy) Dask data structure corresponding to this data source. It should be assumed that the data can be read from the Dask workers, so the loads can be done in future tasks. For further information, see the `Dask documentation `_. - ``close(self)``: Close network or file handles and deallocate memory. If other methods are called after ``close()``, the source is automatically reopened. - ``to_*``: for some sources, it makes sense to provide alternative outputs aside from the base container (dataframe, array, ...) and Dask variants. Note that all of these methods typically call ``_get_schema``, to make sure that the source has been initialised. Subclassing ``intake.source.base.DataSource`` --------------------------------------------- ``DataSource`` provides the same functionality as ``DataSourceBase``, but has some additional mixin classes to provide some extras. A developer may choose to derive from ``DataSource`` to get all of these, or from ``DataSourceBase`` and make their own choice of mixins to support. - ``HoloviewsMixin``: provides plotting and GUI capabilities via the `holoviz`_ stack - ``PersistMixin``: allows for storing a local copy in a default format for the given container type - ``CacheMixin``: allows for local storage of data files for a source. Deprecated, you should use one of the caching mechanisms in ``fsspec``. .. _holoviz: https://holoviz.org/index.html .. _driver-discovery: Driver Discovery ---------------- Intake discovers available drivers in three different ways, described below. After the discovery phase, Intake will automatically create ``open_[driver_name]`` convenience functions under the ``intake`` module namespace. Calling a function like ``open_csv()`` is equivalent to instantiating the corresponding data-source class. Entrypoints ''''''''''' If you are packaging your driver into an installable package to be shared, you should add the following to the package's ``setup.py``: .. code-block:: python setup( ... entry_points={ 'intake.drivers': [ 'some_format_name = some_package.and_maybe_a_submodule:YourDriverClass', ... ] }, ) .. important:: Some critical details of Python's entrypoints feature: * Note the unusual syntax of the entrypoints. Each item is given as one long string, with the ``=`` as part of the string. Modules are separated by ``.``, and the final object name is preceded by ``:``. * The right hand side of the equals sign must point to where the object is *actually defined*. If ``YourDriverClass`` is defined in ``foo/bar.py`` and imported into ``foo/__init__.py`` you might expect ``foo:YourDriverClass`` to work, but it does not. You must spell out ``foo.bar:YourDriverClass``. Entry points are a way for Python packages to advertise objects with some common interface. When Intake is imported, it discovers all packages installed in the current environment that advertise ``'intake.drivers'`` in this way. Most packages that define intake drivers have a dependency on ``intake`` itself, for example in order to use intake's base classes. This can create a ciruclar dependency: importing the package imports intake, which tries to discover and import packages that define drivers. To avoid this pitfall, just ensure that ``intake`` is imported first thing in your package's ``__init__.py``. This ensures that the driver-discovery code runs first. Note that you are *not* required to make your package depend on intake. The rule is that *if* you import ``intake`` you must import it first thing. If you do not import intake, there is no circularity. Configuration ''''''''''''' The intake configuration file can be used to: * Specify precedence in the event of name collisions---for example, if two different ``csv`` drivers are installed. * Disable a troublesome driver. * Manually make intake aware of a driver, which can be useful for experimentation and early development until a ``setup.py`` with an entrypoint is prepared. * Assign a driver to a name other than the one assigned by the driver's author. The commandline invocation .. code-block:: bash intake drivers enable some_format_name some_package.and_maybe_a_submodule.YourDriverClass is equivalent to adding this to your intake configuration file: .. code-block:: yaml drivers: some_format_name: some_package.and_maybe_a_submodule.YourDriverClass You can also disable a troublesome driver .. code-block:: bash intake drivers disable some_format_name which is equivalent to .. code-block:: yaml drivers: your_format_name: false Deprecated: Package Scan '''''''''''''''''''''''' When Intake is imported, it will search the Python module path (by default includes ``site-packages`` and other directories in your ``$PYTHONPATH``) for packages starting with ``intake\_`` and discover DataSource subclasses inside those packages to register. drivers will be registered based on the``name`` attribute of the object. By convention, drivers should have names that are lowercase, valid Python identifiers that do not contain the word ``intake``. This approach is deprecated because it is limiting (requires the package to begin with "intake\_") and because the package scan can be slow. Using entrypoints is strongly encouraged. The package scan *may* be disabled by default in some future release of intake. During the transition period, if a package named ``intake_*`` provides an entrypoint for a given name, that will take precedence over any drivers gleaned from the package scan having that name. If intake discovers any names from the package scan for which there are no entrypoints, it will issue a ``FutureWarning``. Python API to Driver Discovery '''''''''''''''''''''''''''''' .. autofunction:: intake.source.discovery.drivers.register_driver .. autofunction:: intake.source.discovery.drivers.enable .. autofunction:: intake.source.discovery.drivers.disable .. _remote_data: Remote Data ----------- For drivers loading from files, the author should be aware that it is easy to implement loading from files stored in remote services. A simplistic case is demonstrated by the included CSV driver, which simply passes a URL to Dask, which in turn can interpret the URL as a remote data service, and use the ``storage_options`` as required (see the Dask documentation on `remote data`_). .. _remote data: http://dask.pydata.org/en/latest/remote-data-services.html More advanced usage, where a Dask loader does not already exist, will likely rely on `fsspec.open_files`_ . Use this function to produce lazy ``OpenFile`` object for local or remote data, based on a URL, which will have a protocol designation and possibly contain glob "*" characters. Additional parameters may be passed to ``open_files``, which should, by convention, be supplied by a driver argument named ``storage_options`` (a dictionary). .. _fsspec.open_files: https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.open_files To use an ``OpenFile`` object, make it concrete by using a context: .. code-block:: python # at setup, to discover the number of files/partitions set_of_open_files = fsspec.open_files(urlpath, mode='rb', **storage_options) # when actually loading data; here we loop over all files, but maybe we just do one partition for an_open_file in set_of_open_files: # `with` causes the object to become concrete until the end of the block with an_open_file as f: # do things with f, which is a file-like object f.seek(); f.read() The ``textfiles`` builtin drivers implements this mechanism, as an example. Structured File Paths --------------------- The CSV driver sets up an example of how to gather data which is encoded in file paths like (``'data_{site}_.csv'``) and return that data in the output. Other drivers could also follow the same structure where data is being loaded from a set of filenames. Typically this would apply to data-frame output. This is possible as long as the driver has access to each of the file paths at some point in ``_get_schema``. Once the file paths are known, the driver developer can use the helper functions defined in ``intake.source.utils`` to get the values for each field in the pattern for each file in the list. These values should then be added to the data, a process which normally would happen within the _get_schema method. The PatternMixin defines driver properties such as urlpath, path_as_pattern, and pattern. The implementation might look something like this:: from intake.source.utils import reverse_formats class FooSource(intake.source.base.DataSource, intake.source.base.PatternMixin): def __init__(self, a, b, path_as_pattern, urlpath, metadata=None): # Do init here with a and b self.path_as_pattern = path_as_pattern self.urlpath = urlpath super(FooSource, self).__init__( container='dataframe', metadata=metadata ) def _get_schema(self): # read in the data values_by_field = reverse_formats(self.pattern, file_paths) # add these fields and map values to the data return data Since dask already has a specific method for including the file paths in the output dataframe, in the CSV driver we set ``include_path_column=True``, to get a dataframe where one of the columns contains all the file paths. In this case, `add these fields and values to data` is a mapping between the categorical file paths column and the ``values_by_field``. In other drivers where each file is read in independently the driver developer can set the new fields on the data from each file before concattenating. This pattern looks more like:: from intake.source.utils import reverse_format class FooSource(intake.source.base.DataSource): ... def _get_schema(self): # get list of file paths for path in file_paths: # read in the file values_by_field = reverse_format(self.pattern, path) # add these fields and values to the data # concatenate the datasets return data To toggle on and off this path as pattern behavior, the CSV and intake-xarray drivers uses the bool ``path_as_pattern`` keyword argument. .. raw:: html ================================================ FILE: docs/source/overview.rst ================================================ Overview ======== Introduction ------------ This page describes the technical design of Intake, with brief details of the aims of the project and components of the library Why Intake? ----------- Intake solves a related set of problems: * Python API standards for loading data (such as DB-API 2.0) are optimized for transactional databases and query results that are processed one row at a time. * Libraries that do load data in bulk tend to each have their own API for doing so, which adds friction when switching data formats. * Loading data into a distributed data structure (like those found in Dask and Spark) often requires writing a separate loader. * Abstractions often focus on just one data model (tabular, n-dimensional array, or semi-structured), when many projects need to work with multiple kinds of data. Intake has the explicit goal of **not** defining a computational expression system. Intake plugins load the data into containers (e.g., arrays or data-frames) that provide their data processing features. As a result, it is very easy to make a new Intake plugin with a relatively small amount of Python. Structure --------- Intake is a Python library for accessing data in a simple and uniform way. It consists of three parts: 1. A lightweight plugin system for adding data loader :term:`drivers` for new file formats and servers (like databases, REST endpoints or other cataloging services) 2. A cataloging system for specifying these sources in simple :term:`YAML` syntax, or with plugins that read source specs from some external data service 3. A server-client architecture that can share data catalog metadata over the network, or even stream the data directly to clients if needed Intake supports loading data into standard Python containers. The list can be easily extended, but the currently supported list is: * Pandas Dataframes - tabular data * NumPy Arrays - tensor data * Python lists of dictionaries - semi-structured data Additionally, Intake can load data into distributed data structures. Currently it supports Dask, a flexible parallel computing library with distributed containers like `dask.dataframe `_, `dask.array `_, and `dask.bag `_. In the future, other distributed computing systems could use Intake to create similar data structures. Concepts -------- Intake is built out of four core concepts: * Data Source classes: the "driver" plugins that each implement loading of some specific type of data into python, with plugin-specific arguments. * Data Source: An object that represents a reference to a data source. Data source instances have methods for loading the data into standard containers, like Pandas DataFrames, but do not load any data until specifically requested. * Catalog: An inventory of catalog entries, each of which defines a Data Source. Catalog objects can be created from local YAML definitions, by connecting to remote servers, or by some driver that knows how to query an external data service. * Catalog Entry: A named data source held internally by catalog objects, which generate data source instances when accessed. The catalog entry includes metadata about the source, as well as the name of the driver and arguments. Arguments can be parameterized, allowing one entry to return different subsets of data depending on the user request. The business of a plugin is to go from some data format (bunch of files or some remote service) to a ":term:`Container`" of the data (e.g., data-frame), a thing on which you can perform further analysis. Drivers can be used directly by the user, or indirectly through data catalogs. Data sources can be pickled, sent over the network to other hosts, and reopened (assuming the remote system has access to the required files or servers). See also the :doc:`glossary`. Future Directions ----------------- Ongoing work for enhancements, as well as requests for plugins, etc., can be found at the `issue tracker `_. See the :ref:`roadmap` for general mid- and long-term goals. .. raw:: html ================================================ FILE: docs/source/persisting.rst ================================================ .. _persisting: Persisting Data =============== (this is an experimental new feature, expect enhancements and changes) Introduction ------------ As defined in the glossary, to :term:`Persist` is to convert data into the storage format most appropriate for the container type, and save a copy of this for rapid lookup in the future. This is of great potential benefit where the creation or transfer of the original data source takes some time. This is not to be confused with the file :term:`Cache`. Usage ----- Any :term:`Data Source` has a method ``.persist()``. The only option that you will need to pick is a :term:`TTL`, the number of seconds that the persisted version lasts before expiry (leave as ``None`` for no expiry). This creates a local copy in the persist directory, which may be in ``"~/.intake/persist``, but can be configured. Each container type (dataframe, array, ...) will have its own implementation of persistence, and a particular file storage format associated. The call to ``.persist()`` may take arguments to tune how the local files are created, and in some cases may require additional optional packages to be installed. Example:: cat = intake.open_catalog('mycat.yaml') # load a remote cat source = cat.csvsource() # source pointing to remote data source.persist() source = cat.csvsource() # future use now gives local intake_parquet.ParquetSource To control whether a catalog will automatically give you the persisted version of a source in this way using the argument ``persist_mode``, e.g., to ignore locally persisted versions, you could have done:: cat = intake.open_catalog('mycat.yaml', persist_mode='never') or source = cat.csvsource(persist_mode='never') Note that if you give a TTL (in seconds), then the original source will be accessed and a new persisted version written transparently when the old persisted version has expired. Note that after persisting, the original source will have ``source.has_been_persisted == True`` and the persisted source (i.e., the one loaded from local files) will have ``source.is_persisted == True``. Export ------ A similar concept to Persist, Export allows you to make a copy of some data source, in the format appropriate for its container, and place this data-set in whichever location suits you, including remote locations. This functionality (``source.export()``) does *not* touch the persist store; instead, it returns a YAML text representation of the output, so that you can put it into a catalog of your own. It would be this catalog that you share with other people. Note that "exported" data-sources like this do contain the information of the original source they were made from in their metadata, so you can recreate the original source, if you want to, and read from there. Persisting to Remote -------------------- If you are typically running your code inside of ephemoral containers, then persisting data-sets may be something that you want to do (because the original source is slow, or parsing is CPU/memory intensive), but the local storage is not useful. In some cases you may have access to some shared network storage mounted on the instance, but in other cases you will want to persist to a remote store. The config value ``'persist_path'``, which can also be set by the environment variable ``INTAKE_PERSIST_PATH`` can be a remote location such as ``s3://mybucket/intake-persist``. You will need to install the appropriate package to talk to the external storage (e.g., ``s3fs``, ``gcsfs``, ``pyarrow``), but otherwise everything should work as before, and you can access the persisted data from any container. The Persist Store ----------------- You can interact directly with the class implementing persistence:: from intake.container.persist import store This singleton instance, which acts like a catalog, allows you to query the contents of the instance store and to add and remove entries. It also allows you to find the original source for any given persisted source, and refresh the persisted version on demand. For details on the methods of the persist store, see the API documentation: :func:`intake.container.persist.PersistStore`. Sources in the store carry a lot of information about the sources they were made from, so that they can be remade successfully. This all appears in the source metadata. The sources use the "token" of the original data source as their key in the store, a value which can be found by ``dask.base.tokenize(source)`` for the original source, or can be taken from the metadata of a persisted source. Note that all of the information about persisted sources is held in a single YAML file in the persist directory (typically ``/persisted/cat.yaml`` within the config directory, but see ``intake.config.conf['persist_path']``). This file can be edited by hand if you wanted to, for example, set some persisted source not to expire. This is only recommended for experts. Future Enhancements ------------------- - CLI functionality to investigate and alter the state of the persist store. - Time check-pointing of persisted data, such that you can not only get the "most recent" but any version in the time-series. - (eventually) pipeline functionality, whereby a persisted data source depends on another persisted data source, and the whole train can be refreshed on a schedule or on demand. .. raw:: html ================================================ FILE: docs/source/plotting.rst ================================================ Plotting ======== Intake provides a plotting API based on the `hvPlot `_ library, which closely mirrors the pandas plotting API but generates interactive plots using `HoloViews `_ and `Bokeh `_. The `hvPlot website `_ provides comprehensive documentation on using the plotting API to quickly visualize and explore small and large datasets. The main features offered by the plotting API include: * Support for tabular data stored in pandas and dask dataframes * Support for gridded data stored in xarray backed nD-arrays * Support for plotting large datasets with `datashader `_ Using Intake alongside hvPlot allows declaratively persisting plot declarations and default options in the regular catalog.yaml files. Setup ''''' For detailed installation instructions see the `getting started section `_ in the hvPlot documentation. To start with install hvplot using conda: .. code-block:: bash conda install -c conda-forge hvplot or using pip: .. code-block:: bash pip install hvplot Usage ''''' The plotting API is designed to work well in and outside the Jupyter notebook, however when using it in JupyterLab the PyViz lab extension must be installed first: .. code-block:: bash jupyter labextension install @pyviz/jupyterlab_pyviz For detailed instructions on displaying plots in the notebook and from the Python command prompt see the `hvPlot user guide `_. Python Command Prompt & Scripts -------------------------------- Assuming the US Crime dataset has been installed (in the `intake-examples repo `_, or from conda with `conda install -c intake us_crime`): Once installed the plot API can be used, by using the ``.plot`` method on an intake ``DataSource``: .. code-block:: python import intake import hvplot as hp crime = intake.cat.us_crime columns = ['Burglary rate', 'Larceny-theft rate', 'Robbery rate', 'Violent Crime rate'] violin = crime.plot.violin(y=columns, group_label='Type of crime', value_label='Rate per 100k', invert=True) hp.show(violin) .. image:: _static/images/plotting_violin.png Notebook -------- Inside the notebook plots will display themselves, however the notebook extension must be loaded first. The extension may be loaded by importing ``hvplot.intake`` module or explicitly loading the holoviews extension, or by calling ``intake.output_notebook()``: .. code-block:: python # To load the extension run this import import hvplot.intake # Or load the holoviews extension directly import holoviews as hv hv.extension('bokeh') # convenience function import intake intake.output_notebook() crime = intake.cat.us_crime columns = ['Violent Crime rate', 'Robbery rate', 'Burglary rate'] crime.plot(x='Year', y=columns, value_label='Rate (per 100k people)') .. raw:: html :file: _static/images/plotting_example.html Predefined Plots ---------------- Some catalogs will define plots appropriate to a specific data source. These will be specified such that the user gets the right view with the right columns and labels, without having to investigate the data in detail -- this is ideal for quick-look plotting when browsing sources. .. code-block:: python import intake intake.us_crime.plots Returns `['example']`. This works whether accessing the entry object or the source instance. To visualise .. code-block:: python intake.us_crime.plot.example() Persisting metadata ''''''''''''''''''' Intake allows catalog yaml files to declare metadata fields for each data source which are made available alongside the actual dataset. The plotting API reserves certain fields to define default plot options, to label and annotate the data fields in a dataset and to declare pre-defined plots. Declaring defaults ------------------ The first set of metadata used by the plotting API is the `plot` field in the metadata section. Any options found in the metadata field will apply to all plots generated from that data source, allowing the definition of plotting defaults. For example when plotting a fairly large dataset such as the NYC Taxi data, it might be desirable to enable datashader by default ensuring that any plot that supports it is datashaded. The syntax to declare default plot options is as follows: .. code-block:: yaml sources: nyc_taxi: description: NYC Taxi dataset driver: parquet args: urlpath: 's3://datashader-data/nyc_taxi_wide.parq' metadata: plot: datashade: true Declaring data fields --------------------- The columns of a CSV or parquet file or the coordinates and data variables in a NetCDF file often have shortened, or cryptic names with underscores. They also do not provide additional information about the units of the data or the range of values, therefore the catalog yaml specification also provides the ability to define additional information about the `fields` in a dataset. Valid attributes that may be defined for the data `fields` include: - `label`: A readable label for the field which will be used to label axes and widgets - `unit`: A unit associated with the values inside a data field - `range`: A range associated with a field declaring limits which will override those computed from the data Just like the default plot options the `fields` may be declared under the metadata section of a data source: .. code-block:: yaml sources: nyc_taxi: description: NYC Taxi dataset driver: parquet args: urlpath: 's3://datashader-data/nyc_taxi_wide.parq' metadata: fields: dropoff_x: label: Longitude dropoff_y: label: Latitude total_fare: label: Fare unit: $ Declaring custom plots ---------------------- As shown in the `hvPlot user guide `__, the plotting API provides a variety of plot types, which can be declared using the `kind` argument or via convenience methods on the plotting API, e.g. `cat.source.plot.scatter()`. In addition to declaring default plot options and field metadata data sources may also declare custom plot, which will be made available as methods on the plotting API. In this way a catalogue may declare any number of custom plots alongside a datasource. To make this more concrete consider the following custom plot declaration on the `plots` field in the metadata section: .. code-block:: yaml sources: nyc_taxi: description: NYC Taxi dataset driver: parquet args: urlpath: 's3://datashader-data/nyc_taxi_wide.parq' metadata: plots: dropoff_scatter: kind: scatter x: dropoff_x y: dropoff_y datashade: True width: 800 height: 600 This declarative specification creates a new custom plot called `dropoff_scatter`, which will be available on the catalog under `cat.nyc_taxi.plot.dropoff_scatter()`. Calling this method on the plot API will automatically generate a datashaded scatter plot of the dropoff locations in the NYC taxi dataset. Of course the three metadata fields may also be used together, declaring global defaults under the `plot` field, annotations for the data `fields` under the `fields` key and custom plots via the `plots` field. .. raw:: html ================================================ FILE: docs/source/plugin-directory.rst ================================================ .. _plugin-directory: Plugin Directory ================ This is a list of known projects which install driver plugins for Intake, and the named drivers each contains: .. raw:: html :file: plugin-list.html Don't see your favorite format? See :doc:`making-plugins` for how to create new plugins. Note that if you want your plugin listed here, open an issue in the `Intake issue repository `_ and add an entry to the `status dashboard repository `_. We also have a `plugin wishlist Github issue `_ that shows the breadth of plugins we hope to see for Intake. .. raw:: html ================================================ FILE: docs/source/quickstart.rst ================================================ Quickstart ========== This guide will show you how to get started using Intake to read data, and give you a flavour of how Intake feels to the :term:`Data User`. It assumes you are working in either a conda or a virtualenv/pip environment. For notebooks with executable code, see the :doc:`examples`. This walk-through can be run from a notebook or interactive python session. Installation ------------ If you are using `Anaconda`_ or Miniconda, install Intake with the following commands:: conda install -c conda-forge intake If you are using virtualenv/pip, run the following command:: pip install intake Note that this will install with the minimum of optional requirements. If you want a more complete install, use `intake[complete]` instead. .. _Anaconda: https://www.anaconda.com/download/ Creating Sample Data -------------------- Let's begin by creating a sample data set and catalog. At the command line, run the ``intake example`` command. This will create an example data :term:`Catalog` and two CSV data files. These files contains some basic facts about the 50 US states, and the catalog includes a specification of how to load them. Loading a Data Source --------------------- :term:`Data sources` can be created directly with the ``open_*()`` functions in the ``intake`` module. To read our example data:: >>> import intake >>> ds = intake.open_csv('states_*.csv') >>> print(ds) Each open function has different arguments, specific for the data format or service being used. Reading Data ------------ Intake reads data into memory using :term:`containers` you are already familiar with: * Tables: Pandas DataFrames * Multidimensional arrays: NumPy arrays * Semistructured data: Python lists of objects (usually dictionaries) To find out what kind of container a data source will produce, inspect the ``container`` attribute:: >>> ds.container 'dataframe' The result will be ``dataframe``, ``ndarray``, or ``python``. (New container types will be added in the future.) For data that fits in memory, you can ask Intake to load it directly:: >>> df = ds.read() >>> df.head() state slug code nickname ... 0 Alabama alabama AL Yellowhammer State 1 Alaska alaska AK The Last Frontier 2 Arizona arizona AZ The Grand Canyon State 3 Arkansas arkansas AR The Natural State 4 California california CA Golden State Many data sources will also have quick-look plotting available. The attribute ``.plot`` will list a number of built-in plotting methods, such as ``.scatter()``, see :doc:`plotting`. Intake data sources can have *partitions*. A partition refers to a contiguous chunk of data that can be loaded independent of any other partition. The partitioning scheme is entirely up to the plugin author. In the case of the CSV plugin, each ``.csv`` file is a partition. To read data from a data source one chunk at a time, the ``read_chunked()`` method returns an iterator:: >>> for chunk in ds.read_chunked(): print('Chunk: %d' % len(chunk)) ... Chunk: 24 Chunk: 26 Working with Dask ----------------- Working with large datasets is much easier with a parallel, out-of-core computing library like `Dask `_. Intake can create Dask containers (like ``dask.dataframe``) from data sources that will load their data only when required:: >>> ddf = ds.to_dask() >>> ddf Dask DataFrame Structure: admission_date admission_number capital_city capital_url code constitution_url facebook_url landscape_background_url map_image_url nickname population population_rank skyline_background_url slug state state_flag_url state_seal_url twitter_url website npartitions=2 object int64 object object object object object object object object int64 int64 object object object object object object object ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... Dask Name: from-delayed, 4 tasks The Dask containers will be partitioned in the same way as the Intake data source, allowing different chunks to be processed in parallel. Please read the Dask documentation to understand the differences when working with Dask collections (Bag, Array or Data-frames). Opening a Catalog ----------------- A :term:`Catalog` is an inventory of data sources, with the type and arguments prescribed for each, and arbitrary metadata about each source. In the simplest case, a catalog can be described by a file in YAML format, a ":term:`Catalog file`". In real usage, catalogues can be defined in a number of ways, such as remote files, by connecting to a third-party data service (e.g., SQL server) or through an Intake :term:`Server` protocol, which can implement any number of ways to search and deliver data sources. The ``intake example`` command, above, created a catalog file with the following :term:`YAML`-syntax content: .. code-block:: yaml sources: states: description: US state information from [CivilServices](https://civil.services/) driver: csv args: urlpath: '{{ CATALOG_DIR }}/states_*.csv' metadata: origin_url: 'https://github.com/CivilServiceUSA/us-states/blob/v1.0.0/data/states.csv' To load a :term:`Catalog` from a :term:`Catalog file`:: >>> cat = intake.open_catalog('us_states.yml') >>> list(cat) ['states'] This catalog contains one data source, called ``states``. It can be accessed by attribute:: >>> cat.states.to_dask()[['state','slug']].head() state slug 0 Alabama alabama 1 Alaska alaska 2 Arizona arizona 3 Arkansas arkansas 4 California california Placing data source specifications into a catalog like this enables declaring data sets in a single canonical place, and not having to use boilerplate code in each notebook/script that makes use of the data. The catalogs can also reference one-another, be stored remotely, and include extra metadata such as a set of named quick-look plots that are appropriate for the particular data source. Note that catalogs are **not** restricted to being stored in YAML files, that just happens to be the simplest way to display them. Many catalog entries will also contain "user_parameter" blocks, which are indications of options explicitly allowed by the catalog author, or for validation or the values passed. The user can customise how a data source is accessed by providing values for the user_parameters, overriding the arguments specified in the entry, or passing extra keyword arguments to be passed to the driver. The keywords that should be passed are limited to the user_parameters defined and the inputs expected by the specific driver - such usage is expected only from those already familiar with the specifics of the given format. In the following example, the user overrides the "csv_kwargs" keyword, which is described in the documentation for :func:`CSVSource ` and gets passed down to the CSV reader:: # pass extra kwargs understood by the csv driver >>> intake.cat.states(csv_kwargs={'header': None, 'skiprows': 1}).read().head() 0 1 ... 17 0 Alabama alabama ... https://twitter.com/alabamagov 1 Alaska alaska ... https://twitter.com/alaska Note that, if you are *creating* such catalogs, you may well start by trying the ``open_csv`` command, above, and then use ``print(ds.yaml())``. If you do this now, you will see that the output is very similar to the catalog file we have provided. Installing Data Source Packages ------------------------------- Intake makes it possible to create :term:`Data packages` (``pip`` or ``conda``) that install data sources into a global catalog. For example, we can install a data package containing the same data we have been working with:: conda install -c intake data-us-states :term:`Conda` installs the catalog file in this package to ``$CONDA_PREFIX/share/intake/us_states.yml``. Now, when we import ``intake``, we will see the data from this package appear as part of a global catalog called ``intake.cat``. In this particular case we use Dask to do the reading (which can handle larger-than-memory data and parallel processing), but ``read()`` would work also:: >>> import intake >>> intake.cat.states.to_dask()[['state','slug']].head() state slug 0 Alabama alabama 1 Alaska alaska 2 Arizona arizona 3 Arkansas arkansas 4 California california The global catalog is a union of all catalogs installed in the conda/virtualenv environment and also any catalogs installed in user-specific locations. Adding Data Source Packages using the Intake path ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Intake checks the Intake config file for ``catalog_path`` or the environment variable ``"INTAKE_PATH"`` for a colon separated list of paths (semicolon on windows) to search for catalog files. When you import ``intake`` we will see all entries from all of the catalogues referenced as part of a global catalog called ``intake.cat``. Using the GUI ------------- A graphical data browser is available in the Jupyter notebook environment or standalone web-server. It will show the contents of any installed catalogs, plus allows for selecting local and remote catalogs, to browse and select entries from these. See :doc:`gui`. .. raw:: html ================================================ FILE: docs/source/reference.rst ================================================ Reference --------- .. toctree:: :maxdepth: 1 api.rst changelog.rst making-plugins.rst data-packages.rst .. raw:: html ================================================ FILE: docs/source/roadmap.rst ================================================ .. _roadmap: Roadmap ======= Some high-level work that we expect to be achieved on the time-scale of months. This list is not exhaustive, but rather aims to whet the appetite for what Intake can be in the future. Since Intake aims to be a community of data-oriented pythoneers, nothing written here is laid in stone, and users and devs are encouraged to make their opinions known! Broaden the coverage of formats ------------------------------- Data-type drivers are easy to write, but still require some effort, and therefore reasonable impetus to get the work done. Conversations over the coming months can help determine the drivers that should be created by the Intake team, and those that might be contributed by the community. The next type that we would specifically like to consider is machine learning model artifacts. **EDIT** see https://github.com/AlbertDeFusco/intake-sklearn , and hopefully more to come. Streaming Source ---------------- Many data sources are inherently time-sensitive and event-wise. These are not covered well by existing Python tools, but the ``streamz`` library may present a nice way to model them. From the Intake point of view, the task would be to develop a streaming type, and at least one data driver that uses it. The most obvious place to start would be read a file: every time a new line appears in the file, an event is emitted. This is appropriate, for instance, for watching the log files of a web-server, and indeed could be extended to read from an arbitrary socket. **EDIT** see: https://github.com/intake/intake-streamz Server publish hooks -------------------- To add API endpoints to the server, so that a user (with sufficient privilege) can post data specifications to a running server, optionally saving the specs to a catalog server-side. Furthermore, we will consider the possibility of being able to upload and/or transform data (rather than refer to it in a third-party location), so that you would have a one-line "publish" ability from the client. The server, in general, could do with a lot of work to become more than the current demonstration/prototype. In particular, it should be able to be performant and scalable, meaning that the server implementation ought to keep as little local state as possible. Simplify dependencies and class hierarchy ----------------------------------------- We would like the make it easier to write Intake drivers which don't need any persist or GUI functionality, and to be able to install Intake core functionality (driver registry, data loading and catalog traversal) without needing many other packages at all. **EDIT** this has been partly done, you can derive from ``DataSourceBase`` and not have to use the full set of Intake's features for simplicity. We have also gone some distance to separate out dependencies for parts of the package, so that you can install Intake and only use some of the subpackages/modules - imports don't happen until those parts of the code are used. We have *not* yet split the intake conda package into, for example, intake-base, intake-server, intake-gui... Reader API ---------- For those that wish to provide Intake's data source API, and make data sources available to Intake cataloguing, but don't wish to take Intake as a direct dependency. The actual API of ``DataSources`` is rather simple: - ``__init__``: collect arguments, minimal IO at this point - ``discover()``: get metadata from the source, by querying the files/service itself - ``read()``: return in-memory version of the data - ``to_*``: return reference objects for the given compute engine, typically Dask - ``read_partition(...)``: read part of the data into memory, where the argument makes sense for the given type of data - ``configure_new()``: create new instance with different arguments - ``yaml()``: representation appropriate for inclusion in a YAML catalogue - ``close()``: release any resources Of these, only the first three are really necessary for a iminal interface, so Intake might do well to publish this *protocol specification*, so that new drivers can be written that can be used by Intake but do not need Intake, and so help adoption. .. raw:: html ================================================ FILE: docs/source/scope2.rst ================================================ Scope ===== Here we lay out what Intake is, why you might want to use it, main features and also a few reasons you may wish to look elsewhere. Motivation ---------- Data scientists, analysis, ML/AI developers and engineers want to spend their time working with data to produce models, results and insights. Those first few lines or cells in a workflow to get to the data are annoying, often wrong and brittle. Consider the following cell of realistic-looking code. .. image:: ./_static/images/complex_cell.png :alt: Complex Cell This cell encodes several steps it takes to produce the object of interest for the user, including defining some location URL, fetching data, loading, cleaning and converting data. It refers to multiple packages, and hard-codes various arguments directly. This is clearly messy code, but it is extremely common - in fact, most data-centric workflows start like this. Now imagine finding and using this dataset for a new project, or when the upstream location, format or types have changed. Further more, the steps are highly specific - if you decide you need a different compute engine, or the data moves to a different storage service, it is painful to fix this. If the same dataset is used in many places, you will need to remember to fix it in each place. Wouldn't it be nice if you could declare your data and pipeline just once? Then you can - version control your data set descriptions as you would for code (and maybe data files) - share your data definitions with others, just by writing this prescription to any shared space - update all users of this data in a single place, single source of truth - encode a set of transforms as part of a data-oriented processing framework - decide to load and process the same data but with different engines - automatically encode python statements into data decriptions - convert between dozens of data types Counter indications ------------------- The following are some situations where you might not be interested in Intake: - you only ever read one type of file with one package, e.g., ``pd.read_csv``. In this case, Intake can still serve to describe and enumerate data sources, but you miss out on most of the functionality. - you only ever work on data sources by yourself and have no reasons to direct other people in how to read data (not be the consumer of such descriptions). - all your data needs are already met by some other data service by itself, for example a set of tables/procedures/views/queries on a SQL database. - you have data that is not read by anything in Intake (but maybe you could pretty easily add any reader you need). - you don't yet see any benefit of describing your datasets in catalogs. What Intake Isn't ----------------- This is not a workflow running system (maybe use Airflow, Celery, Prefect, etc.) nor a compute engine (although we hook into spark, dask, ray and such). Our scope is limited to describing data and how to load it. Here, "load" includes transforms and cleaning with the engine of your choice to get to something ready for analysis. We delegate all calls to other third-party packages, so you can think of the descriptions as being "what to call". Relationship to ``fsspec`` -------------------------- ``fsspec`` is a library concerned only with reading bytes from various stores. It has become the standard (but not only) package for this in the python/data ecosystem and is supported by many other packages. Intake will make use of ``fsspec`` for many file readers, encoding "storage_options` (further arguments that ``fsspec`` understands) when necessary. Where a reader doesn't understand ``fsspec``, "storage_options" are ignored, and in some cases the files might need to be downloaded locally before being accessible. Intake prefers using ``fsspec``, but it is not necessary. Relationship to ``dask`` ------------------------ Intake originally came out of the same stable as `dask`_, and this was implicit in all of the "sources" in V1. Since then, many more compute engines have become available in python and some that already existed have become more convenient (e.g., Spark). We no longer require or even prefer ``dask``, but it is, of course, a fine choice for out-of-core, parallel and distributed workloads. .. _dask: https://dask.org .. _rel_v1: Relationship to V1 ------------------ We aim to be largely backward compatible with pre-V2 Intake sources and catalogs. Many data sources have been rewritten to use the new framework, and many rarely-used features have been removed. In particular, the following features are no longer supported for V1 sources: - the intake server (use ``tiled`` instead) - caching (use ``fsspec`` caching instead or custom caching pipelines) - "persist" and "export" (use the new converters and output classes) - automatic ``hvplot`` (this is now an "output" converter for pandas and xarray types) - some niche source features such as CSV file pattern matching In addition, not all existing ``intake_*`` packages have corresponding readers in Take2, but we are making progress. ================================================ FILE: docs/source/start.rst ================================================ .. _start: Start here ---------- These documents will familiarise you with Intake, show you some basic usage and examples, and describe Intake's place in the wider python data world. .. toctree:: :maxdepth: 1 quickstart.rst use_cases.rst overview.rst examples.rst deployments.rst .. raw:: html ================================================ FILE: docs/source/tools.rst ================================================ Command Line Tools ================== The package installs two executable commands: for starting the catalog server; and a client for accessing catalogs and manipulating the configuration. .. _configuration: Configuration ------------- A file-based configuration service is available to Intake. This file is by default sought at the location ``~/.intake/conf.yaml``, but either of the environment variables ``INTAKE_CONF_DIR`` or ``INTAKE_CONF_FILE`` can be used to specify another directory or file. If both are given, the latter takes priority. At present, the configuration file might look as follows: .. code-block:: yaml auth: cls: "intake.auth.base.BaseAuth" port: 5000 catalog_path: - /home/myusername/special_dir These are the defaults, and any parameters not specified will take the values above * the Intake Server will listen on port 5000 (this can be overridden on the command line, see below) * and the auth system used will be the fully qualified class given (which, for BaseAuth, always allows access). See ``intake.config.defaults`` for a full list of keys and their default values. Log Level --------- The logging level is configurable using Python's built-in logging module. The config option ``'logging'`` holds the current level for the intake logger, and can take values such as ``'INFO'`` or ``'DEBUG'``. This can be set in the ``conf.yaml`` file of the config directory (e.g., ``~/.intake/``), or overriden by the environment variable ``INTAKE_LOG_LEVEL``. Furthermore, the level and settings of the logger can be changed programmatically in code:: import logging logger = logging.getLogger('intake') logger.setLevel(logging.DEBUG) logget.addHandler(..) Intake Server ------------- The server takes one or more catalog files as input and makes them available on port 5000 by default. You can see the full description of the server command with: :: >>> intake-server --help usage: intake-server [-h] [-p PORT] [--list-entries] [--sys-exit-on-sigterm] [--flatten] [--no-flatten] [-a ADDRESS] FILE [FILE ...] Intake Catalog Server positional arguments: FILE Name of catalog YAML file optional arguments: -h, --help show this help message and exit -p PORT, --port PORT port number for server to listen on --list-entries list catalog entries at startup --sys-exit-on-sigterm internal flag used during unit testing to ensure .coverage file is written --flatten --no-flatten -a ADDRESS, --address ADDRESS address to use as a host, defaults to the address in the configuration file, if provided otherwise localhost usage: intake-server [-h] [-p PORT] [--list-entries] [--sys-exit-on-sigterm] [--flatten] [--no-flatten] [-a ADDRESS] FILE [FILE ...] To start the server with a local catalog file, use the following: :: >>> intake-server intake/catalog/tests/catalog1.yml Creating catalog from: - intake/catalog/tests/catalog1.yml catalog_args ['intake/catalog/tests/catalog1.yml'] Entries: entry1,entry1_part,use_example1 Listening on port 5000 You can use the catalog client (defined below) using: :: $ intake list intake://localhost:5000 entry1 entry1_part use_example1 Intake Client ------------- While the Intake data sources will typically be accessed through the Python API, you can use the client to verify a catalog file. Unlike the server command, the client has several subcommands to access a catalog. You can see the list of available subcommands with: :: >>> intake --help usage: intake {list,describe,exists,get,discover} ... We go into further detail in the following sections. List '''' This subcommand lists the names of all available catalog entries. This is useful since other subcommands require these names. If you wish to see the details about each catalog entry, use the ``--full`` flag. This is equivalent to running the ``intake describe`` subcommand for all catalog entries. :: >>> intake list --help usage: intake list [-h] [--full] URI positional arguments: URI Catalog URI optional arguments: -h, --help show this help message and exit --full :: >>> intake list intake/catalog/tests/catalog1.yml entry1 entry1_part use_example1 >>> intake list --full intake/catalog/tests/catalog1.yml [entry1] container=dataframe [entry1] description=entry1 full [entry1] direct_access=forbid [entry1] user_parameters=[] [entry1_part] container=dataframe [entry1_part] description=entry1 part [entry1_part] direct_access=allow [entry1_part] user_parameters=[{'default': '1', 'allowed': ['1', '2'], 'type': u'str', 'name': u'part', 'description': u'part of filename'}] [use_example1] container=dataframe [use_example1] description=example1 source plugin [use_example1] direct_access=forbid [use_example1] user_parameters=[] Describe '''''''' Given the name of a catalog entry, this subcommand lists the details of the respective catalog entry. :: >>> intake describe --help usage: intake describe [-h] URI NAME positional arguments: URI Catalog URI NAME Catalog name optional arguments: -h, --help show this help message and exit :: >>> intake describe intake/catalog/tests/catalog1.yml entry1 [entry1] container=dataframe [entry1] description=entry1 full [entry1] direct_access=forbid [entry1] user_parameters=[] Discover '''''''' Given the name of a catalog entry, this subcommand returns a key-value description of the data source. The exact details are subject to change. :: >>> intake discover --help usage: intake discover [-h] URI NAME positional arguments: URI Catalog URI NAME Catalog name optional arguments: -h, --help show this help message and exit :: >>> intake discover intake/catalog/tests/catalog1.yml entry1 {'npartitions': 2, 'dtype': dtype([('name', 'O'), ('score', '>> intake exists --help usage: intake exists [-h] URI NAME positional arguments: URI Catalog URI NAME Catalog name optional arguments: -h, --help show this help message and exit :: >>> intake exists intake/catalog/tests/catalog1.yml entry1 True >>> intake exists intake/catalog/tests/catalog1.yml entry2 False Get ''' Given the name of a catalog entry, this subcommand outputs the entire data source to standard output. :: >>> intake get --help usage: intake get [-h] URI NAME positional arguments: URI Catalog URI NAME Catalog name optional arguments: -h, --help show this help message and exit :: >>> intake get intake/catalog/tests/catalog1.yml entry1 name score rank 0 Alice1 100.5 1 1 Bob1 50.3 2 2 Charlie1 25.0 3 3 Eve1 25.0 3 4 Alice2 100.5 1 5 Bob2 50.3 2 6 Charlie2 25.0 3 7 Eve2 25.0 3 Config and Cache '''''''''''''''' CLI functions starting with ``intake cache`` and ``intake config`` are available to provide information about the system: the locations and value of configuration parameters, and the state of cached files. .. raw:: html ================================================ FILE: docs/source/tour2.rst ================================================ Developers' Package Tour ======================== General Guidelines ------------------ Intake is an open source project, and all development happens on `github`_. Please open issues or discussions there to talk about problems with the code or request features. To contribute, you should: - clone the repo locally - fork the repo to your personal identity in github using the "fork" button - run ``pre-commit install`` in the repo - make changes locally as you see fit, and commit to a new branch - push the branch to your fork, and follow the prompt to create a Pull Request (PR) You can expect comments on your PR within a couple of days. To have a higher chance of having your changes accepted, a concise title description are best, and ideally new code should be accompanied by tests. .. _github: https://github.com/intake/intake Outline ------- For those interested in Intake Take2, here are the places to look for contributing. All of the implementation code lives under ``intake.readers``, which was developed for a while parallel with and without touching Intake's V1 code. The list below gives summaries of the modules, and the principle classes themselves are in the :ref:`api2`. .. autosummary:: intake.config intake.readers.catalogs intake.readers.convert intake.readers.datatypes intake.readers.entry intake.readers.importlist intake.readers.metadata intake.readers.mixins intake.readers.namespaces intake.readers.output intake.readers.readers intake.readers.search intake.readers.transform intake.readers.user_parameters Creating Datatypes and Readers ------------------------------ Here follows a minimalist complete set of classes to make a complete pipeline. A typical data/reader implementation in Intake Take 2 is very simple. Here is the CSV prototype .. code-block:: python from intake.readers import FileData class CSV(FileData): filepattern = "(csv$|txt$|tsv$)" mimetypes = "(text/csv|application/csv|application/vnd.ms-excel)" structure = {"table"} This specified that CSVs live in files (the superclass), which also implies that they may be local or remote. Further, the block specifies expected URL/filenames and MIME types, as well as an indicator that this filetype is typically used for tables. All of these attributes are optional - an instance just contains enough information to unambiguously identify the source of data. For the case of a CSV dataset, this would be just the URL(s) of the data plus any extra storage backend parameters. Other data types may have other necessary attributes, such as a SQL dataset is a combination of server connection string and query. The pandas CSV reader counterpart looks like this .. code-block:: python from intake.readers import FileReader, datatypes class PandasCSV(FileReader): imports = {"pandas"} output_instance = "pandas:DataFrame" storage_options = True implements = {datatypes.CSV} func = "pandas:read_csv" url_arg = "filepath_or_buffer" This says that: - the data type is made of files - the reader requires "pandas" to be installed - the result will be a DataFrame - if the URL is remote, fsspec-style storage_options are acceptable - it can be used on the CSV type from before (only) - it uses the ``read_csv`` function from the ``pandas`` package - and that the URL of the data source should be passed using the argument name "filepath_or_buffer" (this information can be found from the target function's signature and docstring). Often a reader is this simple, or even simpler when you can group attributes in common subclasses. In other cases, it may be necessary to override the key ``._read()`` method, which is the one that does the work. In fact, PandasCSV does override ``.discover()``, to add the ``nrows=`` argument, but adding such refinements is optional. Doing the above is enough, such that a URL ending in "csv" will be recognised, and pandas offered as one of the potential readers; and thus we can make a reader instance and store it in a Catalog. Next, let's imagine we want to make a super simple converter: .. code-block:: python from intake import BaseConverter class PandasToStr(BaseConverter): instances = {"pandas:DataFrame": "builtins:str"} func = "builtins:str" This just returns the string representation of the dataframe, turning DataFrame instances into ``str`` instances (actually, it would work for just about any python object). The inclusion of "DataFrame" in ``instances`` means that Intake will know that this is a transform that can be applied to readers that produce a DataFrame, and it will appear in tab completions and a reader instance's ``.transform`` attribute. To complete the pipeline, lets make a outputter which writes this back to a file .. code-block:: python class StrToFile(BaseConverter): instances = {"builtins:str": datatypes.Text.qname()} def run(self, x, url, storage_options=None, metadata=None, **kwargs): with fsspec.open(url, mode="wt", **storage_options) as f: f.write(x) return datatypes.Text(url=url, storage_options=storage_options, metadata=metadata) Although we use ``fsspec`` (which is recommended, where possible), the code is again super-simple. It is conventional, but not necessary, to have such "output" nodes return a datatypes instance. All of this now allows: .. code-block:: python >>> import intake >>> intake.auto_pipeline("blah.csv", "Text") PipelineReader: 0: intake.readers.readers:PandasCSV, () {} => pandas:DataFrame 1: PandasToStr, () {} => builtins:str 2: StrToFile, () {} => intake.readers.datatypes:Text (where the output filename remains to be filled in) Packaging --------- Having made a couple of new classes, how would we get these to potential users? Assuming you are already familiar with how to create a python package _in_general_, what you need to know, is that Intake will find the new code so long as the classes are subclasses of BaseData, BaseReader (etc.), and the code is imported. That importing can be done - explicitly (which is good form for ad-hoc/experimental use) - including an `entrypoint`_ for the package in the group "intake.imports", where the value would be of the form "package.module" or "package:module" (the latter for ``import .. from`` style). This requires that the new package is installed via ``pip``, ``conda``, etc. - adding the package/module to ``intake.conf["extra_imports"]`` and saving; this will take effect on the next import of Intake. .. _entrypoint: https://packaging.python.org/en/latest/specifications/entry-points/ Migration from V1 ----------------- Section :ref:`v1` shows the principal differences to Intake before Take2. From a developer's viewpoint, if porting former plugins, here are some things to bear in mind. - in v2 we generally separate out the definition of the data itself versus the specific reader, e.g., HDF5 is a file type, but xarray is a reader which can handle HDF5. It is totally possible to write a reader without a data type if appropriate. See :ref:`base` for an overview of the classes. - the new readers only really have one method that matters, ``.read()``, and will contain all of the previous logic. It should consistently only produce one particular output type. Other attributes of BaseReader (or FileReader) are one-line overrides and mostly provide information rather than functionality; for instance, Intake uses these for recommending readers for a given data instance. - for catalog-producing readers, the output type will be :class:`intake.readers.entry:Catalog`, and the ``.read()`` method will create the Catalog instance and assign readers into it. Module ``intake.readers.catalogs`` contains some patterns to copy. - if using file patterns: the ``DaskCSVPattern`` reader will give an idea of how to implement that in the new framework. - if using V1 plots, dataframe and xarray-producing readers have the ``ToHvPlot`` converter can be used for similar functionality. .. raw:: html ================================================ FILE: docs/source/transforms.rst ================================================ Dataset Transforms ------------------ aka. derived datasets. .. warning:: experimental feature, the API may change. The data sources in ``intake.source.derived`` are not yet declared as top-level named drivers in the package entrypoints. Intake allows for the definition of data sources which take as their input another source in the same directory, so that you have the opportunity to present *processing* to the user of the catalog. The "target" or a derived data source will normally be a string. In the simple case, it is the name of a data source in the same catalog. However, we use the syntax "catalog:source" to refer to sources in other catalogs, where the part before ":" will be passed to :func:`intake.open_catalog`, together with any keyword arguments from ``cat_kwargs``. This can be done by defining classes which inherit from ``intake.source.derived.DerivedSource``, or using one of the pre-defined classes in the same module, which usually need to be passed a reference to a function in a python module. We will demonstrate both. Example ``````` Consider the following *target* dataset, which loads some simple facts about US states from a CSV file. This example is taken from the Intake test suite. .. code-block:: yaml sources: input_data: description: a local data file driver: csv args: urlpath: '{{ CATALOG_DIR }}/cache_data/states.csv' We now show two ways to apply a super-simple transform to this data, which selects two of the dataframe's columns. Class Example ~~~~~~~~~~~~~ The first version uses an approach in which the transform is derived in a data source class, and the parameters passed are specific to the transform type. Note that the driver is referred to by it's fully-qualified name in the Intake package. .. code-block:: yaml derive_cols: driver: intake.source.derived.Columns args: targets: - input_data columns: ["state", "slug"] The source class for this is included in the Intake codebase, but the important part is: .. code-block:: python class Columns(DataFrameTransform): ... def pick_columns(self, df): return df[self._params["columns"]] We see that this specific class inherits from ``DataFrameTransform``, with ``transform=self.pick_columns``. We know that the inputs and outputs are both dataframes. This allows for some additional validation and an automated way to infer the output dataframe's schema that reduces the number of line of code required. The given method does exactly what you might imagine: it takes and input dataframe and applies a column selection to it. Running ``cat.derive_cols.read()`` will indeed, as expected, produce a version of the data with only the selected columns included. It does this by defining the original dataset, appying the selection, and then getting Dask to generate the output. For some datasets, this can mean that the selection is pushed down to the reader, and the data for the dropped columns is never loaded. The user may choose to do ``.to_dask()`` instead, and manipulate the lazy dataframe directly, before loading. Functional Example ~~~~~~~~~~~~~~~~~~ This second version of the same output uses the more generic and flexible ``intake.source.derived.DataFrameTransform``. .. code-block:: yaml derive_cols_func: driver: intake.source.derived.DataFrameTransform args: targets: - input_data transform: "intake.source.tests.test_derived._pick_columns" transform_kwargs: columns: ["state", "slug"] In this case, we pass a reference to a *function* defined in the Intake test suite. Normally this would be declared in user modules, where perhaps those declarations and catalog(s) are distributed together as a package. .. code-block:: python def _pick_columns(df, columns): return df[columns] This is, of course, very similar to the method shown in the previous section, and again applies the selection in the given named argument to the input. Note that Intake does not support including actual code in your catalog, since we would not want to allow arbitrary execution of code on catalog load, as opposed to execution. Loading this data source proceeds exactly the same way as the class-based approach, above. Both Dask and in-memory (Pandas, via ``.read()``) methods work as expected. The declaration in YAML, above, is slightly more verbose, but the amount of code is smaller. This demonstrates a tradeoff between flexibility and concision. If there were validation code to add for the arguments or input dataset, it would be less obvious where to put these things. Barebone Example ~~~~~~~~~~~~~~~~ The previous two examples both did dataframe to dataframe transforms. However, totally arbitrary computations are possible. Consider the following: .. code-block:: yaml barebones: driver: intake.source.derived.GenericTransform args: targets: - input_data transform: builtins.len transform_kwargs: {} This applies ``len`` to the input dataframe. ``cat.barebones.describe()`` gives the output container type as "other", i.e., not specified. The result of ``read()`` on this gives the single number 50, the number of rows in the input data. This class, and ``DerivedDataSource`` and included with the intent as superclasses, and probably will not be used directly often. Execution engine ```````````````` None of the above examples specified explicitly where the compute implied by the transformation will take place. However, most Intake drivers support in-memory containers and Dask; remembering that the input dataset here is a dataframe. However, the behaviour is defined in the driver class itself - so it would be fine to write a driver in which we make different assumptions. Let's suppose, for instance, that the original source is to be loaded from ``spark`` (see the ``intake-spark`` package), the driver could explicitly call ``.to_spark`` on the original source, and be assured that it has a Spark object to work with. It should, of course, explain in its documentation what assumptions are being made and that, presumably, the user is expected to also call ``.to_spark`` if they wished to directly manipulate the spark object. Plugin examples ``````````````` - call `.sel` on xarray datasets `xarray-plugin-transform`_ .. _xarray-plugin-transform: https://github.com/intake/intake-xarray/blob/master/intake_xarray/derived.py#L38 API ``` .. autosummary:: intake.source.derived.DerivedSource intake.source.derived.AliasSource intake.source.derived.GenericTransform intake.source.derived.DataFrameTransform intake.source.derived.Columns .. autoclass:: intake.source.derived.DerivedSource :members: __init__ .. autoclass:: intake.source.derived.GenericTransform :members: __init__ .. autoclass:: intake.source.derived.DataFrameTransform :members: __init__ .. autoclass:: intake.source.derived.Columns :members: __init__ .. raw:: html ================================================ FILE: docs/source/use_cases.rst ================================================ .. _usecases: Use Cases - I want to... ======================== Here follows a list of specific things that people may want to get done, and details of how Intake can help. The details of how to achieve each of these activities can be found in the rest of the detailed documentation. Avoid copy&paste of blocks of code for accessing data ----------------------------------------------------- This is a very common pattern, if you want to load some specific data, to find someone, perhaps a colleague, who has accessed it before, and copy that code. Such a practice is extremely error prone, and cause a proliferation of copies of code, which may evolve over time, with various versions simultaneously in use. Intake separates the concerns of data-source specification from code. The specs are stored separately, and all users can reference the one and only authoritative definition, whether in a shared file, a service visible to everyone or by using the Intake server. This spec can be updated so that everyone gets the current version instead of relying on outdated code. Version control data sources ---------------------------- Version control (e.g., using ``git``) is an essential practice in modern software engineering and data science. It ensures that the change history is recorded, with times, descriptions and authors along with the changes themselves. When data is specified using a well-structured syntax such as YAML, it can be checked into a version controlled repository in the usual fashion. Thus, you can bring rigorous practices to your data as well as your code. If using conda packages to distribute data specifications, these come with a natural internal version numbering system, such that users need only do ``conda update ...`` to get the latest version. "Install" data -------------- Often, finding and grabbing data is a major hurdle to productivity. People may be required to download artifacts from various places or search through storage systems to find the specific thing that they are after. One-line commands which can retrieve data-source specifications or the files themselves can be a massive time-saver. Furthermore, each data-set will typically need its own code to be able to access it, and probably additional software dependencies. Intake allows you to build ``conda`` packages, which can include catalog files referencing online resources, or to include data files directly in that package. Whether uploaded to ``anaconda.org`` or hosted on a private enterprise channel, getting the data becomes a single ``conda install ...`` command, whereafter it will appear as an entry in ``intake.cat``. The conda package brings versioning and dependency declaration for free, and you can include any code that may be required for that specific data-set directly in the package too. Update data specifications in-place ----------------------------------- Individual data-sets often may be static, but commonly, the "best" data to get a job done changes with time as new facts emerge. Conversely, the very same data might be better stored in a different format which is, for instance, better-suited to parallel access in the cloud. In such situations, you really don't want to force all the data scientists who rely on it to have their code temporarily broken and be forced to change this code. By working with a catalog file/service in a fixed shared location, it is possible to update the data source specs in-place. When users now run their code, they will get the latest version. Because all Intake drivers have the same API, the code using the data will be identical and not need to be changed, even when the format has been updated to something more optimised. Access data stored on cloud resources ------------------------------------- Services such as AWS S3, GCS and Azure Datalake (or private enterprise variants of these) are increasingly popular locations to amass large amounts of data. Not only are they relatively cheap per GB, but they provide long-term resilience, metadata services, complex access control patterns and can have very large data throughput when accessed in parallel by machines on the same architecture. Intake comes with integration to cloud-based storage out-of-the box for most of the file-based data formats, to be able to access the data directly in-place and in parallel. For the few remaining cases where direct access is not feasible, the caching system in Intake allows for download of files on first use, so that all further access is much faster. Work with "Big Data" -------------------- The era of Big Data is here! The term means different things to different people, but certainly implies that an individual data-set is too large to fit into the memory of a typical workstation computer (>>10GB). Nevertheless, most data-loading examples available use functions in packages such as ``pandas`` and expect to be able to produce in-memory representations of the whole data. This is clearly a problem, and a more general answer should be available aside from "get more memory in your machine". Intake integrates with ``Dask`` and ``Spark``, which both offer out-of-core computation (loading the data in chunks which fit in memory and aggregating result) or can spread their work over a cluster of machines, effectively making use of the shared memory resources of the whole cluster. Dask integration is built into the majority of the the drivers and exposed with the ``.to_dask()`` method, and Spark integration is available for a small number of drivers with a similar ``.to_spark()`` method, as well as directly with the ``intake-spark`` package. Intake also integrates with many data services which themselves can perform big-data computations, only extracting the smaller aggregated data-sets that *do* fit into memory for further analysis. Services such as SQL systems, ``solr``, ``elastic-search``, ``splunk``, ``accumulo`` and ``hbase`` all can distribute the work required to fulfill a query across many nodes of a cluster. Find the right data-set ----------------------- Browsing for the data-set which will solve a particular problem can be hard, even when the data have been curated and stored in a single, well-structured system. You do *not* want to rely on word-of-mouth to specify which data is right for which job. Intake catalogs allow for self-description of data-sets, with simple text and arbitrary metadata, with a consistent access pattern. Not only can you list the data available to you, but you can find out what exactly that data represents, and the form the data would take if loaded (table versus list of items, for example). This extra metadata is also searchable: you can descend through a hierarchy of catalogs with a single search, and find all the entries containing some particular keywords. You can use the Intake GUI to graphically browse through your available data-sets or point to catalogs available to you, look through the entries listed there and get information about each, or even show a sample of the data or quick-look plots. The GUI is also able to execute searches and browse file-systems to find data artifacts of interest. This same functionality is also available via a command-line interface or programmatically. Work remotely ------------- Interacting with cloud storage resources is very convenient, but you will not want to download large amounts of data to your laptop or workstation for analysis. Intake finds itself at home in the remote-execution world of jupyter and Anaconda Enterprise and other in-browser technologies. For instance, you can run the Intake GUI either as a stand-alone application for browsing data-sets or in a notebook for full analytics, and have all the runtime live on a remote machine, or perhaps a cluster which is co-located with the data storage. Together with cloud-optimised data formats such as parquet, this is an ideal set-up for processing data at web scale. Transform data to efficient formats for sharing ----------------------------------------------- A massive amount of data exists in human-readable formats such as JSON, XML and CSV, which are not very efficient in terms of space usage and need to be parsed on load to turn into arrays or tables. Much faster processing times can be had with modern compact, optimised formats, such as parquet. Intake has a "persist" mechanism to transform any input data-source into the format most appropriate for that type of data, e.g., parquet for tabular data. The persisted data will be used in preference at analysis time, and the schedule for updating from the original source is configurable. The location of these persisted data-sets can be shared with others, so they can also gain the benefits, or the "export" variant can be used to produce an independent version in the same format, together with a spec to reference it by; you would then share this spec with others. Access data without leaking credentials --------------------------------------- Security is important. Users' identity and authority to view specific data should be established before handing over any sensitive bytes. It is, unfortunately, all too common for data scientists to include their username, passwords or other credentials directly in code, so that it can run automatically, thus presenting a potential security gap. Intake does not manage credentials or user identities directly, but does provide hooks for fetching details from the environment or other service, and using the values in templating at the time of reading the data. Thus, the details are not included in the code, but every access still requires for them to be present. In other cases, you may want to require the user to provide their credentials every time, rather that automatically establish them, and "user parameters" can be specified in Intake to cover this case. Establish a data gateway ------------------------ The Intake server protocol allows you fine-grained control over the set of data sources that are listed, and exactly what to return to a user when they want to read some of that data. This is an ideal opportunity to include authorisation checks, audit logging, and any more complicated access patterns, as required. By streaming the data through a single channel on the server, rather than allowing users direct access to the data storage backend, you can log and verify all access to your data. Clear distinction between data curator and analyst roles -------------------------------------------------------- It is desirable to separate out two tasks: the definition of data-source specifications, and accessing and using data. This is so that those who understand the origins of the data and the implications of various formats and other storage options (such as chunk-size) should make those decisions and encode what they have done into specs. It leaves the data users, e.g., data scientists, free to find and use the data-sets appropriate for their work and simply get on with their job - without having to learn about various storage formats and access APIs. This separation is at the very core of what Intake was designed to do. Users to be able to access data without learning every backend API ------------------------------------------------------------------ Data formats and services are a wide mess of many libraries and APIs. A large amount of time can be wasted in the life of a data scientist or engineer in finding out the details of the ones required by their work. Intake wraps these various libraries, REST APIs and similar, to provide a consistent experience for the data user. ``source.read()`` will simply get all of the data into memory in the container type for that source - no further parameters or knowledge required. Even for the curator of data catalogs or data driver authors, the framework established by Intake provides a lot of convenience and simplification which allows each person to deal with only the specifics of their job. Data sources to be self-describing ---------------------------------- Having a bunch of files in some directory is a very common pattern for data storage in the wild. There may or may not be a README file co-located giving some information in a human-readable form, but generally not structured - such files are usually different in every case. When a data source is encoded into a catalog, the spec offers a natural place to describe what that data is, along with the possibility to provide an arbitrary amount of structured metadata and to describe any parameters that are to be exposed for user choice. Furthermore, Intake data sources each have a particular container type, so that users know whether to expect a dataframe, array, etc., and simple introspection methods like ``describe`` and ``discover`` which return basic information about the data without having to load all of it into memory first. A data source hierarchy for natural structuring ----------------------------------------------- Usually, the set of data sources held by an organisation have relationships to one another, and would be poorly served to be provided as a simple flat list of everything available. Intake allows catalogs to refer to other catalogs. This means, that you can group data sources by various facets (type, department, time...) and establish hierarchical data-source trees within which to find the particular data most likely to be of interest. Since the catalogs live outside and separate from the data files themselves, as many hierarchy structures as thought useful could be created. For even more complicated data source meta-structures, it is possible to store all the details and even metadata in some external service (e.g., traditional SQL tables) with which Intake can interact to perform queries and return particular subsets of the available data sources. Expose several data collections under a single system ----------------------------------------------------- There are already several catalog-like data services in existence in the world, and some organisation may have several of these in-house for various different purposes. For example, an SQL server may hold details of customer lists and transactions, but historical time-series and reference data may be held separately in archival data formats like parquet on a file-storage system; while real-time system monitoring is done by a totally unrelated system such as Splunk or elastic search. Of course, Intake can read from various file formats and data services. However, it can also interpret the internal conception of data catalogs that some data services may have. For example, all of the tables known to the SQL server, or all of the pre-defined queries in Splunk can be automatically included as catalogs in Intake, and take their place amongst the regular YAML-specified data sources, with exactly the same usage for all of them. These data sources and their hierarchical structure can then be exposed via the graphical data browser, for searching, selecting and visualising data-sets. Modern visualisations for all data-sets --------------------------------------- Intake is integrated with the comprehensive ``holoviz`` suite, particularly ``hvplot``, to bring simple yet powerful data visualisations to any Intake data source by using just one single method for everything. These plots are interactive, and can include server-side dynamic aggregation of very large data-sets to display more data points than the browser can handle. You can specify specific plot types right in the data source definition, to have these customised visualisations available to the user as simple one-liners known to reveal the content of the data, or even view the same visuals right in the graphical data source browser application. Thus, Intake is already an all-in-one data investigation and dashboarding app. Update data specifications in real time --------------------------------------- Intake data catalogs are not limited to reading static specification from files. They can also execute queries on remote data services and return lists of data sources dynamically at runtime. New data sources may appear, for example, as directories of data files are pushed to a storage service, or new tables are created within a SQL server. Distribute data in a custom format ---------------------------------- Sometimes, the well-known data formats are just not right for a given data-set, and a custom-built format is required. In such cases, the code to read the data may not exist in any library. Intake allows for code to be distributed along with data source specs/catalogs or even files in a single ``conda`` package. That encapsulates everything needed to describe and use that particular data, and can then be distributed as a single entity, and installed with a one-liner. Furthermore, should the few builtin container types (sequence, array, dataframe) not be sufficient, you can supply your own, and then build drivers that use it. This was done, for example, for ``xarray``-type data, where multiple related N-D arrays share a coordinate system and metadata. By creating this container, a whole world of scientific and engineering data was opened up to Intake. Creating new containers is not hard, though, and we foresee more coming, such as machine-learning models and streaming/real-time data. Create Intake data-sets from scratch ------------------------------------ If you have a set of files or a data service which you wish to make into a data-set, so that you can include it in a catalog, you should use the set of functions ``intake.open_*``, where you need to pick the function appropriate for your particular data. You can use tab-completion to list the set of data drivers you have installed, and find others you may not yet have installed at :ref:`plugin-directory`. Once you have determined the right set of parameters to load the data in the manner you wish, you can use the source's ``.yaml()`` method to find the spec that describes the source, so you can insert it into a catalog (with appropriate description and metadata). Alternatively, you can open a YAML file as a catalog with ``intake.open_catalog`` and use its ``.add()`` method to insert the source into the corresponding file. .. raw:: html ================================================ FILE: docs/source/user2.rst ================================================ .. catalog_user: Catalog User ============ So someone has sent you an Intake URL or other way to load a catalog. What happens next? Let's do the simplest thing and load a public catalog .. code-block:: python cat = intake.from_yaml_file("s3://mymdtemp/intake_1.yaml", anon=True) (this is the same catalog as is made in the example `tutorial noteboook`_) .. _tutorial noteboook: https://github.com/intake/intake/blob/master/examples/Take2.ipynb Displaying the catalog shows that it has four named datasets and some automatically populated "user parameters". Interesting attributes of the catalog are: - ``cat.data``: full description of the original datasets .. code-block:: python {'7e0b327a50eef58d': DataDescription type intake.readers.datatypes:CSV kwargs {'metadata': {}, 'storage_options': {'anon': True}, 'url': '{CATALOG_DIR}/intake_1.csv'}} - ``cat.entries``: readers, ways to load the data .. code-block:: python {'capitals': Entry for reader: intake.readers.convert:Pipeline kwargs: {'out_instances': ['pandas:DataFrame', 'pandas:DataFrame', 'pandas:DataFrame', 'pandas:DataFrame'], 'steps': [ ['{data(tute)}', [], {}], ['{func(intake.readers.transform:Method)}', [], {'method_name': 'a'}], ['{func(intake.readers.transform:Method)}', [], {'method_name': 'str'}], ['{func(intake.readers.transform:Method)}', [], {'method_name': 'capitalize'}]]} producing: pandas:DataFrame, 'inverted': Entry for reader: intake.readers.convert:Pipeline kwargs: {'out_instances': ['pandas:DataFrame', 'pandas:DataFrame'], 'steps': [ ['{data(tute)}', [], {}], ['{func(intake.readers.transform:Method)}', [], {'args': ['b'], 'ascending': False, 'method_name': 'sort_values'}]]} producing: pandas:DataFrame, 'multi': Entry for reader: intake.readers.convert:Pipeline kwargs: {'out_instances': ['pandas:DataFrame', 'pandas:DataFrame'], 'steps': [ ['{data(tute)}', [], {}], ['{func(intake.readers.transform:Method)}', [], {'c': '{data(capitals)}', 'method_name': 'assign'}]]} producing: pandas:DataFrame, 'tute': Entry for reader: intake.readers.readers:PandasCSV kwargs: {'data': '{data(7e0b327a50eef58d)}'} producing: pandas:DataFrame} - ``cat.aliases``: names to associate with readers or data (these are the ones used with tab-completion) .. code-block:: python {'capitals': 'capitals', 'inverted': 'inverted', 'multi': 'multi', 'tute': 'tute'} - ``cat.user_parameters``: values that can be used in templated values (see below) .. code-block:: python {'CATALOG_PATH': 's3://mymdtemp/intake_1.yaml', 'CATALOG_DIR': 's3://mymdtemp', 'STORAGE_OPTIONS': {'anon': True}} You can even get an overall view of everything in the catalog using ``cat.to_dict()``, which gives you back essentially the same information as was contained in the YAML file we read the catalog from. Also notice, that there is metadata associated with the whole catalog, and each of the data and reader descriptions. All the readers depend on the one dataset ("multi" depends on it twice) and all are Pipelines (with "steps") except "tute". Key Concepts ------------ A few definitions that will help you: - data: a set of numbers of various forms, which can be used to infer information about some domain - dataset: a specific delimited amount of data, often a single file, a directory of files or a single request or query to some service. The output of any Intake reader is also a "dataset", as represented in a live python session - data: the basic information needed to uniquely identify a dataset, such as data type, URL/paths, server location, query. Intake supports many data types (:ref:`data`). A description ought to also contain descriptive information in its metadata. - reader: how a given dataset should be handled/loaded. This is more specific than the data itself, since there may be many different ways to read the data. For instance, CSVs are a very common and simple data format, and virtually all (table-oriented) data packages can read them. - pipeline: a sequence of operations on a dataset. In Intake, this is just a type of reader, although it is possible to refer to the output of any particular stage. - catalog: a collection of datasets and their reader descriptions. Each dataset may be referred to by multiple readers, and a reader may refer to multiple datasets, although the latter is less common (think of JOIN operations). - templates, user-parameters: in the catalog definition of the one dataset, you will notice special syntax for part of the URL value to be filled in. See below for how to work with this. Reader API ---------- Before accessing any of the entries in a catalog, you should introspect them to see if it is what you are after. There should be descriptive text, other metadata and of course the contents of the data/readers, as shown above. It is important to note, that extracting readers (the next step) already comes with security implications, such as evaluating environment variables and making imports. The "allow_*" keys in the intake configuration, ``intake.conf`` define what is generally allowed. As an end-user, you will generally interact with readers. Get them from the catalog by attribute access or item access; the latter is required where the name is not a valid python identifier or conflicts with a method. The following two line are exactly equivalent: .. code-block:: reader = cat.tute reader = cat["tute"] reader.pprint() {'kwargs': {'data': {'url': 's3://mymdtemp/intake_1.csv', 'storage_options': {'anon': True}, 'metadata': {}}}, 'metadata': {}, 'output_instance': 'pandas:DataFrame'} .. note:: We will work on the best way to represent the various instances, especially in the notebook. For the time being, you can always use the ``.pprint()`` method, or introspect the instance's attributes. We notice that this is NOT exactly the same as the entry in the original catalog with name "tute". In particular: the reader is a concrete instance of a subclass of :class:`intake.readers.readers.BaseReader`, it contains the data definition it referenced and the URL of which has been expanded to the full "s3://..". The most obvious thing to do to a reader is read: this is, after all, what they are for. We already know to expect an output type a pandas DataFrame. ``reader.doc()`` provides the docstring of the target function, in this case ``read_csv()``. You can pass extra or override arguments, with exactly the same names and meaning as the original docstring (some readers might provide extra functionality or possibilities). .. code-block:: python reader.read() Unnamed: 0 a b 0 0 ho 4 1 1 hi 5 reader(index_col=[0]).read() a b 0 ho 4 1 hi 5 For large datasets, you may try ``.discover()`` instead, which is generally a small subset of the data, depending on the format and library. For small datasets like this one, you get exactly the same output. Templates --------- Returning to the mysterious "s3://" URL in the reader instance above. This was created from the URL "{CATALOG_DIR}/intake_1.csv" using templating. You may recall that the catalog had a user_parameter of this name, whose value was auto-populated from the URL we used to read the catalog file. This means that the data file and catalog describing it could be moved together to a new location without having to edit the catalog. On the other hand, if the URL were not templated, moving/copying the catalog would still refer to the original data location (sometimes this is what you want). This particular user_parameter was global to the catalog, and to assign a new value before templating, you would do .. code-block:: python cat2 = cat(CATALOG_DIR="new_value") (so ``cat2.tute`` would not have a different data URL and no longer load!). It is also possible to have parameters associated with the data description and/or specific readers, and for any parameter to be used in multiple places. They can also have specific types, defaults and constraints/choices. If a template refers to a parameter that is missing or has no value set, it will be left unchanged, and the data in question will probably not load. ================================================ FILE: docs/source/walkthrough2.rst ================================================ Creator Walkthrough =================== As soon as you have used a catalog, you may wonder how to create the - look no further. EVen if you don't make catalogs, you may find the data and reader class recommenders useful. Preamble -------- Intake is a bunch of _readers_. A reader is a class that encodes how to load some particular data, including all the arguments that will be passed to some third-party package. Thus, you can encode anything from the simplest ``pd.read_csv`` call to far more complex things. You can also act on readers, to record a set of operations known as a Pipeline. A Pipeline is just a special kind of reader. Catalogs contain readers and data definitions. Readers can and should be saved in Catalogs, which can in the simplest case be saved as text files, but can also be generated from many data services. Catalogs and readers are the central themes of intake. .. note:: "Load" means make a machine representations: many data containers may be lazy, where you get an object on which you can act, but the data is only read on demand for actions that require it. Simple Example -------------- Let's consider a typical Pandas line to read a CSV file. Although there are many ways to load data in python, this is one of the most common. .. code-block:: python import pandas as pd url = "s3://mymdtemp/intake_1.csv" df = pd.read_csv(url, storage_options={"anon": True}, usecols=[1, 2]) Simple enough, so long as you have ``fsspec`` and ``s3fs`` installed. Note the use of extra options to the storage backend (this file is public) and up-front column selection. To encode this with intake, you can do .. code-block:: python import intake reader = intake.reader_from_call( 'df = pd.read_csv(url, storage_options={"anon": True}, usecols=[1, 2])') Where you could have used ``_i``, the IPython shorthand for "the last line of input" instead of the copy/pasted line. The "reader" object encodes all the things we asked for, and we can execute the process by calling ``reader.read()`` to get the same result as before. So what's the point? We can put this reader into a catalog: .. code-block:: python cat = intake.entry.Catalog() cat["tute"] = reader cat.to_yaml_file("intake_1.yaml") where the path could be anyplace you can write to, such as a local file. This can be the whole of the "catalog producer" workflow. A public version has been put alongside the CSV, so as the data consumer, you can read this wherever you are. The following could be the whole of a data consumer workflow: .. code-block:: python import intake cat = intake.from_yaml_file("s3://mymdtemp/intake_1.yaml", anon=True) cat.tute.read() Note that ``cat.tute`` has a lot of optional metadata that we have not filled out. It also provides documentation of the function it will call at read time (``cat.tute.doc()``), in case you want to add or change arguments. The point is not to know every time which specific dataset you need (although that happens too), but a way to view all of your available data and find the right one for your job before loading it. Similarly, we could have started with the URL alone .. code-block:: python intake.datatypes.recommend("s3://mymdtemp/intake_1.csv", storage_options={"anon": True}) Which says that this URL likely refers to a CSV file. In this case, that is not surprising, but in many cases, it can be useful to have Intake guess the file type for us. **Summary**: In this section we saw one way to easily create a data loading spec ("reader"), save it into a catalog and then load this data using the catalog. This is data distribution without any server. Slightly less Simple -------------------- Let's do trivial transforms to our trivial dataset. Continuing from above: .. code-block:: python cat["capitals"] = reader.a.str.capitalize() cat["inverted"] = reader.sort_values("b", ascending=False) cat.to_yaml_file("intake_1.yaml") Again, this could be persisted anywhere, but the path above includes all three datasets: .. code-block:: python import intake cat = intake.from_yaml_file("s3://mymdtemp/intake_1.yaml", anon=True) list(cat) # -> ['capitals', 'inverted', 'tute'] Now we have three datasets all based off the same original file. Investigating ``cat.data``, you can see that there is exactly one definition: a CSV with the URL as defined above, and investigating ``cat.entries``, you can see how the three things reference it and one-another. We can also make readers that depend on multiple data sources: .. code-block:: python cat["multi"] = cat.tute.assign(c=cat.capitals) (Since we only have one base dataset, this depends on itself, but there is still only one value in ``cat.data``, the original CSV) .. note:: We made derived datasets by knowing and calling the pandas API explicitly. These methods are available by tab-completion (in a jupyter session, for example). Or one might interrogate the ``.transform`` attribute of any reader to know which methods are defined in Intake, or expected to be available for the given type (a DataFrame in this case). **Summary** : here we showed that you can create processing pipelines from datasets and save these derived datasets as new definitions. The syntax is the same as whatever package you intend to do the actual processing. Multiple Readers ---------------- In the previous examples, we knew we were starting with a CSV and wanted to make pipelines using pandas. Often, life is not that simple! It can take work to figure out where to look for data, and then what type that data is and which engine is best suited to work with it. In fact, the answer to that might depend on a number of factors and not be the same for all users on all days. So, Intake allows you to make multiple pipelines from the same data with different engines and leave the choice of which to use to runtime. It will also help you find pathways to transform your data between frameworks to get to your desired outcome. There is only one data entity in our example mini-catalog, so let's grab it: .. code-block:: python cat = intake.from_yaml_file("s3://mymdtemp/intake_1.yaml", anon=True) key = list(cat.data)[0] data = cat[key] .. note:: The key of this in ``cat.data`` is a hashed hex string. We could have given this data entry a name, but it was auto-added to the catalog when assing the "tute" reader. This is a ``CSV`` data instance, a subclass of ``intake.readers.datatypes.BaseData``. It can be read with multiple readers, since CSVs are so common. ``data.possible_readers`` lists the classes that can read this grouped by whether they can be imported, and ``data.possible_outputs`` gives the expected output instance from each importable reader. In both case, more specific readers (for CSVs) come before the less specific ones (that read any file). We could instantiate these readers directly, or call the ``.to_reader`` method to pick one. The following two are equivalent, and pick dask.dataframe as the backend to read the data (assuming it is installed). .. code-block:: python intake.readers.readers.DaskCSV(data) data.to_reader("dask") # this searches the outputs for something matching "dask" Both lines produce a DaskCSV reader, which we can also put in a catalog, or just ``.read()`` it to get a ``dask.dataframe.DataFrame`` output. We could have done the same for Ray, Spark or DuckDB (and more to come, like cuDF, Modin, Polars, and that's just for dataframes). **Summary**: a dataset can be read with multiple reader types, and we can save prescriptions in a catalog for each, with various sets of parameters is desired, but still have the ability to go back to the data definition and pick something else. Conversions ----------- Intake can handle many filetypes (see ``intake.datatypes``) and many readers with various engines/packages (see ``intake.readers.readers``). These are, by design, simple to write, and many more will be coming to fill out the data loading space. However, you may well find yourself needing to convert from one representation to another. For example, you may wish to use DuckDB for efficient querying of remote Parquet files, but then want the output as a Pandas dataframe for building an ML model. You can, of course, just save the DuckDB reader with the query, and convert to Pandas in every session, but wouldn't it be nice to encode that you mean to do this conversion = and then you can chain further operations with the Pandas API, if you wish. Intake supports the following web of conversions, and more coming all the time. It's too complex to see the details! .. image:: ./_static/images/out.png :alt: Conversions Web Finding the specific conversion search might look something like .. code-block:: python data = intake.datatypes.Parquet("my.parq") reader = data.to_reader("Duck") reader.transform Where from the repr we see that a converter ``DuckToPandas`` exists and does what you expect. Or, you can often find a whole chain of conversions to get to where you want to go; starting with either a URL (and any storage_options), data instance or reader instance. For the reader above, let's say we want to get an image (PNG) representation: .. code-block:: python intake.auto_pipeline(reader, outtype="PNG") produces a pipeline going DuckDB->Pandas->matplotlib Figure->PNG file. If you try to run it it will fail, saying that the output node (the last one) needs a URL. So you would actually do the following: .. code-block:: python output = intake.auto_pipeline(reader, outtype="PNG") output(url="out.png").read() Where obviously you could make extra arguments to the ToMatplotlib stage to customise the graph. Any arguements or any node in the pipeline can be changed before running or persisting in a catalog, and any pipeline from a catalog can have its arguments overridden at runtime, if desired. The final PNG file would be a nice addition to the metadata of the original data prescription, see ``intake.readers.metadata_fields`` for suggested field names and descriptions. .. note:: There are many classes derived from ``BaseConverter``, and by convention those we call "converters" keep the same data but in different representations, "transforms" change the data but keep the same representations, and "output" produce side-effects and return a BaseData instance. These rules are loose and may be violated. **Summary**: Intake handles not only many data types and compute engines, but knows how to convert between them, providing some handy utilities for guessing the best pipeline to get to a given output. Complex Example --------------- The following code performs a rather typical workflow, recreating the "persist" functionality in V1 (using only standard blocks, no special code). This is somewhat verbose and explicit, for the sake of clarity. .. note:: Saving local copies of files is more common, and can usually be achieved by adding "simplecache::" (and a defined local directory) to the URL for readers that use `fsspec``. However, some data sources are not files, and of course it is often a good idea to save a more efficient format of the data, as in the case here. .. code-block:: python from intake.readers.readers import Condition, PandasCSV, PandasParquet, FileExistsReader fn = f"{tmpdir}/file.parquet" data = intake.readers.datatypes.CSV(url=dataframe_file) part = PandasCSV(data) output = part.PandasToParquet(url=fn).transform(PandasParquet) data2 = intake.readers.datatypes.Parquet(url=fn) cached = PandasParquet(data=data2) reader2 = Condition(cached, if_false=output, condition=FileExistsReader(data2)) The pipeline can be described as: - there is a CSV file, ``dataframe_file`` - there may be a parquet version of this, ``fn`` - if the parquet file does not exist, load the CSV using pandas, save it to parquet and load that - if the parquet file already exists, load that without looking at the CSV. There are of course many ways that one might achieve this and more complex "conditions" for when to run the conversion pipeline. However, the ``reader2`` object encodes the whole thing, and can be safely stored in a catalog. A user can then use this standard condition, choose to remake the parquet, or just load the CSV without accessing the parquet at all. It would be reasonable to update the metadata of ``data`` or the readers to show the expected columns types and row count (if they are not expected to change). **Summary**: you can branch and join pipelines and save the whole complicated tree in catalogs, allowing complex patterns like conditional caching. Extracting User Parameters -------------------------- To come ================================================ FILE: examples/Take2.ipynb ================================================ { "cells": [ { "cell_type": "code", "execution_count": null, "id": "f5f6d518-dfe8-4651-b315-44bb55b885be", "metadata": {}, "outputs": [], "source": [ "import intake\n", "import pandas as pd\n", "import hvplot.pandas" ] }, { "cell_type": "markdown", "id": "718dfddd-14eb-4a4e-ad9c-19f0c9f10f3c", "metadata": {}, "source": [ "#### Guess reader from existing code" ] }, { "cell_type": "code", "execution_count": null, "id": "1eb57314-f056-465e-ba75-22d5d0ca232b", "metadata": {}, "outputs": [], "source": [ "url = \"s3://mymdtemp/intake_1.csv\"" ] }, { "cell_type": "code", "execution_count": null, "id": "31dde28e-718d-462e-b9bf-30601db948ca", "metadata": {}, "outputs": [], "source": [ "df = pd.read_csv(url, storage_options={\"anon\": True}, usecols=[1, 2])" ] }, { "cell_type": "code", "execution_count": null, "id": "1daac4ed-e5af-4a63-8a12-d97022e90d99", "metadata": {}, "outputs": [], "source": [ "reader = intake.reader_from_call(_i)" ] }, { "cell_type": "code", "execution_count": null, "id": "bda407ee-f26c-4788-b367-1fc3f370ccbb", "metadata": {}, "outputs": [], "source": [ "reader" ] }, { "cell_type": "code", "execution_count": null, "id": "3acf782f-b887-4da7-a8b3-78463c52c8eb", "metadata": {}, "outputs": [], "source": [ "reader.kwargs" ] }, { "cell_type": "markdown", "id": "c718b8bf-0489-4e29-b31c-b354f6373224", "metadata": {}, "source": [ "#### Or guess from the URL alone" ] }, { "cell_type": "code", "execution_count": null, "id": "9900e6f4-b896-40ad-bbaf-cb2563ae8252", "metadata": {}, "outputs": [], "source": [ "# uses URL alone, but can also match on magic bytes\n", "intake.datatypes.recommend(url)" ] }, { "cell_type": "code", "execution_count": null, "id": "771538c7-1467-4b1c-82ef-a9ace372465c", "metadata": {}, "outputs": [], "source": [ "data = intake.readers.datatypes.CSV(url, storage_options={\"anon\": True})" ] }, { "cell_type": "markdown", "id": "4d9bd954-7c31-4638-80bf-ffc4cc7fb452", "metadata": {}, "source": [ "#### \"What can read this?\"" ] }, { "cell_type": "code", "execution_count": null, "id": "4dac3e18-f60a-4b91-946b-36d766a73bfe", "metadata": {}, "outputs": [], "source": [ "data.possible_outputs" ] }, { "cell_type": "code", "execution_count": null, "id": "654e74d0-aa6a-4211-a04d-4fc6886c58df", "metadata": {}, "outputs": [], "source": [ "data.possible_readers" ] }, { "cell_type": "code", "execution_count": null, "id": "4ddebb3c-745d-4294-9dee-583214075df7", "metadata": {}, "outputs": [], "source": [ "# same reader as original\n", "# reader = data.to_reader(\"pandas:DataFrame\")\n", "reader = intake.readers.readers.PandasCSV(data)" ] }, { "cell_type": "markdown", "id": "5e7ae6c4-6db4-4c42-b076-ea4225ba08df", "metadata": {}, "source": [ "```python\n", "class PandasCSV(Pandas):\n", " implements = {datatypes.CSV}\n", " func = \"pandas:read_csv\"\n", " url_arg = \"filepath_or_buffer\"\n", "\n", " def discover(self, **kw):\n", " kw[\"nrows\"] = 10\n", " kw.pop(\"skipfooter\", None)\n", " kw.pop(\"chunksize\", None)\n", " return self.read(**kw)\n", "```" ] }, { "cell_type": "markdown", "id": "dc10228a-ae60-4551-a2a0-81e399d51130", "metadata": {}, "source": [ "#### Reader API" ] }, { "cell_type": "code", "execution_count": null, "id": "d4d9651d-ef91-4611-b5ae-ee3d762e7196", "metadata": {}, "outputs": [], "source": [ "reader.read()" ] }, { "cell_type": "code", "execution_count": null, "id": "da8e7b50-f7f5-4520-ae44-3c2e5d6a91ad", "metadata": {}, "outputs": [], "source": [ "print(reader.doc())" ] }, { "cell_type": "code", "execution_count": null, "id": "98112835-41a3-48b5-bc92-7e923c263c38", "metadata": {}, "outputs": [], "source": [ "# known transforms and what they make\n", "reader.transform" ] }, { "cell_type": "code", "execution_count": null, "id": "f9d0ff3e-1723-4b8a-b549-7ad703c56cfa", "metadata": {}, "outputs": [], "source": [ "# but have access to full DataFrame API\n", "dir(reader)" ] }, { "cell_type": "code", "execution_count": null, "id": "4d1fbfc6-70b9-4d84-80f3-fca293d8860f", "metadata": {}, "outputs": [], "source": [ "# or \"pd\" namespace (useful for some packages)\n", "reader.pd" ] }, { "cell_type": "markdown", "id": "08249340-d143-4463-b534-6b6ef83f6f79", "metadata": {}, "source": [ "#### So lets make a catalog and a pipeline using pandas syntax" ] }, { "cell_type": "code", "execution_count": null, "id": "843a8a67-5ac8-4128-b8fc-5a897d344578", "metadata": {}, "outputs": [], "source": [ "cat = intake.entry.Catalog()\n", "cat[\"tute\"] = reader\n", "cat[\"capitals\"] = reader.a.str.capitalize()\n", "cat[\"inverted\"] = reader.sort_values(\"b\", ascending=False)\n", "cat[\"multi\"] = cat.tute.assign(c=cat.capitals) # <- uses multiple readers" ] }, { "cell_type": "code", "execution_count": null, "id": "7ccf12af-dd3b-429f-be8b-a7fcc749c7a1", "metadata": {}, "outputs": [], "source": [ "reader.a.str.capitalize()" ] }, { "cell_type": "code", "execution_count": null, "id": "b1eb4728-eda8-467d-a4f0-be280309cc0f", "metadata": {}, "outputs": [], "source": [ "# what gets stored in the catalog entry?\n", "cat.entries[\"multi\"].kwargs" ] }, { "cell_type": "code", "execution_count": null, "id": "9393b669-9489-41fb-ad2f-c98970410770", "metadata": {}, "outputs": [], "source": [ "cat" ] }, { "cell_type": "code", "execution_count": null, "id": "a8a123f6-17cd-404b-acd4-aef832a6e544", "metadata": {}, "outputs": [], "source": [ "cat.tute.read()" ] }, { "cell_type": "code", "execution_count": null, "id": "18d0bb78-d902-4689-9586-0ebc1686af32", "metadata": {}, "outputs": [], "source": [ "cat.data # just one data item" ] }, { "cell_type": "markdown", "id": "4c2a6379-1bb6-4258-996c-823ffc928eb9", "metadata": {}, "source": [ "#### To and from catalog file, which you can put anywhere" ] }, { "cell_type": "code", "execution_count": null, "id": "ca865c7e-a271-4ec9-a44d-efbd68138da0", "metadata": {}, "outputs": [], "source": [ "cat.to_yaml_file(\"intake_1.yaml\")" ] }, { "cell_type": "code", "execution_count": null, "id": "7b9df0f9-a86a-4b73-9a1f-e355ab5be230", "metadata": {}, "outputs": [], "source": [ "# a \"shared\" one I prepared for everyone\n", "cat = intake.from_yaml_file(\"s3://mymdtemp/intake_1.yaml\", anon=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "529a98be-c500-42c4-8b8e-fc8d535d60f4", "metadata": {}, "outputs": [], "source": [ "# yes, you have completion\n", "cat.tute" ] }, { "cell_type": "code", "execution_count": null, "id": "24e83c33-3ba7-4012-ac08-ac3b40521c42", "metadata": {}, "outputs": [], "source": [ "cat.inverted.read()" ] }, { "cell_type": "markdown", "id": "e3b35a6f-8583-421a-8321-65a62bcf0777", "metadata": {}, "source": [ "#### And now you can go about your work; but some convenience functions might still be useful." ] }, { "cell_type": "code", "execution_count": null, "id": "a27fec42-c46a-442a-9574-83b4d312223f", "metadata": {}, "outputs": [], "source": [ "# add arguments to make a reader you can persist\n", "cat.inverted.ToHvPlot(explorer=True).read()" ] }, { "cell_type": "code", "execution_count": null, "id": "c7b89802-4b83-4587-9103-416ae6423100", "metadata": {}, "outputs": [], "source": [ "cat.inverted.ToMatplotlib.read()" ] }, { "cell_type": "code", "execution_count": null, "id": "68c24625-9aab-405a-b2bd-8b6a7a43bb8d", "metadata": {}, "outputs": [], "source": [ "# you can even have Intake guess the whole pipeline\n", "intake.auto_pipeline(data, \"PNG\", avoid=\"Geo\")" ] }, { "cell_type": "markdown", "id": "50504f5b-622c-43f7-ae68-5e71730bc2ab", "metadata": {}, "source": [ "#### But pandas was not the only engine that can work on this data. We can play with the API or make more readers to persist in the catalog." ] }, { "cell_type": "code", "execution_count": null, "id": "c684789c-0038-4f12-8668-640f3f5553c8", "metadata": {}, "outputs": [], "source": [ "data" ] }, { "cell_type": "code", "execution_count": null, "id": "31a9d94c-5025-4de2-99f8-eb966afa26e4", "metadata": {}, "outputs": [], "source": [ "data.to_reader(\"dask\").read()" ] }, { "cell_type": "code", "execution_count": null, "id": "7a4a9622-0b1f-4bab-887e-0e64e4a5098e", "metadata": {}, "outputs": [], "source": [ "data.to_reader(\"ray\").read()" ] }, { "cell_type": "code", "execution_count": null, "id": "d481550f-4bda-4864-9b29-c163301b1f36", "metadata": {}, "outputs": [], "source": [ "# dask-on-ray!\n", "data.to_reader(\"dask\").DaskToRay.read()" ] }, { "cell_type": "code", "execution_count": null, "id": "d00b8b13-6478-4691-b5d8-b76cde9cecd5", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "language_info": { "name": "python", "pygments_lexer": "ipython3" } }, "nbformat": 4, "nbformat_minor": 5 } ================================================ FILE: intake/__init__.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import os try: from intake._version import __version__ except ImportError: __version__ = "2.dev" # fallback # legacy immediate imports from intake.utils import import_name, logger from intake.catalog.base import VersionError from intake.source import registry from intake.config import conf from intake.readers import ( BaseData, reader_from_call, recommend, BaseReader, BaseConverter, Pipeline, auto_pipeline, path, DataDescription, ReaderDescription, BaseUserParameter, SimpleUserParameter, user_parameters, transform, output, catalogs, entry, datatypes, ) from intake.readers.entry import Catalog import intake.readers.importlist # do this last, as it triggers more imports needing intake # legacy on-demand imports imports = { "DataSource": "intake.source.base:DataSource", "Schema": "intake.source.base:Schema", "load_combo_catalog": "intake.catalog.default:load_combo_catalog", "gui": "intake.interface:instance", "interface": "intake.interface", "cat": "intake.catalog:builtin", "output_notebook": "intake.interface:output_notebook", "register_driver": "intake.source:register_driver", "unregister_driver": "intake.source:unregister_driver", } from_yaml_file = entry.Catalog.from_yaml_file def __getattr__(attr): """Lazy attribute propagator Defers inputs of functions until they are needed, according to the contents of the ``imports`` (submodules and classes) and ``openers`` (functions which instantiate data sources directly) dicts. All keys in ``openers`` must start with "open_", else they will be ignored. """ gl = globals() if attr == "__all__": return __dir__() if attr in gl: return gl[attr] if attr in imports: dest = imports[attr] gl[attr] = import_name(dest) return gl[attr] if attr[:5] == "open_": if attr[5:] in registry.drivers.enabled_plugins(): driver = registry[attr[5:]] # "open_..." return driver else: registered_methods = [f"open_{driver}" for driver in registry.drivers.enabled_plugins()] raise AttributeError( f"Unknown open method '{attr}'. " "Do you need to install a new driver from the plugin directory? " "https://intake.readthedocs.io/en/latest/plugin-directory.html\n" f"Registered opener methods: {registered_methods}" ) raise AttributeError(attr) def __dir__(*_, **__): openers = ["open_" + name for name in registry.drivers.enabled_plugins()] return sorted(list(globals()) + list(imports) + openers) def open_catalog(uri=None, **kwargs): """Create a Catalog object *New in V2*: if the URL is a single file, and loading it as a V1 catalog fails because of the stated version, it will be opened again as a V2 catalog. This will mean reading the file twice, so calling ``from_yaml_file`` directly ie better. Can load YAML catalog files, connect to an intake server, or create any arbitrary Catalog subclass instance. In the general case, the user should supply ``driver=`` with a value from the plugins registry which has a container type of catalog. File locations can generally be remote, if specifying a URL protocol. The default behaviour if not specifying the driver is as follows: - if ``uri`` is a single string ending in "yml" or "yaml", open it as a catalog file - if ``uri`` is a list of strings, a string containing a glob character ("*") or a string not ending in "y(a)ml", open as a set of catalog files. In the latter case, assume it is a directory. - if ``uri`` begins with protocol ``"intake:"``, connect to a remote Intake server - if ``uri`` is ``None`` or missing, create a base Catalog object without entries. Parameters ---------- uri: str or pathlib.Path Designator for the location of the catalog. kwargs: passed to subclass instance, see documentation of the individual catalog classes. For example, ``yaml_files_cat`` (when specifying multiple uris or a glob string) takes the additional parameter ``flatten=True|False``, specifying whether all data sources are merged in a single namespace, or each file becomes a sub-catalog. See also -------- intake.open_yaml_files_cat, intake.open_yaml_file_cat, intake.open_intake_remote """ driver = kwargs.pop("driver", None) if isinstance(uri, os.PathLike): uri = os.fspath(uri) if driver is None: if uri: if (isinstance(uri, str) and "*" in uri) or ( (isinstance(uri, (list, tuple))) and len(uri) > 1 ): # glob string or list of files/globs driver = "yaml_files_cat" elif isinstance(uri, (list, tuple)) and len(uri) == 1: uri = uri[0] if "*" in uri[0]: # single glob string in a list driver = "yaml_files_cat" else: # single filename in a list driver = "yaml_file_cat" elif isinstance(uri, str): # single URL if uri.startswith("intake:"): # server driver = "intake_remote" else: if uri.endswith((".yml", ".yaml")): driver = "yaml_file_cat" else: uri = uri.rstrip("/") + "/*.y*ml" driver = "yaml_files_cat" else: raise ValueError("URI not understood: %s" % uri) else: # empty cat driver = "catalog" if "_file" not in driver: kwargs.pop("fs", None) if driver not in registry: raise ValueError( f"Unknown catalog driver '{driver}'. " "Do you need to install a new driver from the plugin directory? " "https://intake.readthedocs.io/en/latest/plugin-directory.html\n" f"Current registry: {list(sorted(registry))}" ) try: return registry[driver](uri, **kwargs) except VersionError: # warn that we are switching to V2? The file will be read twice return from_yaml_file(uri, **kwargs) ================================================ FILE: intake/catalog/__init__.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from .base import Catalog from .default import load_combo_catalog from .local import EntrypointsCatalog, MergedCatalog def _make_builtin(): return MergedCatalog( [EntrypointsCatalog(), load_combo_catalog()], name="builtin", description="Generated from data packages found on your intake search path", ) def __getattr__(name): """Only make the builtin catalog on request""" global builtin if name == "builtin": builtin = _make_builtin() return builtin raise AttributeError(name) ================================================ FILE: intake/catalog/base.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import keyword import logging import re import time from ..source.base import DataSource, DataSourceBase, NoEntry from .utils import reload_on_change logger = logging.getLogger("intake") class VersionError(Exception): ... class Catalog(DataSource): """Manages a hierarchy of data sources as a collective unit. A catalog is a set of available data sources for an individual entity (remote server, local file, or a local directory of files). This can be expanded to include a collection of subcatalogs, which are then managed as a single unit. A catalog is created with a single URI or a group of URIs. A URI can either be a URL or a file path. Each catalog in the hierarchy is responsible for caching the most recent refresh time to prevent overeager queries. Attributes ---------- metadata : dict Arbitrary information to carry along with the data source specs. """ # emulate a DataSource container = "catalog" name = "catalog" auth = None def __init__( self, entries=None, name=None, description=None, metadata=None, ttl=60, getenv=True, getshell=False, persist_mode="default", storage_options=None, user_parameters=None, ): """ Parameters ---------- entries : dict, optional Mapping of {name: entry} name : str, optional Unique identifier for catalog. This takes precedence over whatever is stated in the cat file itself. Defaults to None. description : str, optional Description of the catalog. This takes precedence over whatever is stated in the cat file itself. Defaults to None. metadata: dict Additional information about this data ttl : float, optional Lifespan (time to live) of cached modification time. Units are in seconds. Defaults to 1. getenv: bool Can parameter default fields take values from the environment getshell: bool Can parameter default fields run shell commands persist_mode: ['always', 'default', 'never'] Defines the use of persisted sources: if 'always', will use a persisted version of a data source, if it exists, if 'never' will always use the original source. If 'default', persisted sources will be used if they have not expired, and re-persisted and used if they have. storage_options : dict If using a URL beginning with 'intake://' (remote Intake server), parameters to pass to requests when issuing http commands; otherwise parameters to pass to remote backend file-system. Ignored for normal local files. """ super(Catalog, self).__init__() self.name = name self.description = description self.metadata = metadata or {} self.ttl = ttl self.getenv = getenv self.getshell = getshell self.storage_options = storage_options if isinstance(user_parameters, dict) and user_parameters: from .local import UserParameter self.user_parameters = { name: (UserParameter(name=name, **up) if isinstance(up, dict) else up) for name, up in user_parameters.items() } elif isinstance(user_parameters, (list, tuple)): self.user_parameters = {up["name"]: up for up in user_parameters} else: self.user_parameters = {} if persist_mode not in ["always", "never", "default"]: # should be True, False, None ? raise ValueError("Persist mode (%s) not understood" % persist_mode) self.pmode = persist_mode if entries and isinstance(entries, str): raise ValueError( "The class intake.Catalog does not accept a string for " "`entries`\n" "Did you mean to use `intake.open_catalog`? Note that in " "versions of intake <=0.5.4 `intake.Catalog` was an " "alias for `intake.open_catalog`. It is now the intake base " "Catalog class." ) self.updated = time.time() self._entries = entries if entries is not None else self._make_entries_container() self.force_reload() @classmethod def from_dict(cls, entries, **kwargs): """ Create Catalog from the given set of entries Parameters ---------- entries : dict-like A mapping of name:entry which supports dict-like functionality, e.g., is derived from ``collections.abc.Mapping``. kwargs : passed on the constructor Things like metadata, name; see ``__init__``. Returns ------- Catalog instance """ cat = cls(**kwargs) cat._entries = entries return cat @property def kwargs(self): return dict(name=self.name, ttl=self.ttl) def _make_entries_container(self): """Subclasses may override this to return some other dict-like. See RemoteCatalog below for the motivating example for this hook. This is typically useful for large Catalogs backed by dynamic resources such as databases. The object returned by this method must implement: * ``__iter__()`` -> an iterator of entry names * ``__getitem__(key)`` -> an Entry * ``items()`` -> an iterator of ``(key, Entry)`` pairs For best performance the object should also implement: * ``__len__()`` -> int * ``__contains__(key)`` -> boolean In ``__len__`` or ``__contains__`` are not implemented, intake will fall back on iterating through the entire catalog to compute its length or check for containment, which may be expensive on large catalogs. """ return {} def _load(self): """Override this: load catalog entries""" pass def force_reload(self): """Imperative reload data now""" self.updated = time.time() self._load() def reload(self): """Reload catalog if sufficient time has passed""" if (self.ttl is not None) and (time.time() - self.updated > self.ttl): self.force_reload() @property def version(self): # default version for pre-v1 files return self.metadata.get("version", 1) @reload_on_change def search(self, text, depth=2): import copy words = text.lower().split() entries = { k: copy.copy(v) for k, v in self.walk(depth=depth).items() if any(word in str(v.describe().values()).lower() for word in words) } cat = Catalog.from_dict( entries, name=self.name + "_search", ttl=self.ttl, getenv=self.getenv, getshell=self.getshell, metadata=(self.metadata or {}).copy(), storage_options=self.storage_options, user_parameters=self.user_parameters.copy(), ) cat.metadata["search"] = {"text": text, "upstream": self.name} cat.cat = self for e in entries.values(): e._catalog = cat return cat def filter(self, func): """ Create a Catalog of a subset of entries based on a condition .. warning :: This function operates on CatalogEntry objects not DataSource objects. .. note :: Note that, whatever specific class this is performed on, the return instance is a Catalog. The entries are passed unmodified, so they will still reference the original catalog instance and include its details such as directory,. Parameters ---------- func : function This should take a CatalogEntry and return True or False. Those items returning True will be included in the new Catalog, with the same entry names Returns ------- Catalog New catalog with Entries that still refer to their parents """ return Catalog.from_dict( {key: entry for key, entry in self._entries.items() if func(entry)} ) @reload_on_change def walk(self, sofar=None, prefix=None, depth=2): """Get all entries in this catalog and sub-catalogs Parameters ---------- sofar: dict or None Within recursion, use this dict for output prefix: list of str or None Names of levels already visited depth: int Number of levels to descend; needed to truncate circular references and for cleaner output Returns ------- Dict where the keys are the entry names in dotted syntax, and the values are entry instances. """ out = sofar if sofar is not None else {} prefix = [] if prefix is None else prefix for name, item in self._entries.items(): if item._container == "catalog" and depth > 1: # recurse with default open parameters try: item().walk(out, prefix + [name], depth - 1) except Exception as e: print(e) pass # ignore inability to descend n = ".".join(prefix + [name]) out[n] = item return out def items(self): """Get an iterator over (key, source) tuples for the catalog entries.""" for name, entry in self._get_entries().items(): yield name, entry() def values(self): """Get an iterator over the sources for catalog entries.""" for entry in self._get_entries().values(): yield entry() def serialize(self): """ Produce YAML version of this catalog. Note that this is not the same as ``.yaml()``, which produces a YAML block referring to this catalog. """ import yaml output = { "metadata": self.metadata, "sources": {}, "name": self.name, "description": self.description, } for key, entry in self._entries.items(): kw = entry._captured_init_kwargs.copy() kw.pop("catalog", None) kw["parameters"] = { k.name: k.__getstate__()["kwargs"] for k in kw.get("parameters", []) } try: if issubclass(kw["driver"], DataSourceBase): kw["driver"] = ".".join([kw["driver"].__module__, kw["driver"].__name__]) except TypeError: pass # ignore exception for a string input output["sources"][key] = kw return yaml.dump(output) def save(self, url, storage_options=None): """ Output this catalog to a file as YAML Parameters ---------- url : str Location to save to, perhaps remote storage_options : dict Extra arguments for the file-system """ from fsspec import open_files with open_files([url], **(storage_options or {}), mode="wt")[0] as f: f.write(self.serialize()) @reload_on_change def _get_entry(self, name): entry = self._entries[name] entry._catalog = self entry._pmode = self.pmode up_names = set( (up["name"] if isinstance(up, dict) else up.name) for up in entry._user_parameters ) ups = [up for name, up in self.user_parameters.items() if name not in up_names] entry._user_parameters = ups + (entry._user_parameters or []) return entry() def configure_new(self, **kwargs): from .local import UserParameter ups = {} for k, v in kwargs.copy().items(): for up in self.user_parameters.values(): if isinstance(up, dict): if k == up["name"]: kw = up.copy() kw["default"] = v ups[k] = UserParameter(**kw) kwargs.pop(k) else: if k == up.name: kw = up._captured_init_kwargs.copy() kw["default"] = v kw["name"] = k ups[k] = UserParameter(**kw) kwargs.pop(k) new = super().configure_new(**kwargs) new.user_parameters.update(ups) return new __call__ = get = configure_new @reload_on_change def _get_entries(self): return self._entries def __iter__(self): """Return an iterator over catalog entry names.""" return iter(self._get_entries()) def keys(self): """Entry names in this catalog as an iterator (alias for __iter__)""" return iter(self) def __len__(self): return len(self._get_entries()) def __contains__(self, key): # Avoid iterating through all entries. return key in self._get_entries() # triggers reload_on_change def __dir__(self): # Include tab-completable entries and normal attributes. return [ entry for entry in self if re.match("[_A-Za-z][_a-zA-Z0-9]*$", entry) and not keyword.iskeyword(entry) # valid Python identifer ] + list( # not a Python keyword self.__dict__.keys() ) def _ipython_key_completions_(self): return list(self) def __repr__(self): return "" % self.name def __getattr__(self, item): # we need this special case here because the (deprecated) entry # property on the base class if item == "entry": raise NoEntry("Source was not made from a catalog entry") if not item.startswith("_"): # Fall back to __getitem__. try: return self[item] # triggers reload_on_change except KeyError as e: raise AttributeError(item) from e raise AttributeError(item) def __setitem__(self, key, entry): """Add entry to catalog This relies on the `_entries` attribute being mutable, which it normally is. Note that if a catalog automatically reloads, any entry added here may be very transient Parameters ---------- key : str Key to give the entry in the cat entry : CatalogEntry The entry to include (could be local, remote) """ self._entries[key] = entry def pop(self, key): """Remove entry from catalog and return it This relies on the `_entries` attribute being mutable, which it normally is. Note that if a catalog automatically reloads, any entry removed here may soon reappear Parameters ---------- key : str Key to give the entry in the cat """ return self._entries.pop(key) def __getitem__(self, key): """Return a catalog entry by name. Can also use attribute syntax, like ``cat.entry_name``, or item lookup cat['non-python name']. This enables walking through nested directories with cat.name1.name2, cat['name1.name2'] *or* cat['name1', 'name2'] """ if not isinstance(key, list) and key in self: # triggers reload_on_change s = self._get_entry(key) if s.container == "catalog": s.name = key s.user_parameters.update(self.user_parameters.copy()) return s return s if isinstance(key, str) and "." in key: key = key.split(".") if isinstance(key, list): parts = list(key)[:] prefix = "" while parts: bit = parts.pop(0) prefix = prefix + ("." if prefix else "") + bit if prefix in self._entries: rest = ".".join(parts) try: out = self._entries[prefix][rest] return out() except KeyError: # name conflict like "thing" and "think.oi", where it's # the latter we are after continue elif isinstance(key, tuple): out = self for part in key: out = out[part] return out() raise KeyError(key) def discover(self): return { "container": "catalog", "shape": None, "dtype": None, "metadata": self.metadata, } def _close(self): # TODO: maybe close all entries? pass @property def gui(self): if not hasattr(self, "_gui"): from ..interface import output_notebook from ..interface.gui import GUI output_notebook() self._gui = GUI({self.name: self}) else: self._gui.visible = True return self._gui ================================================ FILE: intake/catalog/default.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import json import os import subprocess import sys import platformdirs from intake.config import conf from intake.utils import make_path_posix from .local import Catalog, YAMLFilesCatalog def load_user_catalog(): """Return a catalog for the platform-specific user Intake directory""" cat_dir = user_data_dir() if not os.path.isdir(cat_dir): return Catalog() else: return YAMLFilesCatalog(cat_dir) def user_data_dir(): """Return the user Intake catalog directory""" return platformdirs.user_data_dir(appname="intake", appauthor="intake") def load_global_catalog(): """Return a catalog for the environment-specific Intake directory""" cat_dir = global_data_dir() if not os.path.isdir(cat_dir): return Catalog() else: return YAMLFilesCatalog(cat_dir) CONDA_VAR = "CONDA_PREFIX" VIRTUALENV_VAR = "VIRTUAL_ENV" def conda_prefix(): """Fallback: ask conda in PATH for its prefix""" try: out = subprocess.check_output(["conda", "info", "--json"]) return json.loads(out.decode())["default_prefix"] except (subprocess.CalledProcessError, json.JSONDecodeError, OSError): return False def which(program): """Emulate posix ``which``""" import distutils.spawn return distutils.spawn.find_executable(program) def global_data_dir(): """Return the global Intake catalog dir for the current environment""" prefix = False if VIRTUALENV_VAR in os.environ: prefix = os.environ[VIRTUALENV_VAR] elif CONDA_VAR in os.environ: prefix = sys.prefix elif which("conda"): # conda exists but is not activated prefix = conda_prefix() if prefix: # conda and virtualenv use Linux-style directory pattern return make_path_posix(os.path.join(prefix, "share", "intake")) else: return platformdirs.site_data_dir(appname="intake", appauthor="intake") def load_combo_catalog(): """Load a union of the user and global catalogs for convenience""" user_dir = user_data_dir() global_dir = global_data_dir() desc = "Generated from data packages found on your intake search path" cat_dirs = [] if os.path.isdir(user_dir): cat_dirs.append(user_dir + "/*.yaml") cat_dirs.append(user_dir + "/*.yml") if os.path.isdir(global_dir): cat_dirs.append(global_dir + "/*.yaml") cat_dirs.append(global_dir + "/*.yml") for path_dir in conf.get("catalog_path", []): if path_dir != "": if not path_dir.endswith(("yaml", "yml")): cat_dirs.append(path_dir + "/*.yaml") cat_dirs.append(path_dir + "/*.yml") else: cat_dirs.append(path_dir) return YAMLFilesCatalog(cat_dirs, name="builtin", description=desc) ================================================ FILE: intake/catalog/entry.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from ..utils import DictSerialiseMixin, pretty_describe class CatalogEntry(DictSerialiseMixin): """A single item appearing in a catalog This is the base class, used by local entries (i.e., read from a YAML file) and by remote entries (read from a server). """ def __init__(self, getenv=True, getshell=False): self._default_source = None self.getenv = getenv self.getshell = getshell self._pmode = "default" def describe(self): """Get a dictionary of attributes of this entry. Returns: dict with keys name: str The name of the catalog entry. container : str kind of container used by this data source description : str Markdown-friendly description of data source direct_access : str Mode of remote access: forbid, allow, force user_parameters : list[dict] List of user parameters defined by this entry """ raise NotImplementedError def get(self, **user_parameters): """Open the data source. Equivalent to calling the catalog entry like a function. Parameters ---------- user_parameters : dict Values for user-configurable parameters for this data source Returns ------- DataSource """ raise NotImplementedError def __call__(self, persist=None, **kwargs): """Instantiate DataSource with given user arguments""" s = self.get(**kwargs) s._entry = self s._passed_kwargs = list(kwargs) return s @property def container(self): return getattr(self, "_container", None) @container.setter def container(self, cont): # so that .container (which sources always have) always reflects ._container, # which is the variable name for entries. self._container = cont @property def plots(self): """List custom associated quick-plots""" return list(self._metadata.get("plots", {})) def _ipython_display_(self): """Display the entry as a rich object in an IPython session.""" import json from IPython.display import display contents = self.describe() display( { "application/json": json.dumps(contents), "text/plain": pretty_describe(contents), }, metadata={"application/json": {"root": contents["name"]}}, raw=True, ) def _yaml(self): return {"sources": {self.name: self.describe()}} def __iter__(self): # If the entry is a catalog, this allows list(cat.entry) if self._container == "catalog": return iter(self()) else: raise ValueError("Cannot iterate a catalog entry") def __getitem__(self, item): """Pass getitem to data source, assuming default parameters Also supports multiple items ([.., ..]), in which case the first component only will be used to instantiate, and the rest passed on. """ if isinstance(item, tuple): if len(item) > 1: return self()[item[0]].__getitem__(item[1:]) else: item = item[0] return self()[item] def __repr__(self): return pretty_describe(self.describe()) @property def gui(self): if not hasattr(self, "_gui"): from .gui import EntryGUI self._gui = EntryGUI(source=self, visible=True) else: self._gui.visible = True return self._gui ================================================ FILE: intake/catalog/exceptions.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- class CatalogException(Exception): """Basic exception for errors raised by catalog""" class PermissionDenied(CatalogException): """Raised when user requests functionality that they do not have permission to access. """ class ShellPermissionDenied(PermissionDenied): """The user does not have permission to execute shell commands.""" def __init__(self, msg=None): if msg is None: msg = "Additional permissions needed to execute shell commands." super(ShellPermissionDenied, self).__init__(msg) class EnvironmentPermissionDenied(PermissionDenied): """The user does not have permission to read environment variables.""" def __init__(self, msg=None): if msg is None: msg = "Additional permissions needed to read environment variables." super(EnvironmentPermissionDenied, self).__init__(msg) class ValidationError(CatalogException): """Something's wrong with the catalog spec""" def __init__(self, message, errors): super(ValidationError, self).__init__(message) self.errors = errors class DuplicateKeyError(ValidationError): """Catalog contains key duplications""" def __init__(self, context, context_mark, problem, problem_mark): line = problem_mark.line column = problem_mark.column msg = "duplicate key found on line {}, column {}".format(line + 1, column + 1) super(DuplicateKeyError, self).__init__(msg, []) class ObsoleteError(ValidationError): pass class ObsoleteParameterError(ObsoleteError): def __init__(self): msg = """Detected old syntax. See details for upgrade instructions to new syntax: [old syntax] parameters: - name: abc type: str [new syntax] parameters: abc: type: str """ super(ObsoleteParameterError, self).__init__(msg, []) class ObsoleteDataSourceError(ObsoleteError): def __init__(self): msg = """Detected old syntax. See details for upgrade instructions to new syntax: [old syntax] sources: - name: abc driver: csv [new syntax] sources: abc: driver: csv """ super(ObsoleteDataSourceError, self).__init__(msg, []) ================================================ FILE: intake/catalog/gui.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2019, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- try: from ..interface.gui import GUI except ImportError: class GUI(object): def __init__(self, *args, **kwargs): pass def __repr__(self): raise RuntimeError( "Please install panel to use the GUI (`conda " "install -c conda-forge panel>0.8.0`)" ) except Exception: class GUI(object): def __init__(self, *args, **kwargs): pass def __repr__(self): raise RuntimeError( "Initialization of GUI failed, even though " "panel is installed. Please update it " "to a more recent version (`conda install -c " "conda-forge panel==0.5.1`)." ) ================================================ FILE: intake/catalog/local.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import collections from importlib.metadata import entry_points import inspect import logging import os import warnings from fsspec import get_filesystem_class, open_files from fsspec.core import split_protocol from .. import __version__ from ..source import get_plugin_class, register_driver from ..utils import DictSerialiseMixin, classname, make_path_posix, yaml_load from . import exceptions from .base import Catalog, DataSource, VersionError from .entry import CatalogEntry from .utils import COERCION_RULES, _has_catalog_dir, coerce, expand_defaults, merge_pars logger = logging.getLogger("intake") class UserParameter(DictSerialiseMixin): """ A user-settable item that is passed to a DataSource upon instantiation. For string parameters, default may include special functions ``func(args)``, which *may* be expanded from environment variables or by executing a shell command. Parameters ---------- name: str the key that appears in the DataSource argument strings description: str narrative text type: str one of list ``(COERSION_RULES)`` default: type value same type as ``type``. It a str, may include special functions env, shell, client_env, client_shell. min, max: type value for validation of user input allowed: list of type for validation of user input """ def __init__( self, name, description=None, type=None, default=None, min=None, max=None, allowed=None, ): self.name = name self.description = description self.type = type or __builtins__["type"](default).__name__ self.min = min self.max = max self.allowed = allowed self._default = default try: self.default = coerce(self.type, default) except (ValueError, TypeError): self.default = None self.expanded_default = self.default if self.min: self.min = coerce(self.type, self.min) if self.max: self.max = coerce(self.type, self.max) if self.allowed and type != "mlist": self.allowed = [coerce(self.type, item) for item in self.allowed] def __repr__(self): return f"<{self.__class__.__name__} {self.name!r}>" __str__ = __repr__ def describe(self): """Information about this parameter""" desc = { "name": self.name, "description": self.description, # the Parameter might not have a type at all "type": self.type or "unknown", } for attr in ["min", "max", "allowed", "default"]: v = getattr(self, attr) if v is not None: desc[attr] = v return desc def expand_defaults(self, client=False, getenv=True, getshell=False): """Compile env, client_env, shell and client_shell commands""" if not isinstance(self._default, str): self.expanded_default = self._default else: self.expanded_default = coerce( self.type, expand_defaults(self._default, client, getenv, getshell) ) def validate(self, value): """Does value meet parameter requirements?""" if self.type is not None: value = coerce(self.type, value) if self.type == "mlist": for v in value: if v not in self.allowed: raise ValueError("Item %s not in allowed list", v) return value if self.min is not None and value < self.min: raise ValueError("%s=%s is less than %s" % (self.name, value, self.min)) if self.max is not None and value > self.max: raise ValueError("%s=%s is greater than %s" % (self.name, value, self.max)) if self.allowed is not None and value not in self.allowed: raise ValueError( "%s=%s is not one of the allowed values: %s" % (self.name, value, ",".join(map(str, self.allowed))) ) return value class LocalCatalogEntry(CatalogEntry): """A catalog entry on the local system""" def __init__( self, name, description, driver, direct_access=True, args={}, cache=[], parameters=[], metadata={}, catalog_dir="", getenv=True, getshell=False, catalog=None, ): """ Parameters ---------- name: str How this entry is known, normally from its key in a YAML file, or if that is not provided then from name of file, or name of dir if file name is 'catalog.yaml' or 'catalog.yml'. description: str Brief text about the target source driver: str, list, dict or DataSource subclass The plugin(s) that can load this. Can be a simple name like "csv", which will be looked up in the registry, a fully-qualified class name ("package.mod.Class"), a list of these which would all work, a dictionary of the same with reasonable names, or an explicit class derived from DataSource. direct_access: bool Is the client allowed to attempt to reach this data args: dict Passed when instantiating the plugin DataSource parameters: list UserParameters that can be set metadata: dict Additional information about this data catalog_dir: str Location of the catalog, if known getenv: bool Can parameter default fields take values from the environment getshell: bool Can parameter default fields run shell commands catalog: bool Catalog object in which this entry belongs """ self._name = name self._default_source = None self._description = description self._driver = driver self._direct_access = direct_access self._open_args = args self._cache = cache self._user_parameters = parameters self._metadata = metadata or {} self._catalog_dir = catalog_dir self._filesystem = None self._catalog = catalog if isinstance(driver, str): dr = get_plugin_class(driver) self._plugin = [dr] if dr is not None else [] containers = set(p.container for p in self._plugin) elif isinstance(driver, list): self._plugin = [get_plugin_class(d) for d in driver] self._plugin = [p for p in self._plugin if p is not None] containers = set(p.container for p in self._plugin) elif isinstance(driver, dict): self._plugin = {d: get_plugin_class(driver[d]["class"]) for d in driver} self._plugin = {k: v for k, v in self._plugin.items() if v is not None} containers = set(p.container for p in self._plugin.values()) elif inspect.isclass(driver) and issubclass(driver, DataSource): self._plugin = [driver] containers = {driver.container} else: raise TypeError("Driver was not a string, list, dict or DataSource:" " %s" % driver) if len(containers) > 1: # this is an error, because cat is poorly specified, even if other # plugins are OK raise ValueError("Plugins for a data source must have only one " "container.") if len(containers) == 0: # this is only debug, this single entry won't work, but cat is OK. # you get an error if you try to actually use this entry logger.debug("No plugins for entry: %s" % self.name) containers = [None] self._container = list(containers)[0] super(LocalCatalogEntry, self).__init__(getenv=getenv, getshell=getshell) @property def name(self): return self._name def describe(self): """Basic information about this entry""" if isinstance(self._plugin, list): pl = [p.name for p in self._plugin] elif isinstance(self._plugin, dict): pl = {k: classname(v) for k, v in self._plugin.items()} else: pl = self._plugin if isinstance(self._plugin, str) else self._plugin.name return { "name": self._name, "container": self._container, "plugin": pl, # deprecated "driver": pl, "description": self._description, "direct_access": self._direct_access, "user_parameters": [u.describe() for u in self._user_parameters], "metadata": self._metadata, "args": self._open_args, } def _create_open_args(self, user_parameters): plugin = user_parameters.pop("plugin", None) md = self._metadata.copy() if self._metadata is not None else {} md["catalog_dir"] = self._catalog_dir if user_parameters.pop("cache", None) or self._cache: md["cache"] = user_parameters.pop("cache", None) or self._cache params = { "metadata": md, "CATALOG_DIR": self._catalog_dir, } params.update(self._open_args) if ( "storage_options" not in params and self._filesystem is not None and self._filesystem.storage_options and _has_catalog_dir(params) ): params["storage_options"] = self._filesystem.storage_options open_args = merge_pars( params, user_parameters, self._user_parameters, getshell=self.getshell, getenv=self.getenv, client=False, ) if len(self._plugin) == 0: raise ValueError( "No plugins loaded for this entry: %s\n" "A listing of installable plugins can be found " "at https://intake.readthedocs.io/en/latest/plugin" "-directory.html ." % self._driver ) elif isinstance(self._plugin, list): plugin = self._plugin[0] else: # dict if plugin is None: # default selection for dict plugin = list(self._plugin)[0] spec = self._driver[plugin] open_args.update(spec.get("args", {})) try: plugin = self._plugin[plugin] except KeyError: raise ValueError( "Attempt to select unavailable plugin %s, " "perhaps import of plugin failed" % plugin ) return plugin, open_args def get(self, **user_parameters): """Instantiate the DataSource for the given parameters""" if not user_parameters and self._default_source is not None: return self._default_source plugin, open_args = self._create_open_args(user_parameters) data_source = plugin(**open_args) data_source.catalog_object = self._catalog data_source.name = self.name data_source.description = self._description data_source.cat = self._catalog # Cache the default source if there are no user parameters. if not user_parameters: self._default_source = data_source return data_source def clear_cached_default_source(self): """ Clear a cached default source so it can be created anew (if, for instance, it depends on changing environment variables or execution context) """ self._default_source = None class CatalogParser(object): """Loads entries from a YAML spec""" def __init__(self, data, getenv=True, getshell=False, context=None): self._context = context if context else {} self._errors = [] self._warnings = [] self.getenv = getenv self.getshell = getshell self._data = self._parse(data) @property def ok(self): return len(self._errors) == 0 @property def data(self): return self._data @property def errors(self): return self._errors @property def warnings(self): return self._warnings def error(self, msg, obj, key=None): if key is not None: self._errors.append(str((msg, obj, key))) else: self._errors.append(str((msg, obj))) def warning(self, msg, obj, key=None): if key is None: self._warnings.append(str((msg, obj))) else: self._warnings.append(str((msg, obj, key))) def _parse_plugins(self, data): if "plugins" not in data: return if not isinstance(data["plugins"], dict): self.error("value of key 'plugins' must be a dictionary", data, "plugins") return if "source" not in data["plugins"]: self.error("missing key 'source'", data["plugins"]) return if not isinstance(data["plugins"]["source"], list): self.error("value of key 'source' must be a list", data["plugins"], "source") return for plugin_source in data["plugins"]["source"]: if not isinstance(plugin_source, dict): self.error( "value in list of plugins sources must be a " "dictionary", data["plugins"], "source", ) continue elif "module" in plugin_source: import intake intake.import_name(plugin_source["module"]) elif "dir" in plugin_source: self.error( "The key 'dir', and in general the feature of registering " "plugins from a directory of Python scripts outside of " "sys.path, is no longer supported. Use 'module'.", plugin_source, ) else: self.error("missing 'module'", plugin_source) def _getitem(self, obj, key, dtype, required=True, default=None, choices=None): if key in obj: if isinstance(obj[key], dtype): if choices and obj[key] not in choices: self.error( "value '{}' is invalid (choose from {})".format(obj[key], choices), obj, key, ) else: return obj[key] else: self.error( "value '{}' is not expected type '{}'".format(obj[key], dtype.__name__), obj, key, ) return None elif required: self.error("missing required key '{}'".format(key), obj) return None elif default: return default return None if dtype is object else dtype() def _parse_user_parameter(self, name, data): valid_types = list(COERCION_RULES) params = { "name": name, "description": self._getitem(data, "description", str), "type": self._getitem(data, "type", str, choices=valid_types), "default": self._getitem(data, "default", object, required=False), "min": self._getitem(data, "min", object, required=False), "max": self._getitem(data, "max", object, required=False), "allowed": self._getitem(data, "allowed", object, required=False), } if params["description"] is None or params["type"] is None: return None return UserParameter(**params) def _parse_data_source(self, name, data): if data.pop("remote", False): return elif "cls" in data: from intake.utils import remake_instance return remake_instance(data) else: return self._parse_data_source_local(name, data) def _parse_data_source_local(self, name, data): ds = { "name": name, "description": self._getitem(data, "description", str, required=False), "driver": self._getitem(data, "driver", object), "direct_access": self._getitem( data, "direct_access", str, required=False, default="forbid", choices=["forbid", "allow", "force"], ), "args": self._getitem(data, "args", dict, required=False), "cache": self._getitem(data, "cache", list, required=False), "metadata": self._getitem(data, "metadata", dict, required=False), } if ds["driver"] is None: return None ds["parameters"] = [] if "parameters" in data: if isinstance(data["parameters"], list): raise exceptions.ObsoleteParameterError if not isinstance(data["parameters"], dict): self.error("value of key 'parameters' must be a dictionary", data, "parameters") return None for name, parameter in data["parameters"].items(): if not isinstance(name, str): self.error( "key '{}' must be a string".format(name), data["parameters"], name, ) continue if not isinstance(parameter, dict): self.error( "value of key '{}' must be a dictionary" "".format(name), data["parameters"], name, ) continue obj = self._parse_user_parameter(name, parameter) if obj: ds["parameters"].append(obj) return LocalCatalogEntry( catalog_dir=self._context["root"], getenv=self.getenv, getshell=self.getshell, **ds, ) def _parse_data_sources(self, data): sources = [] if "sources" not in data: self.error("missing key 'sources'", data) return sources if isinstance(data["sources"], list): raise exceptions.ObsoleteDataSourceError if not isinstance(data["sources"], dict): self.error("value of key 'sources' must be a dictionary", data, "sources") return sources for name, source in data["sources"].items(): if not isinstance(name, str): self.error("key '{}' must be a string".format(name), data["sources"], name) continue if not isinstance(source, dict): self.error( "value of key '{}' must be a dictionary" "".format(name), data["sources"], name, ) continue obj = self._parse_data_source(name, source) if obj: sources.append(obj) return sources def _parse(self, data): if not isinstance(data, dict): self.error("catalog must be a dictionary", data) return if (data.get("version", None) or data.get("metadata", {}).get("version", None) or 1) > 1: raise VersionError("Not a V1 Catalog; perhaps use intake.open_catalog") return dict( plugin_sources=self._parse_plugins(data), data_sources=self._parse_data_sources(data), metadata=data.get("metadata", {}), name=data.get("name"), description=data.get("description"), ) def get_dir(path): if "://" in path: protocol, _ = split_protocol(path) out = get_filesystem_class(protocol)._parent(path) if "://" not in out: # some FSs strip this, some do not out = protocol + "://" + out return out path = make_path_posix(os.path.join(os.getcwd(), os.path.dirname(path))) if path[-1] != "/": path += "/" return path class YAMLFileCatalog(Catalog): """Catalog as described by a single YAML file""" version = __version__ container = "catalog" partition_access = None name = "yaml_file_cat" def __init__(self, path=None, text=None, autoreload=True, **kwargs): """ Parameters ---------- path: str Location of the file to parse (can be remote) text: str (DEPRECATED) YAML contents of catalog, takes precedence over path autoreload : bool Whether to watch the source file for changes; make False if you want an editable Catalog """ self.path = path if text is not None: logger.warning("YAMLFileCatalog `text` argument is deprecated") warnings.warn("`text` argument is deprecated", DeprecationWarning) self.parse(text) self.autoreload = False else: self.autoreload = autoreload # set this to False if don't want reloads self.filesystem = kwargs.pop("fs", None) self.access = "name" not in kwargs super(YAMLFileCatalog, self).__init__(**kwargs) def _load(self, reload=False): """Load text of catalog file and pass to parse Will do nothing if auto-reload is off and reload is not explicitly requested """ if self.access is False: # skip first load, if cat has given name (i.e., is subcat) self.updated = 0 self.access = True return if self.autoreload or reload: # First, we load from YAML, failing if syntax errors are found options = self.storage_options or {} if hasattr(self.path, "path") or hasattr(self.path, "read"): file_open = self.path self.path = make_path_posix( getattr(self.path, "path", getattr(self.path, "name", "file")) ) elif self.filesystem is None: file_open = open_files(self.path, mode="rb", **options) assert len(file_open) == 1 file_open = file_open[0] self.filesystem = file_open.fs else: file_open = self.filesystem.open(self.path, mode="rb") self._dir = get_dir(self.path) with file_open as f: text = f.read().decode() if "!template " in text: logger.warning("Use of '!template' deprecated - fixing") text = text.replace("!template ", "") self.parse(text) def add(self, source, name=None, path=None, storage_options=None): """Add sources to the catalog and save into the original file This adds the source into the catalog dictionary, and saves the resulting catalog as YAML. Typically, this would be used to update a catalog file in-place. Optionally, the new catalog can be saved to a new location, in which case the new catalog is returned. Note that if a source of the given name exists, it will be clobbered. Parameters ---------- source : DataSource instance The source whose spec we want to save name : str or None The name the source is to have in the catalog; use the source's name attribute, if not given. path : str or None Location to save the new catalog; if None, the original location from which it was loaded storage_options : dict or None If saving to a new location, use these arguments for the filesystem backend Returns ------- YAMLFileCatalog instance, containing the new entry """ import yaml entries = self._entries.copy() name = name or source.name or "source" entries[name] = source if path is None: options = self.storage_options or {} file_open = open_files([self.path], mode="wt", **options) else: options = storage_options or {} file_open = open_files([path], mode="wt", **options) assert len(file_open) == 1 file_open = file_open[0] data = {"metadata": self.metadata, "sources": {}} for e in entries: data["sources"][e] = list(entries[e]._yaml()["sources"].values())[0] with file_open as f: yaml.dump(data, f, default_flow_style=False) if path: return self else: return YAMLFileCatalog( self.path, storage_options=storage_options, autoreload=self.autoreload ) def parse(self, text): """Create entries from catalog text Normally the text comes from the file at self.path via the ``_load()`` method, but could be explicitly set instead. A copy of the text is kept in attribute ``.text`` . Parameters ---------- text : str YAML formatted catalog spec """ self.text = text data = yaml_load(self.text) if data is None: raise exceptions.CatalogException("No YAML data in file") # Second, we validate the schema and semantics context = dict(root=self._dir) result = CatalogParser(data, context=context, getenv=self.getenv, getshell=self.getshell) if result.errors: raise exceptions.ValidationError( "Catalog '{}' has validation errors:\n\n{}" "".format(self.path, "\n".join(result.errors)), result.errors, ) cfg = result.data self._entries = {} shared_parameters = data.get("metadata", {}).get("parameters", {}) self.user_parameters.update( {name: UserParameter(name, **attrs) for name, attrs in shared_parameters.items()} ) for entry in cfg["data_sources"]: entry._catalog = self self._entries[entry.name] = entry entry._filesystem = self.filesystem meta = self.metadata.copy() meta.update(cfg.get("metadata", {})) self.metadata = meta self.name = self.name or cfg.get("name") or self.name_from_path self.description = self.description or cfg.get("description") @property def name_from_path(self): """If catalog is named 'catalog' take name from parent directory""" name = os.path.splitext(os.path.basename(self.path))[0] if name == "catalog": name = os.path.basename(os.path.dirname(self.path)) return name.replace(".", "_") class YAMLFilesCatalog(Catalog): """Catalog as described by a multiple YAML files""" version = (__version__,) container = "catalog" partition_access = None name = "yaml_files_cat" def __init__(self, path, flatten=True, **kwargs): """ Parameters ---------- path: str Location of the files to parse (can be remote), including possible glob (*) character(s). Can also be list of paths, without glob characters. flatten: bool (True) Whether to list all entries in the cats at the top level (True) or create sub-cats from each file (False). """ self.path = path self._flatten = flatten self._kwargs = kwargs.copy() self._cat_files = [] self._cats = {} self.access = "name" not in kwargs super(YAMLFilesCatalog, self).__init__(**kwargs) def _load(self): # initial: find cat files # if flattening, need to get all entries from each. if self.access is False: # skip first load, if cat has given name (i.e., is subcat) self.updated = 0 self.access = True return self._entries.clear() options = self.storage_options or {} if isinstance(self.path, (list, tuple)): files = sum([open_files(p, mode="rb", **options) for p in self.path], []) self.name = self.name or "%i files" % len(files) self.description = self.description or f"Catalog generated from {len(files)} files" self.path = [make_path_posix(p) for p in self.path] else: if isinstance(self.path, str) and "*" not in self.path: self.path = self.path + "/*" files = open_files(self.path, mode="rb", **options) self.path = make_path_posix(self.path) self.name = self.name or self.path self.description = ( self.description or f"Catalog generated from all files found in {self.path}" ) if not set(f.path for f in files) == set(f.path for f in self._cat_files): # glob changed, reload all self._cat_files = files self._cats.clear() for f in files: name = os.path.split(f.path)[-1].replace(".yaml", "").replace(".yml", "") kwargs = self.kwargs.copy() kwargs["path"] = f.path d = make_path_posix(os.path.dirname(f.path)) if f.path not in self._cats: entry = LocalCatalogEntry( name, "YAML file: %s" % name, "yaml_file_cat", True, kwargs, [], [], self.metadata, d, ) if self._flatten: # store a concrete Catalog try: cat = entry() cat.reload() self.user_parameters.update(cat.user_parameters) self._cats[f.path] = cat except IOError as e: logger.info('Loading "%s" as a catalog failed: %s' "" % (entry, e)) else: # store a catalog entry self._cats[f.path] = entry entries = {} for name, entry in list(self._cats.items()): if self._flatten: entry.reload() inter = set(entry._entries).intersection(entries) if inter: raise ValueError( "Conflicting names when flattening multiple" " catalogs. Sources %s exist in more than" " one" % inter ) entries.update(entry._entries) else: entries[entry._name] = entry self._entries.update(entries) class MergedCatalog(Catalog): """ A Catalog that merges the entries of a list of catalogs. """ def __init__(self, catalogs, *args, **kwargs): self._catalogs = catalogs super().__init__(*args, **kwargs) def _load(self): for catalog in self._catalogs: catalog._load() self._entries = collections.ChainMap(*(catalog._entries for catalog in self._catalogs)) class EntrypointEntry(CatalogEntry): """ A catalog entry for an entrypoint. """ def __init__(self, entrypoint): self._entrypoint = entrypoint self._container = None self._user_parameters = [] super().__init__() def __repr__(self): return f"" @property def name(self): return self._entrypoint.name def describe(self): """Basic information about this entry""" if self._container is None: self._container = self().container return { "name": self.name, "module_name": self._entrypoint.module_name, "object_name": self._entrypoint.object_name, "distro": self._entrypoint.distro, "extras": self._entrypoint.extras, "container": self._container, } def __call__(self, **kwargs): """Instantiate the DataSource for the given parameters""" source = self._entrypoint.load() if kwargs: source = source.configure_new(**kwargs) return source get = __call__ class EntrypointsCatalog(Catalog): """ A catalog of discovered entrypoint catalogs. """ def __init__(self, *args, entrypoints_group="intake.catalogs", paths=None, **kwargs): self._entrypoints_group = entrypoints_group self._paths = paths super().__init__(*args, **kwargs) def _load(self): eps = entry_points() if hasattr(eps, "select"): # Python 3.10+ / importlib_metadata >= 3.9.0 catalogs = eps.select(group=self._entrypoints_group) catalogs = {ep.name: ep for ep in catalogs} else: catalogs = eps.get("intake.drivers", []) self.name = self.name or "EntrypointsCatalog" self.description = self.description or f"EntrypointsCatalog of {len(catalogs)} catalogs." for name, entrypoint in catalogs.items(): try: self._entries[name] = EntrypointEntry(entrypoint) except Exception as e: warnings.warn(f"Failed to load {name}, {entrypoint}, {e!r}.") # Register these early in the import process to support the default catalog # which is built at import time. (Without this, 'yaml_file_cat' is looked for # in intake.registry before the registry has been populated.) register_driver("yaml_file_cat", YAMLFileCatalog, clobber=True) register_driver("yaml_files_cat", YAMLFilesCatalog, clobber=True) ================================================ FILE: intake/catalog/tests/__init__.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- ================================================ FILE: intake/catalog/tests/cache_data/states.csv ================================================ "state","slug","code","nickname","website","admission_date","admission_number","capital_city","capital_url","population","population_rank","constitution_url","state_flag_url","state_seal_url","map_image_url","landscape_background_url","skyline_background_url","twitter_url","facebook_url" "Alabama","alabama","AL","Yellowhammer State","http://www.alabama.gov","1819-12-14",22,"Montgomery","http://www.montgomeryal.gov",4833722,23,"http://alisondb.legislature.state.al.us/alison/default.aspx","https://cdn.civil.services/us-states/flags/alabama-large.png","https://cdn.civil.services/us-states/seals/alabama-large.png","https://cdn.civil.services/us-states/maps/alabama-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/alabama.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/alabama.jpg","https://twitter.com/alabamagov","https://www.facebook.com/alabamagov" "Alaska","alaska","AK","The Last Frontier","http://alaska.gov","1959-01-03",49,"Juneau","http://www.juneau.org",735132,47,"http://www.legis.state.ak.us/basis/folioproxy.asp?url=http://wwwjnu01.legis.state.ak.us/cgi-bin/folioisa.dll/acontxt/query=*/doc/{t1}?","https://cdn.civil.services/us-states/flags/alaska-large.png","https://cdn.civil.services/us-states/seals/alaska-large.png","https://cdn.civil.services/us-states/maps/alaska-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/alaska.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/alaska.jpg","https://twitter.com/alaska","https://www.facebook.com/AlaskaLocalGovernments" "Arizona","arizona","AZ","The Grand Canyon State","https://az.gov","1912-02-14",48,"Phoenix","https://www.phoenix.gov",6626624,15,"http://www.azleg.gov/Constitution.asp","https://cdn.civil.services/us-states/flags/arizona-large.png","https://cdn.civil.services/us-states/seals/arizona-large.png","https://cdn.civil.services/us-states/maps/arizona-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/arizona.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/arizona.jpg",, "Arkansas","arkansas","AR","The Natural State","http://arkansas.gov","1836-06-15",25,"Little Rock","http://www.littlerock.org",2959373,32,"http://www.arkleg.state.ar.us/assembly/Summary/ArkansasConstitution1874.pdf","https://cdn.civil.services/us-states/flags/arkansas-large.png","https://cdn.civil.services/us-states/seals/arkansas-large.png","https://cdn.civil.services/us-states/maps/arkansas-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/arkansas.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/arkansas.jpg","https://twitter.com/arkansasgov","https://www.facebook.com/Arkansas.gov" "California","california","CA","Golden State","http://www.ca.gov","1850-09-09",31,"Sacramento","http://www.cityofsacramento.org",38332521,1,"http://www.leginfo.ca.gov/const-toc.html","https://cdn.civil.services/us-states/flags/california-large.png","https://cdn.civil.services/us-states/seals/california-large.png","https://cdn.civil.services/us-states/maps/california-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/california.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/california.jpg","https://twitter.com/cagovernment", "Colorado","colorado","CO","The Centennial State","https://www.colorado.gov","1876-08-01",38,"Denver","http://www.denvergov.org",5268367,22,"https://www.colorado.gov/pacific/archives/government","https://cdn.civil.services/us-states/flags/colorado-large.png","https://cdn.civil.services/us-states/seals/colorado-large.png","https://cdn.civil.services/us-states/maps/colorado-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/colorado.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/colorado.jpg","https://twitter.com/coloradogov","https://www.facebook.com/Colorado.gov" "Connecticut","connecticut","CT","Constitution State","http://www.ct.gov","1788-01-09",5,"Hartford","http://www.hartford.gov",3596080,29,"http://www.ct.gov/sots/cwp/view.asp?a=3188&q=392288","https://cdn.civil.services/us-states/flags/connecticut-large.png","https://cdn.civil.services/us-states/seals/connecticut-large.png","https://cdn.civil.services/us-states/maps/connecticut-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/connecticut.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/connecticut.jpg",, "Delaware","delaware","DE","The First State / The Diamond State","http://delaware.gov","1787-12-07",1,"Dover","http://www.cityofdover.com",925749,45,"http://www.state.de.us/facts/constit/welcome.htm","https://cdn.civil.services/us-states/flags/delaware-large.png","https://cdn.civil.services/us-states/seals/delaware-large.png","https://cdn.civil.services/us-states/maps/delaware-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/delaware.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/delaware.jpg","https://twitter.com/delaware_gov","https://www.facebook.com/delaware.gov" "Florida","florida","FL","Sunshine State","http://www.myflorida.com","1845-03-03",27,"Tallahassee","https://www.talgov.com/Main/Home.aspx",19552860,4,"http://www.leg.state.fl.us/Statutes/index.cfm","https://cdn.civil.services/us-states/flags/florida-large.png","https://cdn.civil.services/us-states/seals/florida-large.png","https://cdn.civil.services/us-states/maps/florida-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/florida.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/florida.jpg",, "Georgia","georgia","GA","Peach State","http://georgia.gov","1788-01-02",4,"Atlanta","http://www.atlantaga.gov",9992167,8,"http://sos.ga.gov/admin/files/Constitution_2013_Final_Printed.pdf","https://cdn.civil.services/us-states/flags/georgia-large.png","https://cdn.civil.services/us-states/seals/georgia-large.png","https://cdn.civil.services/us-states/maps/georgia-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/georgia.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/georgia.jpg","http://twitter.com/georgiagov","http://www.facebook.com/pages/georgiagov/29760668054" "Hawaii","hawaii","HI","Aloha State","https://www.ehawaii.gov","1959-08-21",50,"Honolulu","http://www.co.honolulu.hi.us",1404054,40,"http://lrbhawaii.org/con","https://cdn.civil.services/us-states/flags/hawaii-large.png","https://cdn.civil.services/us-states/seals/hawaii-large.png","https://cdn.civil.services/us-states/maps/hawaii-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/hawaii.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/hawaii.jpg","https://twitter.com/ehawaiigov","https://www.facebook.com/ehawaii.gov" "Idaho","idaho","ID","Gem State","https://www.idaho.gov","1890-07-03",43,"Boise","http://www.cityofboise.org",1612136,39,"http://www.legislature.idaho.gov/idstat/IC/Title003.htm","https://cdn.civil.services/us-states/flags/idaho-large.png","https://cdn.civil.services/us-states/seals/idaho-large.png","https://cdn.civil.services/us-states/maps/idaho-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/idaho.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/idaho.jpg","https://twitter.com/IDAHOgov", "Illinois","illinois","IL","Prairie State","https://www.illinois.gov","1818-12-03",21,"Springfield","http://www.springfield.il.us",12882135,5,"http://www.ilga.gov/commission/lrb/conmain.htm","https://cdn.civil.services/us-states/flags/illinois-large.png","https://cdn.civil.services/us-states/seals/illinois-large.png","https://cdn.civil.services/us-states/maps/illinois-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/illinois.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/illinois.jpg",, "Indiana","indiana","IN","Hoosier State","http://www.in.gov","1816-12-11",19,"Indianapolis","http://www.indy.gov/Pages/Home.aspx",6570902,16,"http://www.law.indiana.edu/uslawdocs/inconst.html","https://cdn.civil.services/us-states/flags/indiana-large.png","https://cdn.civil.services/us-states/seals/indiana-large.png","https://cdn.civil.services/us-states/maps/indiana-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/indiana.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/indiana.jpg","https://twitter.com/in_gov","https://www.facebook.com/IndianaGovernment" "Iowa","iowa","IA","Hawkeye State","https://www.iowa.gov","1846-12-28",29,"Des Moines","http://www.ci.des-moines.ia.us",3090416,30,"http://publications.iowa.gov/135/1/history/7-7.html","https://cdn.civil.services/us-states/flags/iowa-large.png","https://cdn.civil.services/us-states/seals/iowa-large.png","https://cdn.civil.services/us-states/maps/iowa-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/iowa.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/iowa.jpg","https://twitter.com/IAGOVTWEETS", "Kansas","kansas","KS","Sunflower State","https://www.kansas.gov","1861-01-29",34,"Topeka","http://www.topeka.org",2893957,34,"https://kslib.info/405/Kansas-Constitution","https://cdn.civil.services/us-states/flags/kansas-large.png","https://cdn.civil.services/us-states/seals/kansas-large.png","https://cdn.civil.services/us-states/maps/kansas-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/kansas.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/kansas.jpg","http://www.twitter.com/ksgovernment","http://www.facebook.com/pages/Topeka-KS/Kansasgov-Kansas-Government-Online/52068474220" "Kentucky","kentucky","KY","Bluegrass State","http://kentucky.gov","1792-06-01",15,"Frankfort","http://frankfort.ky.gov",4395295,26,"http://www.lrc.state.ky.us/Legresou/Constitu/intro.htm","https://cdn.civil.services/us-states/flags/kentucky-large.png","https://cdn.civil.services/us-states/seals/kentucky-large.png","https://cdn.civil.services/us-states/maps/kentucky-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/kentucky.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/kentucky.jpg","https://twitter.com/kygov","https://www.facebook.com/kygov" "Louisiana","louisiana","LA","Pelican State","http://louisiana.gov","1812-04-30",18,"Baton Rouge","http://brgov.com",4625470,25,"http://senate.legis.state.la.us/Documents/Constitution","https://cdn.civil.services/us-states/flags/louisiana-large.png","https://cdn.civil.services/us-states/seals/louisiana-large.png","https://cdn.civil.services/us-states/maps/louisiana-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/louisiana.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/louisiana.jpg",, "Maine","maine","ME","Pine Tree State","http://www.maine.gov","1820-03-15",23,"Augusta","http://www.augustamaine.gov",1328302,41,"http://www.maine.gov/legis/const","https://cdn.civil.services/us-states/flags/maine-large.png","https://cdn.civil.services/us-states/seals/maine-large.png","https://cdn.civil.services/us-states/maps/maine-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/maine.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/maine.jpg","https://twitter.com/mainegov_news","http://www.facebook.com/pages/Augusta-ME/Mainegov/98519328240" "Maryland","maryland","MD","Old Line State","http://www.maryland.gov","1788-04-28",7,"Annapolis","http://www.annapolis.gov",5928814,19,"http://msa.maryland.gov/msa/mdmanual/43const/html/const.html","https://cdn.civil.services/us-states/flags/maryland-large.png","https://cdn.civil.services/us-states/seals/maryland-large.png","https://cdn.civil.services/us-states/maps/maryland-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/maryland.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/maryland.jpg","https://twitter.com/statemaryland","https://www.facebook.com/statemaryland" "Massachusetts","massachusetts","MA","Bay State","http://www.mass.gov","1788-02-06",6,"Boston","http://www.ci.boston.ma.us",6692824,14,"http://www.state.ma.us/legis/const.htm","https://cdn.civil.services/us-states/flags/massachusetts-large.png","https://cdn.civil.services/us-states/seals/massachusetts-large.png","https://cdn.civil.services/us-states/maps/massachusetts-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/massachusetts.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/massachusetts.jpg","http://twitter.com/massgov","https://www.facebook.com/massgov" "Michigan","michigan","MI","Wolverine State / Great Lakes State","http://www.michigan.gov","1837-01-26",26,"Lansing","http://cityoflansingmi.com",9895622,9,"http://www.legislature.mi.gov/(S(hrowl12tg05hemnnkidim1jb))/mileg.aspx?page=GetObject&objectname=mcl-Constitution","https://cdn.civil.services/us-states/flags/michigan-large.png","https://cdn.civil.services/us-states/seals/michigan-large.png","https://cdn.civil.services/us-states/maps/michigan-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/michigan.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/michigan.jpg","https://twitter.com/migov","https://www.facebook.com/MIgovernment" "Minnesota","minnesota","MN","North Star State / Land of 10,000 Lakes","https://mn.gov","1858-05-11",32,"Saint Paul","http://www.stpaul.gov",5420380,21,"http://www.house.leg.state.mn.us/cco/rules/mncon/preamble.htm","https://cdn.civil.services/us-states/flags/minnesota-large.png","https://cdn.civil.services/us-states/seals/minnesota-large.png","https://cdn.civil.services/us-states/maps/minnesota-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/minnesota.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/minnesota.jpg",, "Mississippi","mississippi","MS","Magnolia State","http://www.ms.gov","1817-12-10",20,"Jackson","http://www.city.jackson.ms.us",2991207,31,"http://law.justia.com/constitution/mississippi","https://cdn.civil.services/us-states/flags/mississippi-large.png","https://cdn.civil.services/us-states/seals/mississippi-large.png","https://cdn.civil.services/us-states/maps/mississippi-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/mississippi.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/mississippi.jpg","https://twitter.com/msdotgov","https://www.facebook.com/msdotgov" "Missouri","missouri","MO","Show Me State","https://www.mo.gov","1821-08-10",24,"Jefferson City","http://www.jeffcitymo.org",6044171,18,"http://www.moga.mo.gov/mostatutes/moconstn.html","https://cdn.civil.services/us-states/flags/missouri-large.png","https://cdn.civil.services/us-states/seals/missouri-large.png","https://cdn.civil.services/us-states/maps/missouri-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/missouri.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/missouri.jpg","https://twitter.com/MoGov","https://www.facebook.com/mogov" "Montana","montana","MT","Treasure State","http://mt.gov","1889-11-08",41,"Helena","http://www.ci.helena.mt.us",1015165,44,"http://courts.mt.gov/content/library/docs/72constit.pdf","https://cdn.civil.services/us-states/flags/montana-large.png","https://cdn.civil.services/us-states/seals/montana-large.png","https://cdn.civil.services/us-states/maps/montana-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/montana.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/montana.jpg",, "Nebraska","nebraska","NE","Cornhusker State","http://www.nebraska.gov","1867-03-01",37,"Lincoln","http://lincoln.ne.gov",1868516,37,"http://www.state.ne.us/legislative/statutes/C","https://cdn.civil.services/us-states/flags/nebraska-large.png","https://cdn.civil.services/us-states/seals/nebraska-large.png","https://cdn.civil.services/us-states/maps/nebraska-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/nebraska.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/nebraska.jpg","https://twitter.com/Nebraskagov","https://www.facebook.com/nebraska.gov" "Nevada","nevada","NV","The Silver State","http://nv.gov","1864-10-31",36,"Carson City","http://www.carson.org",2790136,35,"http://www.leg.state.nv.us/Const/NvConst.html","https://cdn.civil.services/us-states/flags/nevada-large.png","https://cdn.civil.services/us-states/seals/nevada-large.png","https://cdn.civil.services/us-states/maps/nevada-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/nevada.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/nevada.jpg",, "New Hampshire","new-hampshire","NH","Granite State","https://www.nh.gov","1788-06-21",9,"Concord","http://www.concordnh.gov",1323459,42,"http://www.state.nh.us/constitution/constitution.html","https://cdn.civil.services/us-states/flags/new-hampshire-large.png","https://cdn.civil.services/us-states/seals/new-hampshire-large.png","https://cdn.civil.services/us-states/maps/new-hampshire-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/new-hampshire.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/new-hampshire.jpg","https://twitter.com/nhgov", "New Jersey","new-jersey","NJ","Garden State","http://www.state.nj.us","1787-12-18",3,"Trenton","http://www.trentonnj.org",8899339,11,"http://www.njleg.state.nj.us/lawsconstitution/consearch.asp","https://cdn.civil.services/us-states/flags/new-jersey-large.png","https://cdn.civil.services/us-states/seals/new-jersey-large.png","https://cdn.civil.services/us-states/maps/new-jersey-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/new-jersey.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/new-jersey.jpg",, "New Mexico","new-mexico","NM","Land of Enchantment","http://www.newmexico.gov","1912-01-06",47,"Santa Fe","http://www.santafenm.gov",2085287,36,"http://www.loc.gov/law/guide/us-nm.html","https://cdn.civil.services/us-states/flags/new-mexico-large.png","https://cdn.civil.services/us-states/seals/new-mexico-large.png","https://cdn.civil.services/us-states/maps/new-mexico-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/new-mexico.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/new-mexico.jpg",, "New York","new-york","NY","Empire State","http://www.ny.gov","1788-07-26",11,"Albany","http://www.albanyny.org",19651127,3,"https://www.dos.ny.gov/info/constitution.htm","https://cdn.civil.services/us-states/flags/new-york-large.png","https://cdn.civil.services/us-states/seals/new-york-large.png","https://cdn.civil.services/us-states/maps/new-york-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/new-york.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/new-york.jpg","https://twitter.com/nygov", "North Carolina","north-carolina","NC","Old North State / Tar Heel State","http://www.nc.gov","1789-11-21",12,"Raleigh","http://www.raleigh-nc.org",9848060,10,"http://statelibrary.dcr.state.nc.us/nc/stgovt/preconst.htm","https://cdn.civil.services/us-states/flags/north-carolina-large.png","https://cdn.civil.services/us-states/seals/north-carolina-large.png","https://cdn.civil.services/us-states/maps/north-carolina-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/north-carolina.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/north-carolina.jpg","https://twitter.com/NCdotGov", "North Dakota","north-dakota","ND","Peace Garden State / Flickertail State / Roughrider State","http://www.nd.gov","1889-11-02",39,"Bismarck","http://www.bismarck.org",723393,48,"http://www.legis.nd.gov/information/statutes/const-laws.html","https://cdn.civil.services/us-states/flags/north-dakota-large.png","https://cdn.civil.services/us-states/seals/north-dakota-large.png","https://cdn.civil.services/us-states/maps/north-dakota-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/north-dakota.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/north-dakota.jpg","https://twitter.com/ExperienceND","https://www.facebook.com/ExperienceND" "Ohio","ohio","OH","Buckeye State","https://ohio.gov","1803-03-01",17,"Columbus","http://ci.columbus.oh.us",11570808,7,"http://www.legislature.state.oh.us/constitution.cfm","https://cdn.civil.services/us-states/flags/ohio-large.png","https://cdn.civil.services/us-states/seals/ohio-large.png","https://cdn.civil.services/us-states/maps/ohio-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/ohio.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/ohio.jpg","https://twitter.com/ohgov", "Oklahoma","oklahoma","OK","Sooner State","https://www.ok.gov","1907-11-16",46,"Oklahoma City","http://www.okc.gov",3850568,28,"http://oklegal.onenet.net/okcon","https://cdn.civil.services/us-states/flags/oklahoma-large.png","https://cdn.civil.services/us-states/seals/oklahoma-large.png","https://cdn.civil.services/us-states/maps/oklahoma-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/oklahoma.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/oklahoma.jpg","https://twitter.com/okgov","https://www.facebook.com/okgov" "Oregon","oregon","OR","Beaver State","http://www.oregon.gov","1859-02-14",33,"Salem","http://www.cityofsalem.net/Pages/default.aspx",3930065,27,"http://bluebook.state.or.us/state/constitution/constitution.htm","https://cdn.civil.services/us-states/flags/oregon-large.png","https://cdn.civil.services/us-states/seals/oregon-large.png","https://cdn.civil.services/us-states/maps/oregon-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/oregon.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/oregon.jpg",, "Pennsylvania","pennsylvania","PA","Keystone State","http://www.pa.gov","1787-12-12",2,"Harrisburg","http://harrisburgpa.gov",12773801,6,"http://sites.state.pa.us/PA_Constitution.html","https://cdn.civil.services/us-states/flags/pennsylvania-large.png","https://cdn.civil.services/us-states/seals/pennsylvania-large.png","https://cdn.civil.services/us-states/maps/pennsylvania-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/pennsylvania.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/pennsylvania.jpg","https://www.facebook.com/visitPA","https://twitter.com/visitPA" "Rhode Island","rhode-island","RI","The Ocean State","https://www.ri.gov","1790-05-29",13,"Providence","http://www.providenceri.com",1051511,43,"http://webserver.rilin.state.ri.us/RiConstitution","https://cdn.civil.services/us-states/flags/rhode-island-large.png","https://cdn.civil.services/us-states/seals/rhode-island-large.png","https://cdn.civil.services/us-states/maps/rhode-island-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/rhode-island.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/rhode-island.jpg","https://twitter.com/rigov","https://www.facebook.com/RIgov-Rhode-Island-Government-Online-24056655991" "South Carolina","south-carolina","SC","Palmetto State","http://www.sc.gov","1788-05-23",8,"Columbia","http://www.columbiasc.net",4774839,24,"http://www.scstatehouse.gov/scconstitution/scconst.php","https://cdn.civil.services/us-states/flags/south-carolina-large.png","https://cdn.civil.services/us-states/seals/south-carolina-large.png","https://cdn.civil.services/us-states/maps/south-carolina-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/south-carolina.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/south-carolina.jpg","https://twitter.com/scgov","http://www.facebook.com/pages/SCgov/12752057990" "South Dakota","south-dakota","SD","Mount Rushmore State","http://sd.gov","1889-11-02",40,"Pierre","http://ci.pierre.sd.us",844877,46,"http://legis.sd.gov/statutes/Constitution","https://cdn.civil.services/us-states/flags/south-dakota-large.png","https://cdn.civil.services/us-states/seals/south-dakota-large.png","https://cdn.civil.services/us-states/maps/south-dakota-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/south-dakota.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/south-dakota.jpg",, "Tennessee","tennessee","TN","Volunteer State","https://www.tn.gov","1796-06-01",16,"Nashville","http://www.nashville.gov",6495978,17,"http://www.capitol.tn.gov/about/docs/TN-Constitution.pdf","https://cdn.civil.services/us-states/flags/tennessee-large.png","https://cdn.civil.services/us-states/seals/tennessee-large.png","https://cdn.civil.services/us-states/maps/tennessee-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/tennessee.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/tennessee.jpg","https://twitter.com/TNVacation","https://www.facebook.com/tnvacation" "Texas","texas","TX","Lone Star State","https://www.texas.gov","1845-12-29",28,"Austin","http://www.austintexas.gov",26448193,2,"http://www.constitution.legis.state.tx.us","https://cdn.civil.services/us-states/flags/texas-large.png","https://cdn.civil.services/us-states/seals/texas-large.png","https://cdn.civil.services/us-states/maps/texas-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/texas.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/texas.jpg","https://twitter.com/texasgov","http://www.facebook.com/Texas.gov" "Utah","utah","UT","The Beehive State","https://utah.gov","1896-01-04",45,"Salt Lake City","http://www.slcgov.com",2900872,33,"http://le.utah.gov/UtahCode/chapter.jsp?code=Constitution","https://cdn.civil.services/us-states/flags/utah-large.png","https://cdn.civil.services/us-states/seals/utah-large.png","https://cdn.civil.services/us-states/maps/utah-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/utah.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/utah.jpg","https://twitter.com/UtahGov","https://www.facebook.com/utahgov" "Vermont","vermont","VT","Green Mountain State","http://vermont.gov","1791-03-04",14,"Montpelier","http://www.montpelier-vt.org",626630,49,"http://www.leg.state.vt.us/statutes/const2.htm","https://cdn.civil.services/us-states/flags/vermont-large.png","https://cdn.civil.services/us-states/seals/vermont-large.png","https://cdn.civil.services/us-states/maps/vermont-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/vermont.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/vermont.jpg","https://twitter.com/vermontgov","https://www.facebook.com/MyVermont" "Virginia","virginia","VA","Old Dominion State","https://www.virginia.gov","1788-06-25",10,"Richmond","http://www.richmondgov.com",8260405,12,"http://hodcap.state.va.us/publications/Constitution-01-13.pdf","https://cdn.civil.services/us-states/flags/virginia-large.png","https://cdn.civil.services/us-states/seals/virginia-large.png","https://cdn.civil.services/us-states/maps/virginia-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/virginia.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/virginia.jpg",, "Washington","washington","WA","The Evergreen State","http://www.wa.gov","1889-11-11",42,"Olympia","http://www.ci.olympia.wa.us",6971406,13,"http://www.leg.wa.gov/lawsandagencyrules/pages/constitution.aspx","https://cdn.civil.services/us-states/flags/washington-large.png","https://cdn.civil.services/us-states/seals/washington-large.png","https://cdn.civil.services/us-states/maps/washington-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/washington.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/washington.jpg","https://twitter.com/wagov","" "West Virginia","west-virginia","WV","Mountain State","http://www.wv.gov","1863-06-20",35,"Charleston","http://www.cityofcharleston.org",1854304,38,"http://www.legis.state.wv.us/WVCODE/WV_CON.cfm","https://cdn.civil.services/us-states/flags/west-virginia-large.png","https://cdn.civil.services/us-states/seals/west-virginia-large.png","https://cdn.civil.services/us-states/maps/west-virginia-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/west-virginia.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/west-virginia.jpg","https://twitter.com/wvgov","https://www.facebook.com/wvgov" "Wisconsin","wisconsin","WI","Badger State","https://www.wisconsin.gov","1848-05-29",30,"Madison","http://www.ci.madison.wi.us",5742713,20,"http://www.legis.state.wi.us/rsb/2wiscon.html","https://cdn.civil.services/us-states/flags/wisconsin-large.png","https://cdn.civil.services/us-states/seals/wisconsin-large.png","https://cdn.civil.services/us-states/maps/wisconsin-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/wisconsin.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/wisconsin.jpg",, "Wyoming","wyoming","WY","Equality State","http://www.wyo.gov","1890-07-10",44,"Cheyenne","http://www.cheyennecity.org",582658,50,"http://legisweb.state.wy.us/statutes/constitution.aspx?file=titles/97Title97.htm","https://cdn.civil.services/us-states/flags/wyoming-large.png","https://cdn.civil.services/us-states/seals/wyoming-large.png","https://cdn.civil.services/us-states/maps/wyoming-large.png","https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/wyoming.jpg","https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/wyoming.jpg",, ================================================ FILE: intake/catalog/tests/catalog.yml ================================================ plugins: source: - module: intake.catalog.tests.example1_source sources: use_example1: description: example1 source plugin driver: example1 args: {} allowed_list_absolute: description: pick one list out of a few driver: example1 args: arg1: "{{up}}" parameters: mylist: description: "" type: list default: [] allowed: - [] - ["one"] - ["one", "two"] allowed_list_multi: description: pick multiple values from an allowed list driver: example1 args: args1: "{{up}}" parameters: mylist: description: "" type: list default: [] allowed: ["one", "two", "three"] ================================================ FILE: intake/catalog/tests/catalog1.yml ================================================ name: name_in_cat metadata: test: true plugins: source: - module: intake.catalog.tests.example1_source - module: intake.catalog.tests.example_plugin_dir.example2_source sources: use_example1: description: example1 source plugin driver: example1 args: {} nested: description: around again driver: yaml_file_cat args: path: '{{ CATALOG_DIR }}/catalog1.yml' entry1: description: entry1 full metadata: foo: 'bar' bar: [1, 2, 3] driver: csv # Default direct_access is "forbid" by default args: # passed to the open() method urlpath: '{{ CATALOG_DIR }}/entry1_*.csv' entry1_part: description: entry1 part parameters: # User defined parameters part: description: part of filename type: str default: "1" allowed: ["1", "2"] metadata: foo: 'baz' bar: [2, 4, 6] driver: csv direct_access: "allow" args: # passed to the open() method urlpath: '{{ CATALOG_DIR }}/entry1_{{ part }}.csv' remote_env: description: env gets interpreted in server driver: intake.conftest.TestSource args: urlpath: 'path-{{intake_test}}' parameters: intake_test: description: none type: str default: 'env(INTAKE_TEST)' local_env: description: env gets interpreted in client driver: csv args: urlpath: 'path-{{intake_test}}' parameters: intake_test: description: none type: str default: 'client_env(INTAKE_TEST)' text: description: textfiles in this dir driver: textfiles args: urlpath: "{{ CATALOG_DIR }}/*.yml" arr: description: small array driver: numpy args: path: "{{ CATALOG_DIR }}/small.npy" chunks: 5 datetime: description: datetime parameters driver: intake.conftest.TestSource args: urlpath: "{{time}}" parameters: time: description: some time type: datetime ================================================ FILE: intake/catalog/tests/catalog_alias.yml ================================================ sources: input_data: description: a local data file driver: csv args: urlpath: '{{ CATALOG_DIR }}/cache_data/states.csv' arr_cache: description: small array driver: numpy args: path: "{{ CATALOG_DIR }}/small.npy" chunks: 5 alias0: driver: intake.source.derived.AliasSource args: target: input_data barebones: driver: intake.source.derived.GenericTransform args: targets: - input_data transform: builtins.len transform_kwargs: {} alias1: driver: alias args: target: "{{choice}}" mapping: first: input_data second: arr_cache parameters: choice: description: which to alias type: str default: first allowed: ["first", "second"] derive_cols: driver: intake.source.derived.Columns args: targets: - input_data columns: ["state", "slug"] derive_cols_func: driver: intake.source.derived.DataFrameTransform args: targets: - input_data transform: "intake.source.tests.test_derived._pick_columns" transform_kwargs: columns: ["state", "slug"] other_cat: driver: intake.source.derived.Columns args: targets: - "{{CATALOG_DIR}}/catalog1.yml:entry1" columns: ["name", "score"] alias_other_cat: driver: alias args: target: "{{CATALOG_DIR}}/catalog1.yml:entry1" kwargs: csv_kwargs: {parse_dates: True} cat_kwargs: getenv: true ================================================ FILE: intake/catalog/tests/catalog_caching.yml ================================================ metadata: test: true plugins: source: - module: intake.catalog.tests.example1_source - module: intake.catalog.tests.example2_source sources: test_cache: description: cache a csv file from the local filesystem driver: csv cache: - argkey: urlpath regex: '{{ CATALOG_DIR }}/cache_data' type: file args: urlpath: '{{ CATALOG_DIR }}/cache_data/states.csv' test_cache_new: description: cache a csv file from the local filesystem driver: csv args: urlpath: 'filecache://{{ CATALOG_DIR }}/cache_data/states.csv' storage_options: target_protocol: 'file' cache_storage: "{{env(TEST_CACHE_DIR)}}" test_multiple_cache: description: testing what happens when there are multiple cache specs driver: csv cache: - argkey: urlpath regex: '{{ CATALOG_DIR }}/cache_data' type: file - argkey: urlpath regex: '{{ CATALOG_DIR }}' type: file args: urlpath: '{{ CATALOG_DIR }}/cache_data/states.csv' test_list_cache: description: testing what happens when there are multiple cache specs driver: csv cache: - argkey: urlpath regex: '{{ CATALOG_DIR }}/cache_data' type: file args: urlpath: ['{{ CATALOG_DIR }}/cache_data/states.csv', '{{ CATALOG_DIR }}/cache_data/states.csv'] test_bad_type_cache_spec: description: cache a csv file from the local filesystem driver: csv cache: - argkey: urlpath regex: '{{ CATALOG_DIR }}/cache_data' type: noidea args: urlpath: '{{ CATALOG_DIR }}/cache_data/states.csv' text_cache: description: textfiles in this dir driver: textfiles cache: - argkey: urlpath regex: '{{ CATALOG_DIR }}' type: file args: urlpath: "{{ CATALOG_DIR }}/*.yml" arr_cache: description: small array driver: numpy cache: - argkey: path regex: '{{ CATALOG_DIR }}' type: file args: path: "{{ CATALOG_DIR }}/small.npy" chunks: 5 test_no_regex: description: cache a csv file from the local filesystem driver: csv cache: - argkey: urlpath type: file args: urlpath: '{{ CATALOG_DIR }}/cache_data/states.csv' test_regex_no_match: description: regex does not match urlpath driver: csv cache: - argkey: urlpath regex: 'xxx' type: file args: urlpath: '{{ CATALOG_DIR }}/cache_data/states.csv' test_regex_partial_match: description: regex matches some part of the url driver: csv cache: - argkey: urlpath regex: '_data' type: file args: urlpath: '{{ CATALOG_DIR }}/cache_data/states.csv' ================================================ FILE: intake/catalog/tests/catalog_dup_parameters.yml ================================================ sources: entry1_part: description: entry1 part parameters: part: description: a type: str part: description: b type: int driver: csv args: # passed to the open() method urlpath: '{{ CATALOG_DIR }}/entry1_{{ part }}.csv' entry2_part: description: entry2 part parameters: part: description: a type: str driver: csv args: # passed to the open() method urlpath: '{{ CATALOG_DIR }}/entry2_{{ part }}.csv' ================================================ FILE: intake/catalog/tests/catalog_dup_sources.yml ================================================ sources: entry1_part: description: entry1 part parameters: part: description: a type: str driver: csv args: # passed to the open() method urlpath: '{{ CATALOG_DIR }}/entry1_{{ part }}.csv' entry1_part: description: entry1 part parameters: part: description: a type: str driver: csv args: # passed to the open() method urlpath: '{{ CATALOG_DIR }}/entry1_{{ part }}.csv' ================================================ FILE: intake/catalog/tests/catalog_hierarchy.yml ================================================ sources: a.b.c: description: abc driver: csv args: urlpath: '{{ CATALOG_DIR }}/entry1_*.csv' a.b.d: description: abc driver: csv args: urlpath: '{{ CATALOG_DIR }}/entry1_*.csv' c: description: abc driver: csv args: urlpath: '{{ CATALOG_DIR }}/entry1_*.csv' a.c: description: abc driver: csv parameters: part: description: part of filename type: str default: "1" allowed: ["1", "2"] driver: csv args: urlpath: '{{ CATALOG_DIR }}/entry1_{{ part }}.csv' ================================================ FILE: intake/catalog/tests/catalog_named.yml ================================================ name: name_in_spec description: This is a catalog with a description in the yaml metadata: some: thing plugins: source: - module: intake.catalog.tests.example1_source sources: use_example1: description: example1 source plugin driver: example1 args: {} ================================================ FILE: intake/catalog/tests/catalog_non_dict.yml ================================================ - 1 - 2 - 3 ================================================ FILE: intake/catalog/tests/catalog_search/example_packages/ep/__init__.py ================================================ class TestCatalog: ... ================================================ FILE: intake/catalog/tests/catalog_search/example_packages/ep-0.1.dist-info/entry_points.txt ================================================ [intake.catalogs] ep1 = ep:TestCatalog ================================================ FILE: intake/catalog/tests/catalog_search/yaml.yml ================================================ plugins: source: - module: intake.catalog.tests.example1_source sources: use_example1: description: example1 source plugin driver: example1 args: {} ================================================ FILE: intake/catalog/tests/catalog_union_1.yml ================================================ plugins: source: - module: intake.catalog.tests.example1_source sources: use_example1: description: example1 source plugin driver: example1 args: {} ================================================ FILE: intake/catalog/tests/catalog_union_2.yml ================================================ plugins: source: - module: intake.catalog.tests.example1_source - module: intake.catalog.tests.example2_source sources: entry1: description: entry1 full metadata: foo: 'bar' bar: [1, 2, 3] driver: csv # Default direct_access is "forbid" by default args: # passed to the open() method urlpath: '{{ CATALOG_DIR }}/entry1_*.csv' entry1_part: description: entry1 part parameters: # User defined parameters part: description: part of filename type: str default: "1" allowed: ["1", "2"] metadata: foo: 'baz' bar: [2, 4, 6] driver: csv direct_access: "allow" args: # passed to the open() method urlpath: '{{ CATALOG_DIR }}/entry1_{{ part }}.csv' ================================================ FILE: intake/catalog/tests/conftest.py ================================================ import os.path import pytest from intake import open_catalog @pytest.fixture def catalog1(): path = os.path.dirname(__file__) return open_catalog(os.path.join(path, "catalog1.yml")) ================================================ FILE: intake/catalog/tests/data_source_missing.yml ================================================ plugins: source: [] ================================================ FILE: intake/catalog/tests/data_source_name_non_string.yml ================================================ sources: 1: foo ================================================ FILE: intake/catalog/tests/data_source_non_dict.yml ================================================ sources: foo ================================================ FILE: intake/catalog/tests/data_source_value_non_dict.yml ================================================ sources: foo: 1 ================================================ FILE: intake/catalog/tests/dot-nest.yaml ================================================ sources: self: description: this cat driver: yaml_file_cat args: path: "{{CATALOG_DIR}}/dot-nest.yaml" selfdot.dot: description: this cat driver: yaml_file_cat args: path: "{{CATALOG_DIR}}/dot-nest.yaml" self.dot: description: this cat driver: yaml_file_cat args: path: "{{CATALOG_DIR}}/dot-nest.yaml" leaf: description: leaf driver: csv args: urlpath: "" leafdot.dot: description: leaf-dot driver: csv args: urlpath: "" leaf.dot: description: leaf-dot driver: csv args: urlpath: "" ================================================ FILE: intake/catalog/tests/entry1_1.csv ================================================ name,score,rank Alice1,100.5,1 Bob1,50.3,2 Charlie1,25,3 Eve1,25,3 ================================================ FILE: intake/catalog/tests/entry1_2.csv ================================================ name,score,rank Alice2,100.5,1 Bob2,50.3,2 Charlie2,25,3 Eve2,25,3 ================================================ FILE: intake/catalog/tests/example1_source.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from intake.source.base import DataSource class ExampleSource(DataSource): name = "example1" version = "0.1" container = "dataframe" partition_access = True def __init__(self, **kwargs): self.kwargs = kwargs super(ExampleSource, self).__init__() ================================================ FILE: intake/catalog/tests/example_plugin_dir/example2_source.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from intake.source.base import DataSource class Ex2Plugin(DataSource): name = "example2" version = "0.1" container = "dataframe" partition_access = True def __init__(self): super(Ex2Plugin, self).__init__() ================================================ FILE: intake/catalog/tests/multi_plugins.yaml ================================================ sources: tables0: args: urlpath: "{{ CATALOG_DIR }}/files*" description: "short form" driver: - csv metadata: {} tables1: args: urlpath: "{{ CATALOG_DIR }}/files*" description: "long form" driver: - intake.source.csv.CSVSource metadata: {} tables2: args: urlpath: "{{ CATALOG_DIR }}/files*" description: "same plugin twice" driver: - csv - intake.source.csv.CSVSource metadata: {} tables3: args: urlpath: "{{ CATALOG_DIR }}/files*" description: "user's choice with extra param" driver: myplug: class: intake.source.csv.CSVSource myplug2: class: intake.source.csv.CSVSource args: csv_kwargs: true metadata: {} tables4: args: urlpath: "{{ CATALOG_DIR }}/files*" description: "neither plugins exist" driver: myplug: class: doesnotexist myplug2: class: also.none.Class metadata: {} tables5: args: urlpath: "{{ CATALOG_DIR }}/files*" description: "only second plugin exists" driver: myplug: class: doesnotexist myplug2: class: csv metadata: {} tables6: args: urlpath: "{{ CATALOG_DIR }}/files*" description: "no valid plugin in list" driver: myplug: class: doesnotexist metadata: {} tables7: args: urlpath: "{{ CATALOG_DIR }}/files*" description: "no valid plugin" driver: doesnotexist metadata: {} ================================================ FILE: intake/catalog/tests/multi_plugins2.yaml ================================================ sources: tables6: args: urlpath: "{{ CATALOG_DIR }}/files*" description: "incompatible plugins" driver: myplug: class: csv myplug2: class: numpy metadata: {} ================================================ FILE: intake/catalog/tests/obsolete_data_source_list.yml ================================================ sources: - name: a driver: csv ================================================ FILE: intake/catalog/tests/obsolete_params_list.yml ================================================ sources: a: driver: csv parameters: - name: b ================================================ FILE: intake/catalog/tests/params_missing_required.yml ================================================ sources: a: description: A ================================================ FILE: intake/catalog/tests/params_name_non_string.yml ================================================ sources: a: driver: csv parameters: 1: {} ================================================ FILE: intake/catalog/tests/params_non_dict.yml ================================================ sources: a: driver: csv parameters: b ================================================ FILE: intake/catalog/tests/params_value_bad_choice.yml ================================================ sources: a: driver: csv parameters: b: description: B type: string ================================================ FILE: intake/catalog/tests/params_value_bad_type.yml ================================================ sources: a: driver: csv parameters: b: description: 1 type: str ================================================ FILE: intake/catalog/tests/params_value_non_dict.yml ================================================ sources: a: driver: csv parameters: b: 1 ================================================ FILE: intake/catalog/tests/plugins_non_dict.yml ================================================ plugins: 0 sources: {} ================================================ FILE: intake/catalog/tests/plugins_source_missing.yml ================================================ plugins: s0urce: [] sources: {} ================================================ FILE: intake/catalog/tests/plugins_source_missing_key.yml ================================================ plugins: source: - directory: /tmp sources: {} ================================================ FILE: intake/catalog/tests/plugins_source_non_dict.yml ================================================ plugins: source: - module sources: {} ================================================ FILE: intake/catalog/tests/plugins_source_non_list.yml ================================================ plugins: source: module sources: {} ================================================ FILE: intake/catalog/tests/plugins_source_non_string.yml ================================================ plugins: source: - module: 0 sources: {} ================================================ FILE: intake/catalog/tests/test_alias.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import os import intake here = os.path.abspath(os.path.dirname(__file__)) fn = os.path.join(here, "catalog_alias.yml") def test_simple(): cat = intake.open_catalog(fn) s = cat.alias0() assert s.container == "other" out = str(s.discover()) assert s.container == "dataframe" assert "state" in out def test_mapping(): cat = intake.open_catalog(fn) s = cat.alias1() assert s.container == "other" out = str(s.discover()) assert s.container == "dataframe" assert "state" in out s = cat.alias1(choice="second") assert s.container == "other" out = str(s.discover()) assert s.container == "ndarray" assert "int64" in out def test_other_cat(): cat = intake.open_catalog(fn) other = cat.alias_other_cat assert other.source is None assert other.cat.name == cat.name _ = other.discover() assert other.source is not None assert other.source.cat.name == "name_in_cat" assert other.source._csv_kwargs == {"parse_dates": True} assert other.source.cat._captured_init_kwargs == {"getenv": True} def test_alias(): """make sure aliases are set up correctly""" cat = intake.entry.Catalog() filepath = os.path.join(here, "entry1_1.csv") data = intake.readers.datatypes.CSV(filepath) reader = intake.readers.readers.PandasCSV(data) # modify reader reader_modified = reader.name.lower() # create catalog entry cat["entry1_1"] = reader_modified # Make sure that only the key/alias/name is # in the list of entries and aliases assert list(cat) == ["entry1_1"] assert cat.aliases == {"entry1_1": "entry1_1"} ================================================ FILE: intake/catalog/tests/test_catalog_save.py ================================================ """ Test saving catalogs. """ import os import intake from intake.catalog import Catalog from intake.catalog.local import LocalCatalogEntry def test_catalog_description(tmpdir): """Make sure the description comes through the save.""" tmpdir = str(tmpdir) cat_path = os.path.join(tmpdir, "desc_test.yaml") cat1 = Catalog.from_dict( { "name": LocalCatalogEntry( "name", description="description", driver=intake.catalog.local.YAMLFileCatalog, ) }, name="overall catalog name", description="overall catalog description", ) cat1.save(cat_path) cat2 = intake.open_catalog(cat_path) assert cat2.description is not None ================================================ FILE: intake/catalog/tests/test_core.py ================================================ import pytest from intake.catalog.base import Catalog def test_no_entry(): cat = Catalog() cat2 = cat.configure_new() assert isinstance(cat2, Catalog) assert cat.auth is None assert cat2.auth is None def test_regression(): with pytest.raises(ValueError): Catalog("URI") ================================================ FILE: intake/catalog/tests/test_default.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import sys from pathlib import Path from intake.catalog import default from intake.catalog.base import Catalog def test_which(): p = default.which("python") assert Path(p).resolve() == Path(sys.executable).resolve() def test_load(): cat = default.load_user_catalog() assert isinstance(cat, Catalog) cat = default.load_global_catalog() assert isinstance(cat, Catalog) ================================================ FILE: intake/catalog/tests/test_discovery.py ================================================ import os import sys from ..local import EntrypointsCatalog, MergedCatalog, YAMLFilesCatalog def test_catalog_discovery(): basedir = os.path.dirname(__file__) yaml_glob = os.path.join(basedir, "catalog_search", "*.yml") example_packages = os.path.join(basedir, "catalog_search", "example_packages") test_catalog = MergedCatalog( [ EntrypointsCatalog(paths=[example_packages]), YAMLFilesCatalog(path=[yaml_glob]), ] ) assert "use_example1" in test_catalog assert "ep1" in test_catalog def test_deferred_import(): "See https://github.com/intake/intake/pull/541" # We are going to mess with sys.modules here, so to be safe let's put it # back the way it was at the end. import intake.catalog intake.catalog.builtin = None mods = sys.modules.copy() try: sys.modules.pop("intake") sys.modules.pop("intake.catalog") intake.catalog.__dict__.pop("builtin") assert "builtin" not in intake.catalog.__dict__ assert intake.cat is not None finally: sys.modules.update(mods) ================================================ FILE: intake/catalog/tests/test_gui.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2019, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import pytest def panel_importable(): try: import panel as pn # noqa return True except: return False EXPECTED_ERROR_TEXT = "Please install panel to use the GUI" @pytest.mark.skipif(panel_importable(), reason="panel is importable, so skip") def test_cat_no_panel_does_not_raise_errors(catalog1): assert catalog1.name == "name_in_cat" @pytest.mark.skipif(panel_importable(), reason="panel is importable, so skip") def test_cat_no_panel_display_gui(catalog1): with pytest.raises(RuntimeError, match=EXPECTED_ERROR_TEXT): repr(catalog1.gui) def test_cat_gui(catalog1): pytest.importorskip("panel") assert repr(catalog1.gui).startswith("Intake") @pytest.mark.skipif(panel_importable(), reason="panel is importable, so skip") def test_entry_no_panel_does_not_raise_errors(catalog1): assert catalog1.entry1.name == "entry1" @pytest.mark.skipif(panel_importable(), reason="panel is importable, so skip") def test_entry_no_panel_display_gui(catalog1): with pytest.raises(RuntimeError, match=EXPECTED_ERROR_TEXT): repr(catalog1.entry1.gui) ================================================ FILE: intake/catalog/tests/test_local.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import datetime import os.path import shutil import tempfile import time import pandas import pytest from fsspec.implementations.local import make_path_posix from intake import open_catalog from intake.catalog import exceptions, local from intake.catalog.local import LocalCatalogEntry, UserParameter, get_dir from .util import assert_items_equal def abspath(filename): return make_path_posix(os.path.join(os.path.dirname(__file__), filename)) def test_local_catalog(catalog1): assert_items_equal( list(catalog1), [ "use_example1", "nested", "entry1", "entry1_part", "remote_env", "local_env", "text", "arr", "datetime", ], ) assert len(catalog1) == 9 assert catalog1["entry1"].describe() == { "name": "entry1", "container": "dataframe", "direct_access": "forbid", "user_parameters": [], "description": "entry1 full", "args": {"urlpath": "{{ CATALOG_DIR }}/entry1_*.csv"}, "metadata": {"bar": [1, 2, 3], "foo": "bar"}, "plugin": ["csv"], "driver": ["csv"], } assert catalog1["entry1_part"].describe() == { "name": "entry1_part", "container": "dataframe", "user_parameters": [ { "name": "part", "description": "part of filename", "default": "1", "type": "str", "allowed": ["1", "2"], } ], "description": "entry1 part", "direct_access": "allow", "args": {"urlpath": "{{ CATALOG_DIR }}/entry1_{{ part }}.csv"}, "metadata": {"foo": "baz", "bar": [2, 4, 6]}, "plugin": ["csv"], "driver": ["csv"], } assert catalog1["entry1"].container == "dataframe" md = catalog1["entry1"].metadata md.pop("catalog_dir") assert md["foo"] == "bar" assert md["bar"] == [1, 2, 3] # Use default parameters assert catalog1["entry1_part"].container == "dataframe" # Specify parameters assert catalog1["entry1_part"].configure_new(part="2").container == "dataframe" def test_get_items(catalog1): for key, entry in catalog1.items(): assert catalog1[key].describe() == entry.describe() def test_nested(catalog1): assert "nested" in catalog1 assert "entry1" in catalog1.nested.nested() assert catalog1.entry1.read().equals(catalog1.nested.nested.entry1.read()) assert "nested.nested" not in catalog1.walk(depth=1) assert "nested.nested" in catalog1.walk(depth=2) assert catalog1.nested.cat == catalog1 assert catalog1.nested.nested.nested.cat.cat.cat is catalog1 def test_nested_gets_name_from_super(catalog1): assert catalog1.name == "name_in_cat" assert "nested" in catalog1 nested = catalog1.nested assert nested.name == "nested" assert nested().name == "nested" def test_hash(catalog1): assert catalog1.nested() == catalog1.nested.nested() def test_getitem(catalog1): assert list(catalog1) == list(catalog1["nested"]()) assert list(catalog1) == list(catalog1["nested.nested"]()) assert list(catalog1) == list(catalog1["nested", "nested"]()) def test_source_plugin_config(catalog1): from intake import registry assert "example1" in registry assert "example2" in registry def test_metadata(catalog1): assert hasattr(catalog1, "metadata") assert catalog1.metadata["test"] is True def test_use_source_plugin_from_config(catalog1): catalog1["use_example1"] def test_get_dir(): assert get_dir("file:///path/catalog.yml") == "file:///path" assert get_dir("https://example.com/catalog.yml") == "https://example.com" path = "example/catalog.yml" out = get_dir(path) assert os.path.isabs(out) assert out.endswith("/example/") path = "/example/catalog.yml" out = get_dir(path) # it's ok if the first two chars indicate drive for win (C:) assert "/example/" in [out, out[2:]] path = "example" out = get_dir(path) assert os.path.isabs(out) assert not out.endswith("/example") assert out.endswith("/") def test_entry_dir_function(catalog1): assert "nested" in dir(catalog1.nested) @pytest.mark.parametrize( "dtype,expected", [ ("bool", False), ("datetime", pandas.Timestamp(1970, 1, 1, 0, 0, 0)), ("float", 0.0), ("int", 0), ("list", []), ("str", ""), ("unicode", ""), ], ) def test_user_parameter_default_value(dtype, expected): p = local.UserParameter("a", "a desc", dtype) assert p.validate(None) == expected def test_user_parameter_repr(): p = local.UserParameter("a", "a desc", "str") expected = "" assert repr(p) == str(p) == expected @pytest.mark.parametrize( "dtype,given,expected", [ ("bool", "true", True), ("bool", 0, False), ( "datetime", datetime.datetime(2018, 1, 1, 0, 34, 0), pandas.Timestamp(2018, 1, 1, 0, 34, 0), ), ("datetime", "2018-01-01 12:34AM", pandas.Timestamp(2018, 1, 1, 0, 34, 0)), ("datetime", 1234567890000000000, pandas.Timestamp(2009, 2, 13, 23, 31, 30)), ("float", "3.14", 3.14), ("int", "1", 1), ("list", (3, 4), [3, 4]), ("str", 1, "1"), ("unicode", "foo", "foo"), ], ) def test_user_parameter_coerce_value(dtype, given, expected): p = local.UserParameter("a", "a desc", dtype, given) assert p.validate(given) == expected @pytest.mark.parametrize("given", ["now", "today"]) def test_user_parameter_coerce_special_datetime(given): p = local.UserParameter("a", "a desc", "datetime", given) assert type(p.validate(given)) == pandas.Timestamp @pytest.mark.parametrize( "dtype,given,expected", [ ("float", "100.0", 100.0), ("int", "20", 20), ("int", 20.0, 20), ], ) def test_user_parameter_coerce_min(dtype, given, expected): p = local.UserParameter("a", "a desc", dtype, expected, min=given) assert p.min == expected @pytest.mark.parametrize( "dtype,given,expected", [ ("float", "100.0", 100.0), ("int", "20", 20), ("int", 20.0, 20), ], ) def test_user_parameter_coerce_max(dtype, given, expected): p = local.UserParameter("a", "a desc", dtype, expected, max=given) assert p.max == expected @pytest.mark.parametrize( "dtype,given,expected", [ ("float", [50, "100.0", 150.0], [50.0, 100.0, 150.0]), ("int", [1, "2", 3.0], [1, 2, 3]), ], ) def test_user_parameter_coerce_allowed(dtype, given, expected): p = local.UserParameter("a", "a desc", dtype, expected[0], allowed=given) assert p.allowed == expected def test_user_parameter_validation_range(): p = local.UserParameter("a", "a desc", "int", 1, min=0, max=3) with pytest.raises(ValueError) as except_info: p.validate(-1) assert "less than" in str(except_info.value) assert p.validate(0) == 0 assert p.validate(1) == 1 assert p.validate(2) == 2 assert p.validate(3) == 3 with pytest.raises(ValueError) as except_info: p.validate(4) assert "greater than" in str(except_info.value) def test_user_parameter_validation_allowed(): p = local.UserParameter("a", "a desc", "int", 1, allowed=[1, 2]) with pytest.raises(ValueError) as except_info: p.validate(0) assert "allowed" in str(except_info.value) assert p.validate(1) == 1 assert p.validate(2) == 2 with pytest.raises(ValueError) as except_info: p.validate(3) assert "allowed" in str(except_info.value) def test_user_pars_list(): # first case: allowed are all lists, must choose exactly one of them # NB: order must match p = local.UserParameter("", "", "list", allowed=[[], ["one"], ["one", "two"]]) with pytest.raises(TypeError): p.validate(0) with pytest.raises((TypeError, ValueError)): # unfortunately, a string does coerce to a list p.validate("one") with pytest.raises(ValueError, match="allowed"): p.validate(["two"]) with pytest.raises(ValueError, match="allowed"): p.validate(["two", "one"]) p.validate(["one"]) p.validate(["one", "two"]) def test_user_pars_mlist(): # second case: allowed are not lists, can choose any number of them # NB: repeats are allowed p = local.UserParameter("", "", "mlist", allowed=["one", "two", "three"]) with pytest.raises(TypeError): p.validate(0) with pytest.raises((TypeError, ValueError)): # unfortunately, a string does coerce to a list p.validate("one") with pytest.raises(ValueError, match="allowed"): p.validate(["two", "other"]) p.validate(["two"]) p.validate(["two", "two"]) p.validate(["two", "one"]) p.validate([]) @pytest.mark.parametrize( "filename", [ "catalog_non_dict", "data_source_missing", "data_source_name_non_string", "data_source_non_dict", "data_source_value_non_dict", "params_missing_required", "params_name_non_string", "params_non_dict", "params_value_bad_choice", "params_value_bad_type", "params_value_non_dict", "plugins_non_dict", "plugins_source_missing", "plugins_source_missing_key", "plugins_source_non_dict", "plugins_source_non_list", ], ) def test_parser_validation_error(filename): with pytest.raises(exceptions.ValidationError): list(open_catalog(abspath(filename + ".yml"))) @pytest.mark.parametrize( "filename", [ "obsolete_data_source_list", "obsolete_params_list", ], ) def test_parser_obsolete_error(filename): with pytest.raises(exceptions.ObsoleteError): open_catalog(abspath(filename + ".yml")) def test_union_catalog(): path = os.path.dirname(__file__) uri1 = os.path.join(path, "catalog_union_1.yml") uri2 = os.path.join(path, "catalog_union_2.yml") union_cat = open_catalog([uri1, uri2]) assert_items_equal(list(union_cat), ["entry1", "entry1_part", "use_example1"]) expected = { "name": "entry1_part", "container": "dataframe", "user_parameters": [ { "name": "part", "description": "part of filename", "default": "1", "type": "str", "allowed": ["1", "2"], } ], "description": "entry1 part", "direct_access": "allow", } for k in expected: assert union_cat.entry1_part.describe()[k] == expected[k] # Implied creation of data source assert union_cat.entry1.container == "dataframe" md = union_cat.entry1.describe()["metadata"] assert md == dict(foo="bar", bar=[1, 2, 3]) # Use default parameters in explict creation of data source assert union_cat.entry1_part().container == "dataframe" # Specify parameters in creation of data source assert union_cat.entry1_part(part="2").container == "dataframe" def test_persist_local_cat(temp_cache): # when persisted, multiple cat become one from intake.catalog.local import YAMLFileCatalog path = os.path.dirname(__file__) uri1 = os.path.join(path, "catalog_union_1.yml") uri2 = os.path.join(path, "catalog_union_2.yml") s = open_catalog([uri1, uri2]) s2 = s.persist() assert isinstance(s2, YAMLFileCatalog) assert set(s) == set(s2) def test_empty_catalog(): cat = open_catalog() assert list(cat) == [] def test_nonexistent_error(): with pytest.raises(IOError): local.YAMLFileCatalog("nonexistent") def test_duplicate_data_sources(): path = os.path.dirname(__file__) uri = os.path.join(path, "catalog_dup_sources.yml") with pytest.raises(exceptions.DuplicateKeyError): open_catalog(uri) def test_duplicate_parameters(): path = os.path.dirname(__file__) uri = os.path.join(path, "catalog_dup_parameters.yml") with pytest.raises(exceptions.DuplicateKeyError): open_catalog(uri) @pytest.fixture def temp_catalog_file(): path = tempfile.mkdtemp() catalog_file = os.path.join(path, "catalog.yaml") with open(catalog_file, "w") as f: f.write( """ sources: a: driver: csv args: urlpath: /not/a/file b: driver: csv args: urlpath: /not/a/file """ ) yield catalog_file shutil.rmtree(path) def test_catalog_file_removal(temp_catalog_file): cat_dir = os.path.dirname(temp_catalog_file) cat = open_catalog(cat_dir + "/*", ttl=0.1) assert set(cat) == {"a", "b"} os.remove(temp_catalog_file) time.sleep(0.5) # wait for catalog refresh assert set(cat) == set() def test_flatten_duplicate_error(): path = tempfile.mkdtemp() f1 = os.path.join(path, "catalog.yaml") path = tempfile.mkdtemp() f2 = os.path.join(path, "catalog.yaml") for f in [f1, f2]: with open(f, "w") as fo: fo.write( """ sources: a: driver: csv args: urlpath: /not/a/file """ ) with pytest.raises(ValueError): open_catalog([f1, f2]) def test_multi_cat_names(): fn = abspath("catalog_union*.yml") cat = open_catalog(fn) assert cat.name == fn assert fn in repr(cat) fn1 = abspath("catalog_union_1.yml") fn2 = abspath("catalog_union_2.yml") cat = open_catalog([fn1, fn2]) assert cat.name == "2 files" assert cat.description == "Catalog generated from 2 files" cat = open_catalog([fn1, fn2], name="special_name", description="Special description") assert cat.name == "special_name" assert cat.description == "Special description" def test_name_of_builtin(): import intake assert intake.cat.name == "builtin" assert intake.cat.description == "Generated from data packages found on your intake search path" def test_cat_with_declared_name(): fn = abspath("catalog_named.yml") description = "Description declared in the open function" cat = open_catalog(fn, name="name_in_func", description=description) assert cat.name == "name_in_func" assert cat.description == description cat._load() # we don't get metadata until load/list/getitem assert cat.metadata.get("some") == "thing" cat = open_catalog(fn) assert cat.name == "name_in_spec" assert cat.description == "This is a catalog with a description in the yaml" def test_cat_with_no_declared_name_gets_name_from_dir_if_file_named_catalog(): fn = abspath("catalog.yml") cat = open_catalog(fn, name="name_in_func", description="Description in func") assert cat.name == "name_in_func" assert cat.description == "Description in func" cat = open_catalog(fn) assert cat.name == "tests" assert cat.description is None def test_default_expansions(): try: os.environ["INTAKE_INT_TEST"] = "1" par = UserParameter("", "", "int", default="env(INTAKE_INT_TEST)") par.expand_defaults() assert par.expanded_default == 1 finally: del os.environ["INTAKE_INT_TEST"] par = UserParameter("", "", "str", default="env(USER)") par.expand_defaults(getenv=False) assert par.expanded_default == "env(USER)" par.expand_defaults() assert par.expanded_default == os.getenv("USER", "") par = UserParameter("", "", "str", default="client_env(USER)") par.expand_defaults() assert par.expanded_default == "client_env(USER)" par.expand_defaults(client=True) assert par.expanded_default == os.getenv("USER", "") par = UserParameter("", "", "str", default="shell(echo success)") par.expand_defaults(getshell=False) assert par.expanded_default == "shell(echo success)" par.expand_defaults() assert par.expanded_default == "success" par = UserParameter("", "", "str", default="client_shell(echo success)") par.expand_defaults(client=True) assert par.expanded_default == "success" par = UserParameter("", "", "int", default=1) par.expand_defaults() # no error from string ops def test_remote_cat(http_server): url = http_server + "catalog1.yml" cat = open_catalog(url) assert "entry1" in cat assert cat.entry1.describe() def test_multi_plugins(): from intake.source.csv import CSVSource fn = abspath("multi_plugins.yaml") cat = open_catalog(fn) s = cat.tables0() assert isinstance(s, CSVSource) s = cat.tables1() assert isinstance(s, CSVSource) s = cat.tables2() assert isinstance(s, CSVSource) s = cat.tables3() assert isinstance(s, CSVSource) assert s._csv_kwargs == {} s = cat.tables3(plugin="myplug") assert isinstance(s, CSVSource) assert s._csv_kwargs == {} s = cat.tables3(plugin="myplug2") assert isinstance(s, CSVSource) assert s._csv_kwargs is True with pytest.raises(ValueError): cat.tables4() with pytest.raises(ValueError): cat.tables4(plugin="myplug") with pytest.raises(ValueError): cat.tables4(plugin="myplug2") s = cat.tables5() assert isinstance(s, CSVSource) with pytest.raises(ValueError): cat.tables5(plugin="myplug") fn = abspath("multi_plugins2.yaml") with pytest.raises(ValueError): open_catalog(fn) def test_no_plugins(): fn = abspath("multi_plugins.yaml") cat = open_catalog(fn) with pytest.raises(ValueError) as e: cat.tables6 assert "doesnotexist" in str(e.value) assert "plugin-directory" in str(e.value) with pytest.raises(ValueError) as e: cat.tables7 assert "doesnotexist" in str(e.value) def test_explicit_entry_driver(): from intake.source.textfiles import TextFilesSource e = LocalCatalogEntry("test", "desc", TextFilesSource, args={"urlpath": None}) assert e.describe()["container"] == "python" assert isinstance(e(), TextFilesSource) with pytest.raises(TypeError): LocalCatalogEntry("test", "desc", None) def test_getitem_and_getattr(): fn = abspath("multi_plugins.yaml") catalog = open_catalog(fn) catalog["tables0"] with pytest.raises(KeyError): catalog["doesnotexist"] with pytest.raises(KeyError): catalog["_doesnotexist"] with pytest.raises(KeyError): # This exists as an *attribute* but not as an item. catalog["metadata"] catalog.tables0 # alias to catalog['tables0'] catalog.metadata # a normal attribute with pytest.raises(AttributeError): catalog.doesnotexit with pytest.raises(AttributeError): catalog._doesnotexit assert catalog.tables0 == catalog["tables0"] assert isinstance(catalog.metadata, (dict, type(None))) def test_dot_names(): fn = abspath("dot-nest.yaml") cat = open_catalog(fn) assert cat.self.leaf.description == "leaf" assert cat.self["leafdot.dot"].description == "leaf-dot" assert cat["selfdot.dot", "leafdot.dot"].description == "leaf-dot" assert cat["self.selfdot.dot", "leafdot.dot"].description == "leaf-dot" assert cat["self.self.dot", "leafdot.dot"].description == "leaf-dot" assert cat["self.self.dot", "leaf"].description == "leaf" assert cat["self.self.dot", "leaf.dot"].description == "leaf-dot" assert cat["self.self.dot.leaf.dot"].description == "leaf-dot" def test_listing(catalog1): assert list(catalog1) == list(catalog1.nested) with pytest.raises(TypeError): list(catalog1.arr) def test_dict_save(): from intake.catalog.base import Catalog fn = os.path.join(tempfile.mkdtemp(), "mycat.yaml") entry = LocalCatalogEntry( name="trial", description="get this back", driver="csv", args=dict(urlpath="") ) cat = Catalog.from_dict({"trial": entry}, name="mycat") cat.save(fn) cat2 = open_catalog(fn) assert "trial" in cat2 assert cat2.name == "mycat" assert "CSV" in cat2.trial.classname def test_dict_save_complex(): from intake.catalog.base import Catalog fn = os.path.join(tempfile.mkdtemp(), "mycat.yaml") cat = Catalog() entry = LocalCatalogEntry( name="trial", description="get this back", driver="csv", cache=[], catalog=cat, parameters=[UserParameter(name="par1", description="desc", type="int")], args={"urlpath": "none"}, ) cat._entries = {"trial": entry} cat.save(fn) cat2 = open_catalog(fn) assert "trial" in cat2 assert cat2.name == "mycat" assert cat2.trial.describe()["plugin"][0] == "csv" def test_dict_adddel(): from intake.catalog.base import Catalog entry = LocalCatalogEntry( name="trial", description="get this back", driver="csv", args=dict(urlpath="") ) cat = Catalog.from_dict({"trial": entry}, name="mycat") assert "trial" in cat cat["trial2"] = entry assert list(cat) == ["trial", "trial2"] cat.pop("trial") assert list(cat) == ["trial2"] assert cat["trial2"].describe() == entry.describe() def test_filter(): from intake.catalog.base import Catalog entry1 = LocalCatalogEntry( name="trial", description="get this back", driver="csv", args=dict(urlpath="") ) entry2 = LocalCatalogEntry( name="trial", description="pass this through", driver="csv", args=dict(urlpath=""), ) cat = Catalog.from_dict({"trial1": entry1, "trial2": entry2}, name="mycat") cat2 = cat.filter(lambda e: "pass" in e._description) assert list(cat2) == ["trial2"] assert cat2.trial2 == entry2() def test_from_dict_with_data_source(): "Check that Catalog.from_dict accepts DataSources not wrapped in Entry." from intake.catalog.base import Catalog entry = LocalCatalogEntry( name="trial", description="get this back", driver="csv", args=dict(urlpath="") ) ds = entry() Catalog.from_dict({"trial": ds}, name="mycat") def test_no_instance(): from intake.catalog.local import LocalCatalogEntry e0 = LocalCatalogEntry("foo", "", "fake") e1 = LocalCatalogEntry("foo0", "", "fake") # this would error on instantiation with driver not found assert e0 != e1 def test_fsspec_integration(): import fsspec import pandas as pd mem = fsspec.filesystem("memory") with mem.open("cat.yaml", "wt") as f: f.write( """ sources: implicit: driver: csv description: o args: urlpath: "{{CATALOG_DIR}}/file.csv" explicit: driver: csv description: o args: urlpath: "memory://file.csv" extra: driver: csv description: o args: urlpath: "{{CATALOG_DIR}}/file.csv" storage_options: {other: option}""" ) with mem.open("/file.csv", "wt") as f: f.write("a,b\n0,1") expected = pd.DataFrame({"a": [0], "b": [1]}) cat = open_catalog("memory://cat.yaml") assert list(cat) == ["implicit", "explicit", "extra"] assert cat.implicit.read().equals(expected) assert cat.explicit.read().equals(expected) s = cat.extra() assert s._storage_options["other"] def test_cat_add(tmpdir): tmpdir = str(tmpdir) fn = os.path.join(tmpdir, "cat.yaml") with open(fn, "w") as f: f.write("sources: {}") cat = open_catalog(fn) assert list(cat) == [] # was added in memory cat.add(cat) cat._load() # this would happen automatically, but not immediately assert list(cat) == ["cat"] # was added to the file cat = open_catalog(fn) assert list(cat) == ["cat"] def test_no_entries_items(catalog1): from intake.catalog.entry import CatalogEntry from intake.source.base import DataSource for k, v in catalog1.items(): assert not isinstance(v, CatalogEntry) assert isinstance(v, DataSource) for k in catalog1: v = catalog1[k] assert not isinstance(v, CatalogEntry) assert isinstance(v, DataSource) for k in catalog1: # we can't do attribute access on "text" because it # collides with a property if k == "text": continue v = getattr(catalog1, k) assert not isinstance(v, CatalogEntry) assert isinstance(v, DataSource) def test_cat_dictlike(catalog1): assert list(catalog1) == list(catalog1.keys()) assert len(list(catalog1)) == len(catalog1) assert list(catalog1.items()) == list(zip(catalog1.keys(), catalog1.values())) def test_inherit_params(inherit_params_cat): assert inherit_params_cat.param._urlpath == "s3://test_bucket/file.parquet" def test_runtime_overwrite_params(inherit_params_cat): assert ( inherit_params_cat.param(bucket="runtime_overwrite")._urlpath == "s3://runtime_overwrite/file.parquet" ) def test_local_param_overwrites(inherit_params_cat): assert inherit_params_cat.local_param_overwrites._urlpath == "s3://local_param/file.parquet" def test_local_and_global_params(inherit_params_cat): assert ( inherit_params_cat.local_and_global_params._urlpath == "s3://test_bucket/local_filename.parquet" ) def test_search_inherit_params(inherit_params_cat): assert ( inherit_params_cat.search("local_and_global").local_and_global_params._urlpath == "s3://test_bucket/local_filename.parquet" ) def test_multiple_cats_params(inherit_params_multiple_cats): assert ( inherit_params_multiple_cats.local_and_global_params._urlpath == "s3://test_bucket/local_filename.parquet" ) ================================================ FILE: intake/catalog/tests/test_parameters.py ================================================ import os import pytest import intake from intake.catalog.local import LocalCatalogEntry, UserParameter from intake.source.base import DataSource class NoSource(DataSource): def __init__(self, **kwargs): self.metadata = kwargs.pop("metadata", {}) self.kwargs = kwargs driver = "intake.catalog.tests.test_parameters.NoSource" def test_simplest(): e = LocalCatalogEntry("", "", driver, args={"arg1": 1}) s = e() assert s.kwargs["arg1"] == 1 def test_cache_default_source(): # If the user provides parameters, don't allow default caching up = UserParameter("name", default="oi") e = LocalCatalogEntry("", "", driver, getenv=False, parameters=[up]) s1 = e(name="oioi") s2 = e() assert s1 is not s2 s1 = e() s2 = e(name="oioi") assert s1 is not s2 # Otherwise, we can cache the default source e = LocalCatalogEntry("", "", driver, getenv=False) s1 = e() s2 = e() assert s1 is s2 def test_parameter_default(): up = UserParameter("name", default="oi") e = LocalCatalogEntry("", "", driver, args={"arg1": "{{name}}"}, parameters=[up]) s = e() assert s.kwargs["arg1"] == "oi" def test_maybe_default_from_env(): # maybe fill in parameter default from the env, depending on getenv up = UserParameter("name", default="env(INTAKE_TEST_VAR)") e = LocalCatalogEntry("", "", driver, args={"arg1": "{{name}}"}, parameters=[up], getenv=False) s = e() assert s.kwargs["arg1"] == "env(INTAKE_TEST_VAR)" os.environ["INTAKE_TEST_VAR"] = "oi" # Clear the cached source so we can (not) pick up the changed environment variable. e.clear_cached_default_source() s = e() assert s.kwargs["arg1"] == "env(INTAKE_TEST_VAR)" up = UserParameter("name", default="env(INTAKE_TEST_VAR)") e = LocalCatalogEntry("", "", driver, args={"arg1": "{{name}}"}, parameters=[up], getenv=True) s = e() assert s.kwargs["arg1"] == "oi" del os.environ["INTAKE_TEST_VAR"] # Clear the cached source so we can pick up the changed environment variable. e.clear_cached_default_source() s = e() assert s.kwargs["arg1"] == "" def test_up_override_and_render(): up = UserParameter("name", default="env(INTAKE_TEST_VAR)") e = LocalCatalogEntry("", "", driver, args={"arg1": "{{name}}"}, parameters=[up], getenv=False) s = e(name="other") assert s.kwargs["arg1"] == "other" def test_user_explicit_override(): up = UserParameter("name", default="env(INTAKE_TEST_VAR)") e = LocalCatalogEntry("", "", driver, args={"arg1": "{{name}}"}, parameters=[up], getenv=False) # override wins over up s = e(arg1="other") assert s.kwargs["arg1"] == "other" def test_auto_env_expansion(): os.environ["INTAKE_TEST_VAR"] = "oi" e = LocalCatalogEntry( "", "", driver, args={"arg1": "{{env(INTAKE_TEST_VAR)}}"}, parameters=[], getenv=False, ) s = e() # when getenv is False, you pass through the text assert s.kwargs["arg1"] == "{{env(INTAKE_TEST_VAR)}}" e = LocalCatalogEntry( "", "", driver, args={"arg1": "{{env(INTAKE_TEST_VAR)}}"}, parameters=[], getenv=True, ) s = e() assert s.kwargs["arg1"] == "oi" # same, but with quoted environment name e = LocalCatalogEntry( "", "", driver, args={"arg1": '{{env("INTAKE_TEST_VAR")}}'}, parameters=[], getenv=True, ) s = e() assert s.kwargs["arg1"] == "oi" del os.environ["INTAKE_TEST_VAR"] # Clear the cached source so we can pick up the changed environment variable. e.clear_cached_default_source() s = e() assert s.kwargs["arg1"] == "" e = LocalCatalogEntry( "", "", driver, args={"arg1": "{{env(INTAKE_TEST_VAR)}}"}, parameters=[], getenv=False, ) s = e() assert s.kwargs["arg1"] == "{{env(INTAKE_TEST_VAR)}}" def test_validate_up(): up = UserParameter("name", default=1, type="int") e = LocalCatalogEntry("", "", driver, args={"arg1": "{{name}}"}, parameters=[up], getenv=False) s = e() # OK assert s.kwargs["arg1"] == "1" with pytest.raises(ValueError): e(name="oi") up = UserParameter("name", type="int") e = LocalCatalogEntry("", "", driver, args={"arg1": "{{name}}"}, parameters=[up], getenv=False) s = e() # OK # arg1 is a string: real int gets rendered by jinja assert s.kwargs["arg1"] == "0" # default default for int s = e(arg1="something") assert s.kwargs["arg1"] == "something" def test_validate_par(): up = UserParameter("arg1", type="int") e = LocalCatalogEntry("", "", driver, args={"arg1": "oi"}, parameters=[up], getenv=False) with pytest.raises(ValueError): e() e = LocalCatalogEntry("", "", driver, args={"arg1": 1}, parameters=[up], getenv=False) e() # OK e = LocalCatalogEntry("", "", driver, args={"arg1": "1"}, parameters=[up], getenv=False) s = e() # OK assert s.kwargs["arg1"] == 1 # a number, not str def test_mlist_parameter(): up = UserParameter("", "", "mlist", allowed=["a", "b"]) up.validate([]) up.validate(["b"]) up.validate(["b", "a"]) with pytest.raises(ValueError): up.validate(["c"]) with pytest.raises(ValueError): up.validate(["a", "c"]) with pytest.raises(ValueError): up.validate("hello") def test_explicit_overrides(): e = LocalCatalogEntry("", "", driver, args={"arg1": "oi"}) s = e(arg1="hi") assert s.kwargs["arg1"] == "hi" e = LocalCatalogEntry("", "", driver, args={"arg1": "{{name}}"}) s = e(name="hi") assert s.kwargs["arg1"] == "hi" os.environ["INTAKE_TEST_VAR"] = "another" e = LocalCatalogEntry("", "", driver, args={"arg1": "oi"}, getenv=True) s = e(arg1="{{env(INTAKE_TEST_VAR)}}") assert s.kwargs["arg1"] == "another" up = UserParameter("arg1", type="int") e = LocalCatalogEntry("", "", driver, args={"arg1": 1}, parameters=[up]) with pytest.raises(ValueError): e(arg1="oi") s = e(arg1="1") assert s.kwargs["arg1"] == 1 def test_extra_arg(): e = LocalCatalogEntry("", "", driver, args={"arg1": "oi"}) s = e(arg2="extra") assert s.kwargs["arg2"] == "extra" def test_unknown(): e = LocalCatalogEntry("", "", driver, args={"arg1": "{{name}}"}) s = e() assert s.kwargs["arg1"] == "" # parameter has no default up = UserParameter("name") e = LocalCatalogEntry("", "", driver, args={"arg1": "{{name}}"}, parameters=[up]) s = e() assert s.kwargs["arg1"] == "" def test_catalog_passthrough(): root = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) cat = intake.open_catalog(os.path.join(root, "tests/catalog_inherit_params.yml")) assert set(cat.subcat.user_parameters) == {"bucket", "inner"} url = cat.subcat.ex2.urlpath assert url == "test_bucket/test_name" url = cat.subcat.ex2(bucket="hi", inner="ho").urlpath assert url == "hi/ho" # test clone cat2 = cat(bucket="yet") url = cat2.subcat(inner="another").ex2.urlpath assert url == "yet/another" url = cat2.subcat(inner="another").ex2(bucket="hi").urlpath assert url == "hi/another" ================================================ FILE: intake/catalog/tests/test_reload_integration.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import os import os.path import shutil import tempfile import time import pytest from intake import open_catalog from .util import assert_items_equal TMP_DIR = tempfile.mkdtemp() TEST_CATALOG_PATH = [TMP_DIR] YAML_FILENAME = "intake_test_catalog.yml" MISSING_PATH = os.path.join(TMP_DIR, "a") def teardown_module(module): try: shutil.rmtree(TMP_DIR) except: pass @pytest.fixture def intake_server_with_config(intake_server): fullname = os.path.join(TMP_DIR, YAML_FILENAME) with open(fullname, "w") as f: f.write( """ plugins: source: - module: intake.catalog.tests.example1_source - module: intake.catalog.tests.example_plugin_dir.example2_source sources: use_example1: description: example1 source plugin driver: example1 args: {} """ ) time.sleep(1) yield intake_server os.remove(fullname) def test_reload_updated_config(intake_server_with_config): catalog = open_catalog(intake_server_with_config, ttl=0.1) entries = list(catalog) assert entries == ["use_example1"] with open(os.path.join(TMP_DIR, YAML_FILENAME), "a") as f: f.write( """ use_example1_1: description: example1 other driver: example1 args: {} """ ) time.sleep(1.2) assert_items_equal(list(catalog), ["use_example1", "use_example1_1"]) def test_reload_updated_directory(intake_server_with_config): catalog = open_catalog(intake_server_with_config, ttl=0.1) orig_entries = list(catalog) assert "example2" not in orig_entries filename = os.path.join(TMP_DIR, "intake_test_catalog2.yml") with open(filename, "w") as f: f.write( """ sources: example2: description: source 2 driver: csv args: urlpath: none """ ) time.sleep(1.2) assert_items_equal(list(catalog), ["example2"] + orig_entries) def test_reload_missing_remote_directory(intake_server): try: shutil.rmtree(TMP_DIR) except: pass time.sleep(1) catalog = open_catalog(intake_server, ttl=0.1) assert_items_equal(list(catalog), []) os.mkdir(TMP_DIR) with open(os.path.join(TMP_DIR, YAML_FILENAME), "w") as f: f.write( """ plugins: source: - module: intake.catalog.tests.example1_source - module: intake.catalog.tests.example_plugin_dir.example2_source sources: use_example1: description: example1 source plugin driver: example1 args: {} """ ) time.sleep(1.2) assert_items_equal(list(catalog), ["use_example1"]) try: shutil.rmtree(TMP_DIR) except: pass def test_reload_missing_local_directory(tempdir): catalog = open_catalog(tempdir + "/*", ttl=0.1) assert_items_equal(list(catalog), []) with open(os.path.join(tempdir, YAML_FILENAME), "w") as f: f.write( """ plugins: source: - module: intake.catalog.tests.example1_source - module: intake.catalog.tests.example_plugin_dir.example2_source sources: use_example1: description: example1 source plugin driver: example1 args: {} """ ) time.sleep(1.2) assert "use_example1" in catalog ================================================ FILE: intake/catalog/tests/test_utils.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import pandas as pd import pytest import intake.catalog.utils as utils def test_expand_templates(): pars = {"a": "{{par}} hi"} context = {"b": 1, "par": "ho"} assert utils.expand_templates(pars, context)["a"] == "ho hi" assert utils.expand_templates(pars, context, True)[1] == {"b"} def test_expand_nested_template(): pars = {"a": ["{{par}} hi"]} context = {"b": 1, "par": "ho"} assert utils.expand_templates(pars, context)["a"] == ["ho hi"] assert utils.expand_templates(pars, context, True)[1] == {"b"} pars = {"a": {"k": {("{{par}} hi",)}}} context = {"b": 1, "par": "ho"} assert utils.expand_templates(pars, context)["a"] == {"k": {("ho hi",)}} assert utils.expand_templates(pars, context, True)[1] == {"b"} @pytest.mark.parametrize( "test_input,expected", [ (None, pd.Timestamp("1970-01-01 00:00:00")), (1, pd.Timestamp("1970-01-01 00:00:00.000000001")), ("1988-02-24T13:37+0100", pd.Timestamp("1988-02-24 13:37+0100")), ( {"__datetime__": True, "as_str": "1988-02-24T13:37+0100"}, pd.Timestamp("1988-02-24T13:37+0100"), ), ], ) def test_coerce_datetime(test_input, expected): assert utils.coerce_datetime(test_input) == expected def test_flatten(): assert list(utils.flatten([["hi"], ["oi"]])) == ["hi", "oi"] @pytest.mark.parametrize( "value,dtype,expected", [ (1, "int", 1), ("1", "int", 1), (1, "str", "1"), ((), "list", []), ((1,), "list", [1]), ((1,), "list[str]", ["1"]), ("{'a': 1}", "dict", {"a": 1}), ("{'a': '1'}", "dict", {"a": "1"}), ("[1, 2]", "list", [1, 2]), ("['1', '2']", "list", ["1", "2"]), ("[{'a': '1'}]", "list", [{"a": "1"}]), ("{'a': [1, 2]}", "dict", {"a": [1, 2]}), ("{'a': ['1', '2']}", "dict", {"a": ["1", "2"]}), ("1988-02-24T13:37+0100", "datetime", pd.Timestamp("1988-02-24 13:37+0100")), ], ) def test_coerce(value, dtype, expected): out = utils.coerce(dtype, value) assert out == expected assert type(out) == type(expected) ================================================ FILE: intake/catalog/tests/test_zarr.py ================================================ import os import shutil import tempfile import pytest from intake import open_catalog from intake.catalog.zarr import ZarrGroupCatalog from intake.source.zarr import ZarrArraySource from .util import assert_items_equal zarr = pytest.importorskip("zarr") @pytest.fixture def temp_zarr(): zarr_path = tempfile.mkdtemp() # setup a zarr hierarchy stored on local file system store = zarr.DirectoryStore(zarr_path) root = zarr.open_group(store=store, mode="w") root.attrs["title"] = "root group" root.attrs["description"] = "a test hierarchy" root.create_group("foo") root["foo"].attrs["title"] = "foo group" root["foo"].attrs["description"] = "a test group" root.create_group("bar") root.create_dataset("baz", shape=100, dtype="i4") root["baz"].attrs["title"] = "baz array" root["baz"].attrs["description"] = "a test array" root["bar"].create_group("spam") root["bar"].create_dataset("eggs", shape=10, dtype="f4") zarr.consolidate_metadata(store=store) # setup a local YAML file catalog including entries pointing to the zarr # hierarchy yaml_path = tempfile.mkdtemp() catalog_file = os.path.join(yaml_path, "catalog.yml") with open(catalog_file, "w") as f: f.write( """ sources: root: driver: zarr_cat args: urlpath: {zarr_path} consolidated: True bar: driver: zarr_cat args: urlpath: {zarr_path} component: bar eggs: driver: ndzarr args: urlpath: {zarr_path} component: bar/eggs """.format( zarr_path=zarr_path ) ) yield zarr_path, store, root, catalog_file shutil.rmtree(zarr_path) shutil.rmtree(yaml_path) @pytest.mark.parametrize("consolidated", [False, True]) def test_zarr_catalog(temp_zarr, consolidated): import dask.array as da path, store, root, _ = temp_zarr # test zarr catalog opened directly, with different urlpath argument types for urlpath in path, store, root: # open catalog cat = ZarrGroupCatalog(urlpath=urlpath, consolidated=consolidated) assert isinstance(cat, ZarrGroupCatalog) assert "catalog" == cat.container # check entries assert_items_equal(["foo", "bar", "baz"], list(cat)) assert isinstance(cat["foo"], ZarrGroupCatalog) assert "catalog" == cat["foo"].describe()["container"] assert isinstance(cat["bar"], ZarrGroupCatalog) assert "catalog" == cat["bar"].describe()["container"] assert isinstance(cat["baz"], ZarrArraySource) assert "ndarray" == cat["baz"].describe()["container"] # check metadata from attributes assert "root group" == cat.metadata["title"] assert "a test hierarchy" == cat.metadata["description"] assert "foo group" == cat["foo"].metadata["title"] assert "a test group" == cat["foo"].metadata["description"] # no attributes assert "title" not in cat["bar"].metadata assert "description" not in cat["bar"].metadata # check nested catalogs assert_items_equal(["spam", "eggs"], list(cat["bar"])) assert isinstance(cat["bar"]["spam"], ZarrGroupCatalog) assert "catalog" == cat["bar"]["spam"].describe()["container"] assert isinstance(cat["bar"]["eggs"], ZarrArraySource) assert "ndarray" == cat["bar"]["eggs"].describe()["container"] # check obtain zarr groups assert isinstance(cat.to_zarr(), zarr.hierarchy.Group) assert isinstance(cat["foo"].to_zarr(), zarr.hierarchy.Group) assert isinstance(cat["bar"].to_zarr(), zarr.hierarchy.Group) assert isinstance(cat["bar"]["spam"].to_zarr(), zarr.hierarchy.Group) # check obtain dask arrays assert isinstance(cat["baz"].to_dask(), da.Array) assert isinstance(cat["bar"]["eggs"].to_dask(), da.Array) # open catalog directly from subgroup via `component` arg cat = ZarrGroupCatalog(urlpath, component="bar") assert_items_equal(["spam", "eggs"], list(cat)) assert isinstance(cat["spam"], ZarrGroupCatalog) assert "catalog" == cat["spam"].describe()["container"] assert isinstance(cat["eggs"], ZarrArraySource) assert "ndarray" == cat["eggs"].describe()["container"] def test_zarr_entries_in_yaml_catalog(temp_zarr): import dask.array as da # open YAML catalog file _, _, _, catalog_file = temp_zarr cat = open_catalog(catalog_file) # test entries assert_items_equal(["root", "bar", "eggs"], list(cat)) # entry pointing to zarr root group assert isinstance(cat["root"], ZarrGroupCatalog) assert_items_equal(["foo", "bar", "baz"], list(cat["root"])) assert "catalog" == cat["root"].describe()["container"] assert isinstance(cat["root"].to_zarr(), zarr.hierarchy.Group) # entry pointing to zarr sub-group assert isinstance(cat["bar"], ZarrGroupCatalog) assert_items_equal(["spam", "eggs"], list(cat["bar"])) assert "catalog" == cat["bar"].describe()["container"] assert isinstance(cat["bar"].to_zarr(), zarr.hierarchy.Group) # entry pointing to zarr array assert isinstance(cat["eggs"], ZarrArraySource) assert "ndarray" == cat["eggs"].describe()["container"] assert isinstance(cat["eggs"].to_dask(), da.Array) ================================================ FILE: intake/catalog/tests/util.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from intake.source import base, registry def assert_items_equal(a, b): assert len(a) == len(b) and sorted(a) == sorted(b) class TestingSource(base.DataSource): """A source that gives back whatever parameters were passed to it""" name = "test" version = "0.0.1" container = "python" partition_access = False def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs super(TestingSource, self).__init__("python") self.npartitions = 1 def _load_metadata(self): pass def _get_partition(self, _): return self.args, self.kwargs def register(): registry["test"] = TestingSource ================================================ FILE: intake/catalog/utils.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import ast import functools import itertools import os import re import shlex import subprocess import sys import warnings def flatten(iterable): """Flatten an arbitrarily deep list""" # likely not used iterable = iter(iterable) while True: try: item = next(iterable) except StopIteration: break if isinstance(item, str): yield item continue try: data = iter(item) iterable = itertools.chain(data, iterable) except Exception: yield item def reload_on_change(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): self.reload() return f(self, *args, **kwargs) return wrapper def clamp(value, lower=0, upper=sys.maxsize): """Clamp float between given range""" return max(lower, min(upper, value)) def _j_getenv(x, default=""): from jinja2 import Undefined if isinstance(x, Undefined): x = x._undefined_name if isinstance(default, Undefined): default = default._undefined_name return os.getenv(x, default) def _j_getshell(x): from jinja2 import Undefined if isinstance(x, Undefined): x = x._undefined_name try: return subprocess.check_output(x).decode() except (IOError, OSError): return "" def _j_passthrough(x, funcname): from jinja2 import Undefined if isinstance(x, Undefined): x = x._undefined_name return "{{%s(%s)}}" % (funcname, x) def _expand(p, context, all_vars, client, getenv, getshell): from jinja2 import Environment, meta if isinstance(p, dict): return {k: _expand(v, context, all_vars, client, getenv, getshell) for k, v in p.items()} elif isinstance(p, (list, tuple, set)): return type(p)(_expand(v, context, all_vars, client, getenv, getshell) for v in p) elif isinstance(p, str): jinja = Environment() if getenv and not client: jinja.globals["env"] = _j_getenv else: jinja.globals["env"] = lambda x: _j_passthrough(x, funcname="env") if getenv and client: jinja.globals["client_env"] = _j_getenv else: jinja.globals["client_env"] = lambda x: _j_passthrough(x, funcname="client_env") if getshell and not client: jinja.globals["shell"] = _j_getshell else: jinja.globals["shell"] = lambda x: _j_passthrough(x, funcname="shell") if getshell and client: jinja.globals["client_shell"] = _j_getshell else: jinja.globals["client_shell"] = lambda x: _j_passthrough(x, funcname="client_shell") ast = jinja.parse(p) all_vars -= meta.find_undeclared_variables(ast) return jinja.from_string(p).render(context) else: # no expansion return p def expand_templates(pars, context, return_left=False, client=False, getenv=True, getshell=False): """ Render variables in context into the set of parameters with jinja2. For variables that are not strings, nothing happens. Parameters ---------- pars: dict values are strings containing some jinja2 controls context: dict values to use while rendering return_left: bool whether to return the set of variables in context that were not used in rendering parameters Returns ------- dict with the same keys as ``pars``, but updated values; optionally also return set of unused parameter names. """ all_vars = set(context) out = _expand(pars, context, all_vars, client, getenv, getshell) if return_left: return out, all_vars return out def expand_defaults(default, client=False, getenv=True, getshell=False): """Compile env, client_env, shell and client_shell commands Execution rules: - env() and shell() execute where the cat is loaded, if getenv and getshell are True, respectively - client_env() and client_shell() execute only if client is True and getenv/getshell are also True. If both getenv and getshell are False, this method does nothing. If the environment variable is missing or the shell command fails, the output is an empty string. """ r = re.match(r"env\((.*),?(.*)\)", default) if r and not client and getenv: gs = r.groups() default = os.environ.get(gs[0], gs[1] if len(gs) > 1 else "") r = re.match(r"client_env\((.*)\)", default) if r and client and getenv: default = os.environ.get(r.groups()[0], "") r = re.match(r"shell\((.*)\)", default) if r and not client and getshell: try: cmd = shlex.split(r.groups()[0]) default = subprocess.check_output(cmd).rstrip().decode("utf8") except (subprocess.CalledProcessError, OSError): default = "" else: warnings.warn("Shell command not executed due to getshell=False") r = re.match(r"client_shell\((.*)\)", default) if r and client and getshell: try: cmd = shlex.split(r.groups()[0]) default = subprocess.check_output(cmd).rstrip().decode("utf8") except (subprocess.CalledProcessError, OSError): default = "" else: warnings.warn("Shell command not executed due to getshell=False") return default def merge_pars(params, user_inputs, spec_pars, client=False, getenv=True, getshell=False): """Produce open arguments by merging various inputs This function is called in the context of a catalog entry, when finalising the arguments for instantiating the corresponding data source. The three sets of inputs to be considered are: - the arguments section of the original spec (params) - UserParameters associated with the entry (spec_pars) - explicit arguments provided at instantiation time, like entry(arg=value) (user_inputs) Both spec_pars and user_inputs can be considered as template variables and used in expanding string values in params. The default value of a spec_par, if given, may have embedded env and shell functions, which will be evaluated before use, if the default is used and the corresponding getenv/getsgell are set. Similarly, string value params will also have access to these functions within jinja template groups, as well as full jinja processing. Where a key exists in both the spec_pars and the user_inputs, the user_input wins. Where user_inputs contains keys not seen elsewhere, they are regarded as extra kwargs to pass to the data source. Where spec pars have the same name as keys in params, their type, max/min and allowed fields are used to validate the final values of the corresponding arguments. Parameters ---------- params : dict From the entry's original spec user_inputs : dict Provided by the user/calling function spec_pars : list of UserParameters Default and validation instances client : bool Whether this is all running on a client to a remote server - sets which of the env/shell functions are in operation. getenv : bool Whether to allow pulling environment variables. If False, the template blocks will pass through unevaluated getshell : bool Whether or not to allow executing of shell commands. If False, the template blocks will pass through unevaluated Returns ------- Final parameter dict """ context = params.copy() for par in spec_pars: val = user_inputs.get(par.name, par.default) if val is not None: if isinstance(val, str): val = expand_defaults(val, getenv=getenv, getshell=getshell, client=client) context[par.name] = par.validate(val) context.update({k: v for k, v in user_inputs.items() if k not in context}) out, left = expand_templates(params, context, True, client, getenv, getshell) context = {k: v for k, v in context.items() if k in left} for par in spec_pars: if par.name in context: # coerces to type context[par.name] = par.validate(context[par.name]) left.remove(par.name) params.update(out) user_inputs = expand_templates(user_inputs, context, False, client, getenv, getshell) params.update({k: v for k, v in user_inputs.items() if k in left}) params.pop("CATALOG_DIR", None) for k, v in params.copy().items(): # final validation/coersion for sp in [p for p in spec_pars if p.name == k]: params[k] = sp.validate(params[k]) return params def coerce_datetime(v=None): import pandas if not v: v = 0 try: iter(v) except TypeError: # not iterable pass else: if "__datetime__" in v: v = v["as_str"] return pandas.to_datetime(v) def with_str_parse(value, rule): import ast if isinstance(value, str) and rule not in [str, coerce_datetime]: try: value = ast.literal_eval(value) except (ValueError, TypeError, RuntimeError): pass return rule(value) COERCION_RULES = { "bool": bool, "datetime": coerce_datetime, "dict": dict, "float": float, "tuple": tuple, "mlist": list, "int": int, "list": list, "str": str, "unicode": str, "other": lambda x: x, } def coerce(dtype, value): """ Convert a value to a specific type. If the value is already the given type, then the original value is returned. If the value is None, then the default value given by the type constructor is returned. Otherwise, the type constructor converts and returns the value. """ if "[" in dtype: dtype, inner = dtype.split("[") inner = inner.rstrip("]") else: inner = None if dtype is None: return value if type(value).__name__ == dtype: return value if dtype == "mlist": if isinstance(value, (tuple, set, dict)): return list(value) if isinstance(value, str): try: value = ast.literal_eval(value) return list(value) except ValueError as e: raise ValueError("Failed to coerce string to list") from e return value op = COERCION_RULES[dtype] out = op() if value is None else with_str_parse(value, op) if isinstance(out, dict) and inner is not None: # TODO: recurse into coerce here, to allow list[list[str]] and such? out = {k: COERCION_RULES[inner](v) for k, v in out.items()} if isinstance(out, (tuple, list, set)) and inner is not None: out = op(COERCION_RULES[inner](v) for v in out) return out class RemoteCatalogError(Exception): pass def _has_catalog_dir(args): """Check is any value in args dict needs CATALOG_DIR variable""" from jinja2 import Environment, meta env = Environment() for k, arg in args.items(): parsed_content = env.parse(arg) vars = meta.find_undeclared_variables(parsed_content) if "CATALOG_DIR" in vars: return True return False ================================================ FILE: intake/catalog/zarr.py ================================================ from .base import Catalog from .local import LocalCatalogEntry class ZarrGroupCatalog(Catalog): """A catalog of the members of a Zarr group.""" version = "0.0.1" container = "catalog" partition_access = None name = "zarr_cat" def __init__( self, urlpath, storage_options=None, component=None, metadata=None, consolidated=False, name=None, ): """ Parameters ---------- urlpath : str Location of data file(s), possibly including protocol information storage_options : dict, optional Passed on to storage backend for remote files component : str, optional If None, build a catalog from the root group. If given, build the catalog from the group at this location in the hierarchy. metadata : dict, optional Catalog metadata. If not provided, will be populated from Zarr group attributes. consolidated : bool, optional If True, assume Zarr metadata has been consolidated. """ self._urlpath = urlpath self._storage_options = storage_options or {} self._component = component self._consolidated = consolidated self._grp = None self.name = name super().__init__(metadata=metadata) def _load(self): import zarr if self._grp is None: # obtain the zarr root group if isinstance(self._urlpath, zarr.hierarchy.Group): # use already-opened group, allows support for nested groups # as catalogs root = self._urlpath else: # obtain store if isinstance(self._urlpath, str): # open store from url from fsspec import get_mapper store = get_mapper(self._urlpath, **self._storage_options) else: # assume store passed directly store = self._urlpath # open root group if self._consolidated: # use consolidated metadata root = zarr.open_consolidated(store=store, mode="r") else: root = zarr.open_group(store=store, mode="r") # deal with component path if self._component is None: self._grp = root else: self._grp = root[self._component] # use zarr attributes as metadata self.metadata.update(self._grp.attrs.asdict()) # build catalog entries entries = {} for k, v in self._grp.items(): if isinstance(v, zarr.core.Array): entry = LocalCatalogEntry( name=k, description="", driver="ndzarr", args=dict(urlpath=v), catalog=self, ) else: entry = LocalCatalogEntry( name=k, description="", driver="zarr_cat", args=dict(urlpath=v) ) entries[k] = entry self._entries = entries def to_zarr(self): return self._grp ================================================ FILE: intake/config.py ================================================ """Intake config manipulations and persistence""" # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import ast import contextlib import copy import logging import os import posixpath import warnings from os.path import expanduser import yaml from fsspec.implementations.local import make_path_posix from intake.utils import yaml_load logger = logging.getLogger("intake") confdir = make_path_posix(os.getenv("INTAKE_CONF_DIR", os.path.join(expanduser("~"), ".intake"))) defaults = { "logging": "INFO", "catalog_path": [], "allow_import": True, "allow_templates": True, "allow_pickle": False, "import_on_startup": True, "extra_imports": [], "import_block_list": [], "reader_avoid": [], "environment_conf_parse": "ignore", # "error", "warn", "ignore" } def cfile(): return make_path_posix(os.getenv("INTAKE_CONF_FILE", posixpath.join(confdir, "conf.yaml"))) class Config(dict): """Intake's dict-like config system Instance ``intake.conf`` is globally used throughout the package Attributes: environment_conf_parse : str "ignore" (default), "warn" or raise an "error" when parsing local environment variables as strings. """ def __init__(self, filename=None, **kwargs): self.filename = filename if filename is not None else cfile() self.reload_all() self.temp = None super().__init__(**kwargs) logger.setLevel(self["logging"]) def reset(self): """Set conf values back to defaults""" self.clear() self.update(defaults) def save(self, fn=None): """Save current configuration to file as YAML Uses ``self.filename`` for target location """ # TODO: fsspec? fn = fn or self.filename if fn is False: return try: os.makedirs(os.path.dirname(fn)) except (OSError, IOError): pass with open(fn, "w") as f: yaml.dump(dict(self), f) @contextlib.contextmanager def _unset(self, temp): yield self.clear() self.update(temp) def set(self, update_dict=None, **kw): """Change config values within a context or for the session values: dict This can be deeply nested to set only leaf values See also: ``intake.readers.utils.nested_keys_to_dict`` Examples -------- Value resets after context ends >>> with intake.conf.set(mybval=5): ... ... Set for whole session >>> intake.conf.set(myval=5) Set only a single leaf value within a nested dict >>> intake.conf.set(intake.readers.utils.nested_keys_to_dict({"deep.2.key": True}) """ temp = copy.deepcopy(self) if update_dict: kw.update(update_dict) from intake.readers.utils import merge_dicts self.update(merge_dicts(self, kw)) return self._unset(temp) def __getitem__(self, item): if item in self: return super().__getitem__(item) elif item in defaults: return defaults[item] else: raise KeyError(item) def get(self, key, default=None): if key in self: return super().__getitem__(key) return default def reload_all(self): self.reset() self.load() self.load_env() def load(self, fn=None): """Update global config from YAML file If fn is None, looks in global config directory, which is either defined by the INTAKE_CONF_DIR env-var or is ~/.intake/ . """ # TODO: if fn is not None, set self.filename # TODO: fsspec? fn = fn or self.filename if os.path.isfile(fn): with open(fn) as f: try: self.update(yaml_load(f)) except Exception as e: logger.warning('Failure to load config file "{fn}": {e}' "".format(fn=fn, e=e)) def load_env(self): """Analyse environment variables and update conf accordingly""" # environment variables take precedence over conf file for k, v in os.environ.items(): if k.startswith("INTAKE_"): k2 = k[7:].lower() try: val = ast.literal_eval(v) except (ValueError, SyntaxError) as e: if self["environment_conf_parse"] == "error": logger.info( f"Environment variable '{v}' cannot be parsed by ast.literal_eval()" ) raise e elif self["environment_conf_parse"] == "warn": warnings.warn( f"Failed to parse environment variable '{k}' (value: '{v}') using ast.literal_eval()." f"\nParsing as string." f"\nError was: '{type(e).__name__}: {e}'" ) val = str(v) # make sure it is a string self[k2] = val def intake_path_dirs(path): """Return a list of directories from the intake path. If a string, perhaps taken from an environment variable, then the list of paths will be split on the character ":" for posix of ";" for windows. Protocol indicators ("protocol://") will be ignored. """ if isinstance(path, (list, tuple)): return path import re pattern = re.compile(";" if os.name == "nt" else r"(?=1' 'hvplot>=0.8.1'`" ) from .gui import GUI css = """ .scrolling { overflow: scroll; } """ pn.config.raw_css.append(css) # add scrolling class from css (panel GH#383, GH#384) ex = pn.extension("codeeditor", template="fast") gl["instance"] = GUI() return ex def __getattr__(attr): if attr in {"instance", "gui"}: do_import() return gl[attr] def output_notebook(*_, **__): """ Load the notebook extension """ return do_import() ================================================ FILE: intake/interface/base.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2019, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import os from collections import OrderedDict import panel as pn MAX_WIDTH = 5000 here = os.path.abspath(os.path.dirname(__file__)) ICONS = { "logo": os.path.join(here, "icons", "logo.png"), "error": os.path.join(here, "icons", "baseline-error-24px.svg"), } def enable_widget(widget, enable=True): """Set disabled on widget""" widget.disabled = not enable def coerce_to_list(items, preprocess=None): """Given an instance or list, coerce to list. With optional preprocessing. """ if not isinstance(items, list): items = [items] if preprocess: items = list(map(preprocess, items)) return items class Base(object): """ Base class for composable panel objects that make up the GUI. Parameters ---------- children: list of panel objects children that will be used to populate the panel when visible panel: panel layout object instance of a panel layout (row or column) that contains children when visible watchers: list of param watchers watchers that are set on children - cleaned up when visible is set to false. visible: bool whether or not the instance should be visible. When not visible ``panel`` is empty. logo : bool, opt whether to show the intake logo in a panel to the left of the main panel. Default is False """ children = None panel = None watchers = None visible_callback = None logo_panel = pn.Column( pn.pane.PNG(ICONS["logo"], align="center"), margin=(25, 0, 0, 0), width=50 ) logo = False def __init__(self, visible=True, visible_callback=None, logo=False): self.visible = visible self.visible_callback = visible_callback self.logo = logo @property def panel(self): if not self.logo: return self._panel return pn.Row(self.logo_panel, self._panel, margin=0) @panel.setter def panel(self, panel): self._panel = panel def servable(self, *args, **kwargs): return self.panel.servable(*args, **kwargs) def show(self, *args, **kwargs): return self.panel.show(*args, **kwargs) def __repr__(self): """Print output""" return self.panel.__repr__() def _repr_mimebundle_(self, *args, **kwargs): """Display in a notebook or a server""" try: return self.panel._repr_mimebundle_(*args, **kwargs) except Exception: raise NotImplementedError("Panel does not seem to be set " "up properly") def setup(self): """Should instantiate widgets, set ``children``, and set watchers""" raise NotImplementedError @property def visible(self): """Whether or not the instance should be visible.""" return self._visible @visible.setter def visible(self, visible): """When visible changed, do setup or unwatch and call visible_callback""" self._visible = visible pan = getattr(self._panel, "_layout", self._panel) if visible and len(pan.objects) == 0: self.setup() self._panel.extend(self.children) elif not visible and len(pan.objects) > 0: self.unwatch() pan.clear() if self.visible_callback: self.visible_callback(visible) def unwatch(self): """Get rid of any lingering watchers and remove from list""" if self.watchers is not None: unwatched = [] for watcher in self.watchers: watcher.inst.param.unwatch(watcher) unwatched.append(watcher) self.watchers = [w for w in self.watchers if w not in unwatched] def __getstate__(self): """Serialize the current state of the object""" return {"visible": self.visible} def __setstate__(self, state): """Set the current state of the object from the serialized version. Works inplace. See ``__getstate__`` to get serialized version and ``from_state`` to create a new object.""" self.visible = state.get("visible", True) return self @classmethod def from_state(cls, state): """Create a new object from a serialized exising object. Examples -------- original = cls() copy = cls.from_state(original.__getstate__()) """ return cls().__setstate__(state) class BaseSelector(Base): """Base class for capturing selector logic. Parameters ---------- preprocess: function run on every input value when creating options widget: panel widget selector widget which this class keeps uptodate with class properties """ preprocess = None widget = None @property def labels(self): """Labels of items in widget""" return self.widget.labels @property def items(self): """Available items to select from""" return self.widget.values @items.setter def items(self, items): """When setting items make sure widget options are uptodate""" if items is not None: self.options = items def _create_options(self, items): """Helper method to create options from list, or instance. Applies preprocess method if available to create a uniform output """ return OrderedDict(map(lambda x: (x.name, x), coerce_to_list(items, self.preprocess))) @property def options(self): """Options available on the widget""" return self.widget.options @options.setter def options(self, new): """Set options from list, or instance of named item Over-writes old options """ options = self._create_options(new) if self.widget.value: self.widget.set_param(options=options, value=list(options.values())[:1]) else: self.widget.options = options self.widget.value = list(options.values())[:1] def add(self, items): """Add items to options""" options = self._create_options(items) for k, v in options.items(): if k in self.labels and v not in self.items: options.pop(k) count = 0 while f"{k}_{count}" in self.labels: count += 1 options[f"{k}_{count}"] = v self.widget.options.update(options) self.widget.param.trigger("options") self.widget.value = list(options.values())[:1] def remove(self, items): """Remove items from options""" items = coerce_to_list(items) new_options = {k: v for k, v in self.options.items() if v not in items} self.widget.options = new_options self.widget.param.trigger("options") @property def selected(self): """Value selected on the widget""" ops = self.widget.options return [ val for val in self.widget.value if (val in ops.values() if isinstance(ops, dict) else val in ops) ] @selected.setter def selected(self, new): """Set selected from list or instance of object or name. Over-writes existing selection """ def preprocess(item): if isinstance(item, str): return self.options[item] return item items = coerce_to_list(new, preprocess) self.widget.value = items class BaseView(Base): def __getstate__(self, include_source=True): """Serialize the current state of the object. Set include_source to False when using with another panel that will include source.""" if include_source: return { "visible": self.visible, "label": self.source._name, "source": self.source.__getstate__(), } else: return {"visible": self.visible} def __setstate__(self, state): """Set the current state of the object from the serialized version. Works inplace. See ``__getstate__`` to get serialized version and ``from_state`` to create a new object.""" if "source" in state: self.source = state["source"] self.visible = state.get("visible", True) @property def source(self): return self._source @source.setter def source(self, source): """When the source gets updated, update the select widget""" if isinstance(source, list): # if source is a list, get first item or None source = source[0] if len(source) > 0 else None self._source = source ================================================ FILE: intake/interface/catalog/__init__.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2019, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- ================================================ FILE: intake/interface/catalog/add.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2019, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import ast import os from functools import partial import fsspec import panel as pn from fsspec.registry import known_implementations import intake from ..base import MAX_WIDTH, Base, enable_widget class FileSelector(Base): """ Panel interface for picking files The current path is stored in .path and the current selection is stored in .url. Parameters ---------- filters: list of string extentions that are included in the list of files - correspond to catalog extensions. done_callback: func, opt called when the object's main job has completed. In this case, selecting a file. Attributes ---------- url: str path to local catalog file children: list of panel objects children that will be used to populate the panel when visible panel: panel layout object instance of a panel layout (row or column) that contains children when visible watchers: list of param watchers watchers that are set on children - cleaned up when visible is set to false. """ def __init__(self, filters=["yaml", "yml"], done_callback=None, **kwargs): self.filters = filters self.panel = pn.Column(name="Local", width_policy="max", margin=0) self.done_callback = done_callback self.fs = fsspec.filesystem("file") super().__init__(**kwargs) def setup(self): self.path_text = pn.widgets.TextInput(value=os.getcwd() + "/", width_policy="max") self.protocol = pn.widgets.Select( options=list(sorted(known_implementations)), value="file", name="protocol" ) self.storage_options = pn.widgets.TextInput(name="kwargs", value="{}") self.go = pn.widgets.Button(name="⇨") self.validator = pn.pane.SVG(None, width=25) self.main = pn.widgets.MultiSelect(size=15, width_policy="max") self.home = pn.widgets.Button(name="🏠", width=40, height=30) self.up = pn.widgets.Button(name="‹", width=30, height=30) self.make_options() self.watchers = [ self.go.param.watch(self.go_clicked, "clicks"), self.protocol.param.watch(self.protocol_changed, "value"), # self.path_text.param.watch(self.validate, ['value']), # self.path_text.param.watch(self.make_options, ['value']), self.home.param.watch(self.go_home, "clicks"), self.up.param.watch(self.move_up, "clicks"), self.main.param.watch(self.move_down, ["value"]), ] self.children = [ pn.Row(self.protocol, self.storage_options), pn.Row(self.home, self.up, self.path_text, self.go, margin=0), self.main, ] def protocol_changed(self, *_): self.path_text.value = "" self.main.options = [] self.main.value = [] def go_clicked(self, *_): self.fs = fsspec.filesystem( self.protocol.value, **ast.literal_eval(self.storage_options.value) ) self.make_options() @property def path(self): return self.path_text.value @property def url(self): """Path to local catalog file""" return self.protocol.value + "://" + os.path.join(self.path, self.main.value[0]) def move_up(self, arg=None): self.path_text.value = self.fs._parent(self.path_text.value) self.make_options() def go_home(self, arg=None): self.protocol.value = "file" self.path_text.value = os.getcwd() + os.path.sep def make_options(self, arg=None): if self.done_callback: self.done_callback(False) out = [] try: for f in self.fs.ls(self.path, True): bn = os.path.basename(f["name"].rstrip("/")) if bn.startswith("."): continue elif f["type"] == "directory": out.append(bn + "/") elif not self.filters or any(bn.endswith(ext) for ext in self.filters): out.append(bn) except Exception as e: print(e) self.main.value = [] self.main.options = sorted(out) def move_down(self, *events): for event in events: if event.name == "value" and len(event.new) > 0: fn = event.new[0] if fn.endswith("/"): if self.path_text.value: self.path_text.value = os.path.join(self.path_text.value, fn) else: self.path_text.value = fn self.make_options() elif self.done_callback: self.done_callback(True) def __getstate__(self): """Serialize the current state of the object.""" return {"path": self.path, "selected": self.main.value} def __setstate__(self, state): """Set the current state of the object from the serialized version. Works inplace. See ``__getstate__`` to get serialized version and ``from_state`` to create a new object.""" self.path_text.value = state["path"] self.main.value = state["selected"] return self class URLSelector(Base): """ Panel interface for inputting a URL to a remote catalog The inputted URL is stored in .url. Attributes ---------- url: str url to remote files (including protocol) children: list of panel objects children that will be used to populate the panel when visible panel: panel layout object instance of a panel layout (row or column) that contains children when visible watchers: list of param watchers watchers that are set on children - cleaned up when visible is set to false. """ def __init__(self, **kwargs): self.panel = pn.Row(name="URL", width_policy="max", margin=0) super().__init__(**kwargs) def setup(self): self.main = pn.widgets.TextInput(placeholder="Full URL with protocol", width_policy="max") self.children = ["URL:", self.main] @property def url(self): """URL to remote files (including protocol)""" return self.main.value def __getstate__(self): """Serialize the current state of the object.""" return {"url": self.url} def __setstate__(self, state): """Set the current state of the object from the serialized version. Works inplace. See ``__getstate__`` to get serialized version and ``from_state`` to create a new object.""" self.main.value = state["url"] return self class CatAdder(Base): """Panel for adding new cats from local file or remote Parameters ---------- done_callback: function with cat as input function that is called when the "Add Catalog" button is clicked. Attributes ---------- cat_url: str url to remote files or path to local files. Depends on active tab cat: catalog catalog object initialized from from cat_url children: list of panel objects children that will be used to populate the panel when visible panel: panel layout object instance of a panel layout (row or column) that contains children when visible watchers: list of param watchers watchers that are set on children - cleaned up when visible is set to false. """ tabs = None def __init__(self, done_callback=None, **kwargs): self.done_callback = done_callback self.panel = pn.Column( name="Add Catalog", width_policy="max", max_width=MAX_WIDTH, margin=0 ) self.widget = pn.widgets.Button(name="Add Catalog", disabled=True, width_policy="min") self.fs = FileSelector(done_callback=partial(enable_widget, self.widget)) self.url = URLSelector() super().__init__(**kwargs) def setup(self): self.selectors = [self.fs, self.url] self.tabs = pn.Tabs(*map(lambda x: x.panel, self.selectors)) self.watchers = [ self.widget.param.watch(self.add_cat, "clicks"), ] self.children = [self.tabs, self.widget] @property def cat_url(self): """URL to remote files or path to local files. Depends on active tab.""" url = self.selectors[self.tabs.active].url if self.selectors[self.tabs.active] is self.fs: fs = self.fs.fs else: fs = None return url, fs @property def cat(self): """Catalog object initialized from from cat_url""" # might want to do some validation in here url, fs = self.cat_url if fs: return intake.open_catalog(url, fs=fs) else: return intake.open_catalog(url) def add_cat(self, arg=None): """Add cat and close panel""" try: self.done_callback(self.cat) self.panel.visible = False except Exception as e: raise e ================================================ FILE: intake/interface/catalog/search.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2019, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import panel as pn class Search: """Search panel for searching a list of catalogs Parameters ---------- done_callback: function with cats as input function that is called when new cats have been generated via the search functionality """ def __init__(self, done_callback: callable): self.done_callback = done_callback self.tinput = pn.widgets.TextInput() ok = pn.widgets.Button(name="OK") ok.on_click(self.go) label = pn.widgets.StaticText(value="Search") self.panel = pn.Row(label, self.tinput, ok) def go(self, *_): if self.tinput.value: self.done_callback(self.tinput.value) ================================================ FILE: intake/interface/gui.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2019, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import panel as pn import intake from intake.interface.base import ICONS from intake.interface.catalog.add import CatAdder from intake.interface.catalog.search import Search from intake.interface.source import defined_plots class GUI: """ Top level GUI panel This class is responsible for coordinating the inputs and outputs of various sup-panels and their effects on each other. Parameters ---------- cats: dict of catalogs catalogs used to initalize the cat panel, {display_name: cat_object} """ def __init__(self, cats=None): # state self._children = {} # cat name in the selector to child catalogs' names: cat objects # mapping of name in the selector to catalog object self._cats = cats or {"builtin": intake.cat} self._sources = {} # source name: source instance # layout col0 = pn.Column(pn.pane.PNG(ICONS["logo"], align="center"), margin=(25, 0, 0, 0), width=50) self.catsel = pn.widgets.MultiSelect( name="Catalogs", options=list(self._cats), value=[], size=13, styles={"width": "25%"}, ) self.catsel.param.watch(self.cat_selected, "value") add = pn.widgets.Button(name="+") sub = pn.widgets.Button(name="-") search = pn.widgets.Button(name="🔍") col1 = pn.Column(self.catsel, pn.Row(add, sub, search)) add.on_click(self.add_clicked) sub.on_click(self.sub_clicked) search.on_click(self.search_clicked) self.sourcesel = pn.widgets.MultiSelect(name="Sources", size=13, styles={"width": "25%"}) plot = pn.widgets.Button(name="📊") plot.on_click(self.plot_clicked) self.sourcesel.param.watch(self.source_selected, "value") col2 = pn.Column(self.sourcesel, plot) self.sourceinf = pn.widgets.CodeEditor( readonly=True, language="yaml", print_margin=False, annotations=[] ) col3 = pn.Column(self.sourceinf) row0 = pn.Row(col0, col1, col2, col3, styles={"width": "100%"}) self.plots = defined_plots.Plots() self.plots.panel.visible = False self.add = CatAdder(done_callback=self.add_catalog) self.add.panel.visible = False self.search = Search(done_callback=self.searched) self.search.panel.visible = False self.row1 = pn.Row(self.plots.panel, self.add.panel, self.search.panel) self.main = pn.Column(row0, self.row1) self.cat_selected(None) def _repr_mimebundle_(self, *args, **kwargs): """Display in a notebook or a server""" return self.main._repr_mimebundle_(*args, **kwargs) def show(self, *args, **kwargs): return self.main.show(*args, **kwargs) def __repr__(self): return "Intake GUI" def cat_selected(self, *_): right = "└─>" cat = self.catsel.value if not cat: return else: catname = cat[0] cat = self._cats[catname] catsel_needs_update = False self._sources.clear() indent = len(catname) - len(catname.lstrip(" ")) + 2 for entry in cat: name = " " * indent + right + entry source = cat[entry] if isinstance(source, intake.catalog.Catalog): if name not in self._cats: self._cats[name] = source self._children.setdefault(catname, []).append(name) catsel_needs_update = True elif "Catalog" in getattr(source, "output_instance", ""): if name not in self._cats: cat = source.read() self._cats[name] = cat self._children.setdefault(catname, []).append(name) catsel_needs_update = True else: self._sources[entry] = source if catsel_needs_update: self.update_catsel() self.sourcesel.param.update(options=list(self._sources)) def update_catsel(self): self.catsel.param.update(options=get_catlist(self._cats, self._children)) def add_catalog(self, cat, name=None, **_): if hasattr(cat, "token"): if "CATALOG_PATH" in cat.user_parameters: par = cat.user_parameters["CATALOG_PATH"] name = getattr(par, "default", str(par)) else: name = cat.token else: name = name or getattr(cat, "token", cat.name) self._cats[name] = cat self.update_catsel() def source_selected(self, *_): from intake import BaseReader import yaml source = self.sourcesel.value if not source: return else: source = self._sources[source[0]] if isinstance(source, BaseReader): # could have reverted to ReaderDescription, but this version will include any # other readers/data, not just references. d = {"cls": source.qname()} d.update(source.to_dict()) txt = yaml.dump(d, default_flow_style=False) else: txt = yaml.dump(source._yaml()["sources"], default_flow_style=False) self.sourceinf.param.update(value=txt) def plot_clicked(self, *_): if self.plots.panel.visible: self.plots.panel.visible = False elif self.sources: self.plots.source = self.sources[0] self.add.panel.visible = False self.plots.panel.visible = True self.search.panel.visible = False def searched(self, searchstring: str): if self.cats: cat = self.cats[0] cat2 = cat.search(searchstring) self.add_catalog(cat2, name=f"search <{searchstring[:10]}>") def add_clicked(self, *_): if self.add.panel.visible: self.add.panel.visible = False else: self.add.panel.visible = True self.plots.panel.visible = False self.search.panel.visible = False def sub_clicked(self, *_): for catname in self.catsel.value: self.remove_cat(catname) def remove_cat(self, catname, done=True): self._cats.pop(catname, None) # remake "builtin" if accidentally removed? for cat in self._children.get(catname, []): self.remove_cat(cat, done=False) if done: self.catsel.param.update(options=list(self._cats)) def search_clicked(self, *_): if self.search.panel.visible: self.search.panel.visible = False else: self.add.panel.visible = False self.plots.panel.visible = False self.search.panel.visible = True @property def cats(self): """Cats that have been selected from the cat sub-panel""" return [self._cats[k] for k in self.catsel.value] @property def sources(self): """Sources that have been selected from the source sub-panel""" return [self._sources[k] for k in self.sourcesel.value] @property def source_instance(self): return self.sources[0] if self.sourcesel.values else None def get_catlist(catnames, children, outlist=None, seen=None): outlist = outlist or [] seen = seen or set() for name in sorted(catnames): if name in seen: continue seen.add(name) outlist.append(name) if name in children: get_catlist(children[name], children, outlist, seen) return outlist ================================================ FILE: intake/interface/source/__init__.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2019, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- ================================================ FILE: intake/interface/source/defined_plots.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2022, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import hvplot from packaging.version import Version try: import xrviz from xrviz.dashboard import Dashboard as XRViz assert Version(xrviz.__version__) >= Version("0.1.1") except ImportError: xrviz = False import panel as pn from ...catalog.local import LocalCatalogEntry, YAMLFileCatalog from ..base import BaseView class Event: def __init__(self, plot, **kwargs): self.object = plot for key, val in kwargs.items(): setattr(self, key, val) class Plots(BaseView): """ Panel for displaying pre-defined plots from catalog. Parameters ---------- source: intake catalog entry, or list of same source to describe in this object edit_callback: callback to alert that plot has been edited Attributes ---------- has_plots: bool whether the source has plots defined options: list plots options defined on the source selected: str name of selected plot children: list of panel objects children that will be used to populate the panel when visible panel: panel layout object instance of a panel layout (row or column) that contains children when visible watchers: list of param watchers watchers that are set on children - cleaned up when visible is set to false. """ select = None def __init__(self, source=None, **kwargs): self.custom = pn.widgets.Button(name="Create", width=70, align="center") self.source = source self.panel = pn.Column(name="Plot", width_policy="max", margin=0) self._behavior_callback = None self._callbacks = [] super().__init__(**kwargs) def setup(self): self.instructions = pn.pane.Markdown("**Select Plot**", align="center", width=70) self.select = pn.widgets.Select( options=self.options, height=30, align="center", min_width=200 ) # Add spaces in front of each option to make it render correctly. # This is a bit of a hack to make a Select which only displays the down arrow. self.edit_options = pn.widgets.Select( options=[" Edit", " Clone", " Rename", " Delete"], align="center", width=25, margin=(5, -10), ) self.pane = pn.pane.HoloViews(self._plot_object(self.selected), name="Plot") self.interact_label = pn.pane.Markdown( "Name", align="center", width_policy="max", min_width=100 ) self.interact_name = pn.widgets.TextInput(placeholder="Name of new plot...") self.interact_cancel = pn.widgets.Button(name="Cancel", width=100) self.interact_save = pn.widgets.Button(name="Save", button_type="primary", width=100) self.watchers = [ self.select.param.watch(self.plot_selected, ["value"]), self.custom.param.watch(self.interact, ["clicks"]), self.edit_options.param.watch(self.interact, ["value"]), self.interact_name.param.watch(self.name_changed, ["value_input"]), self.interact_cancel.param.watch(self.cancel, ["clicks"]), self.interact_save.param.watch(self.interact_action, ["clicks"]), ] self.alert = pn.pane.Alert(alert_type="danger") self.out = pn.Row(self.pane, name="Plot") self.row_select_plots = pn.Row( self.instructions, self.select, self.custom, self.edit_options, ) self.row_dialog_buttons = pn.Row( self.interact_label, self.interact_name, self.interact_cancel, self.interact_save, ) self.children = [ pn.Column( self.row_select_plots, self.row_dialog_buttons, pn.Row(self.alert), self.out, ) ] # Set initial visibility self.row_select_plots[-1].visible = False # edit_options dropdown self.row_dialog_buttons.visible = False self.alert.visible = False @BaseView.source.setter def source(self, source): """When the source gets updated, update the options in the selector""" if source and isinstance(source, list): source = source[0] if isinstance(source, LocalCatalogEntry): source = source() BaseView.source.fset(self, source) if self.select: self.select.options = self.options if source and source.container == "dataframe": self.custom.disabled = False elif source and xrviz and source.container in ["xarray", "ndarray", "numpy"]: self.custom.disabled = False else: self.custom.disabled = True @property def has_plots(self): """Whether the source has plots defined""" return self.source is not None and len(self._source.plots) > 0 @property def options(self): """Plots options defined on the source""" return (["None"] + list(self.source.metadata["plots"])) if self.source is not None else [] @property def selected(self): """Name of selected plot""" return self.select.value if self.select is not None else None @selected.setter def selected(self, selected): """When plot is selected set, make sure widget stays upto date""" self.select.value = selected def watch(self, callback): self._callbacks.append(callback) def plot_selected(self, *events): for event in events: if event.name == "value": self.pane.object = self._plot_object(event.new) self.custom.name = "Create" if str(self.select.value) == "None" else "Edit" self.edit_options.visible = self.custom.name == "Edit" def name_changed(self, *events): for event in events: if event.name == "value_input": self.alert.visible = False # Empty name not allowed and name cannot already be in use if len(event.new) == 0: self.interact_save.disabled = True elif event.new in self.options: self.interact_save.disabled = True self.alert.object = f'Name "{event.new}" already exists' self.alert.visible = True else: self.interact_save.disabled = False def cancel(self, _): self.pane.object = self._plot_object(self.selected) self.out[0] = self.pane self.row_select_plots.visible = True self.row_dialog_buttons.visible = False self._behavior_callback = None def interact_action(self, _): # Stop catalog from autoreloading self.source.cat.ttl = None if self._behavior_callback is not None: # Perform action event = self._behavior_callback() # Save catalog (if YAMLFileCatalog) if isinstance(self.source.cat, YAMLFileCatalog): self.source.cat.add(self.source) # Callbacks for cb in self._callbacks: cb(event) # Reset custom behavior callback self._behavior_callback = None # End action self.cancel(None) def interact(self, _): # Create/Edit/Clone/Rename/Delete was selected if self._behavior_callback is not None: # This is a short-circuit to allow edit options reset # without causing an infinite loop return if self.selected == "None": # Create self._behavior_callback = self._create self.interact_save.name = "Save" self.interact_save.disabled = True self.interact_label.object = "Name " self.interact_name.value = "" self.interact_name.visible = True viz = self.draw() self.out[0] = viz elif self.edit_options.value == " Clone": self._behavior_callback = self._clone self.interact_save.name = "Clone" self.interact_save.disabled = True self.interact_label.object = f'Clone "**{self.selected}**" as ' self.interact_name.value = "" self.interact_name.visible = True elif self.edit_options.value == " Rename": self._behavior_callback = self._rename self.interact_save.name = "Rename" self.interact_save.disabled = True self.interact_label.object = f'Rename "**{self.selected}**" to ' self.interact_name.value = "" self.interact_name.visible = True elif self.edit_options.value == " Delete": self._behavior_callback = self._delete self.interact_save.name = "Delete" self.interact_save.disabled = False self.interact_label.object = f'Really delete "**{self.selected}**" ?' self.interact_name.visible = False elif self.edit_options.value == " Edit": # Edit self._behavior_callback = self._edit self.interact_save.name = "Save" self.interact_save.disabled = False self.interact_label.object = f'Editing "**{self.selected}**"' self.interact_name.visible = False viz = self.draw() self.out[0] = viz else: raise ValueError(self.edit_options.value) # Update visibility of components self.row_dialog_buttons.visible = True self.row_select_plots.visible = False # Reset edit options selection (won't trigger # edit action because _behavior_callback is set) self.edit_options.value = "Edit" def draw(self): if self.selected == "None": kwargs = {"y": []} else: kwargs = self.source.metadata["plots"][self.selected] if self.source.container == "dataframe": df = self.source.to_dask() if df.npartitions == 1: df = df.compute() self._viz = viz = hvplot.explorer(df, **kwargs) elif self.source.container in ["xarray", "ndarray", "numpy"]: import xarray try: data = self.source.to_dask() except NotImplementedError: data = self.source.read() if not isinstance(data, (xarray.DataArray, xarray.Dataset)): data = xarray.DataArray(data) self._viz = XRViz(data, **kwargs) viz = self._viz.panel else: raise ValueError(f"Unhandled container type {self.source.container}") return viz def _plot_object(self, selected): from hvplot import hvPlot if selected and str(selected) != "None": args = self.source.metadata["plots"][str(selected)] plot_method = hvPlot(self.source)(**args) self.out[0] = self.pane return plot_method def _create(self): plot_name = self.interact_name.value # Add plot metadata to both DataSource and CatalogEntry self.source.metadata.setdefault("plots", {})[plot_name] = self._viz.settings() self.source.entry._metadata.setdefault("plots", {})[plot_name] = self._viz.settings() # Add new plot name to self.options self.select.options = self.options # Select new graph self.selected = plot_name return Event(self, kind="create", plot_name=plot_name) def _edit(self): # Update plot metadata for both DataSource and CatalogEntry self.source.metadata["plots"][self.selected] = self._viz.settings() self.source.entry._metadata["plots"][self.selected] = self._viz.settings() return Event(self, kind="edit", plot_name=self.selected) def _clone(self): plot_name = self.interact_name.value # Clone plot metadata for both DataSource and CatalogEntry self.source.metadata["plots"][plot_name] = self.source.metadata["plots"][ self.selected ].copy() self.source.entry._metadata["plots"][plot_name] = self.source.entry._metadata["plots"][ self.selected ].copy() # Add new plot name to self.options self.select.options = self.options # Select new graph self.selected = plot_name return Event(self, kind="clone", plot_name=plot_name, cloned_from=self.selected) def _rename(self): plot_name = self.interact_name.value # Rename plot metadata for both DataSource and CatalogEntry self.source.metadata["plots"][plot_name] = self.source.metadata["plots"].pop(self.selected) self.source.entry._metadata["plots"][plot_name] = self.source.entry._metadata["plots"].pop( self.selected ) # Update available options in dropdown self.select.options = self.options # Select renamed graph self.selected = plot_name return Event(self, kind="rename", plot_name=plot_name, prev_name=self.selected) def _delete(self): # Delete plot metadata from both DataSource and CatalogEntry del self.source.metadata["plots"][self.selected] del self.source.entry._metadata["plots"][self.selected] # Update available options in dropdown self.select.options = self.options self.selected = "None" return Event(self, kind="delete", plot_name=self.selected) def __getstate__(self, include_source=True): """Serialize the current state of the object. Set include_source to False when using with another panel that will include source.""" state = super().__getstate__(include_source) state.update( { "selected": self.selected, } ) return state def __setstate__(self, state): """Set the current state of the object from the serialized version. Works inplace. See ``__getstate__`` to get serialized version and ``from_state`` to create a new object.""" super().__setstate__(state) if self.visible: self.selected = state.get("selected") return self ================================================ FILE: intake/readers/__init__.py ================================================ from intake.readers.datatypes import * # noqa: F403 from intake.readers.readers import * # noqa: F403 from intake.readers.convert import BaseConverter, Pipeline, auto_pipeline, path from intake.readers.entry import DataDescription, ReaderDescription from intake.readers.user_parameters import BaseUserParameter, SimpleUserParameter import intake.readers.user_parameters import intake.readers.transform import intake.readers.output import intake.readers.catalogs import intake.readers.examples from intake.readers.metadata import metadata_fields def recommend(data, *args, reader=False, **kwargs): """Recommend datatypes or readers depending on what is passed Calls intake.readers.datatypes.recommend or intake.readers.readers.recommend Please see their respective docstrings; or, better, explicitly call the one which you intend to use. Parameters ---------- reader: bool (False) If the "data" is a URL or other base string that can be used to recommend a datatype, setting this flag True will take the first guess and return the readers that can handle it. """ if isinstance(data, intake.readers.datatypes.BaseData): return intake.readers.readers.recommend(data, *args, **kwargs) datat = intake.readers.datatypes.recommend(data, *args, **kwargs) if reader is False: return datat return datat[0](data).possible_outputs ================================================ FILE: intake/readers/catalogs.py ================================================ """Data readers which create Catalog objects""" from __future__ import annotations import itertools import json import fsspec from intake.readers import datatypes from intake.readers.entry import Catalog, DataDescription, ReaderDescription from intake.readers.readers import BaseReader from intake.readers.utils import LazyDict class TiledLazyEntries(LazyDict): """A dictionary-like, which only loads a key's value from Tiled on demand""" def __init__(self, client): self.client = client def __getitem__(self, item: str) -> ReaderDescription: from intake.readers.readers import TiledClient client = self.client[item] data = datatypes.TiledService(url=client.uri, metadata=client.item) if type(client).__name__ == "Node": reader = TiledCatalogReader(data=data) else: reader = TiledClient( data, output_instance=f"{type(client).__module__}:{type(client).__name__}", ) return reader.to_entry() def __len__(self): return len(self.client) def __iter__(self): return iter(self.client) def __repr__(self): return f"TiledEntries {sorted(set(self))}" class TiledCatalogReader(BaseReader): """Creates a catalog of Tiled datasets from a root URL The generated catalog is lazy, only the list of entries is got eagerly, but they are fetched only on demand. """ implements = {datatypes.TiledService} output_instance = "intake.readers.entry:Catalog" imports = {"tiled"} def _read(self, data, **kwargs): from tiled.client import from_uri opts = data.options.copy() opts.update(kwargs) client = from_uri(data.url, **opts) entries = TiledLazyEntries(client) return Catalog( entries=entries, aliases={k: k for k in sorted(client)}, metadata=client.item, ) class SQLAlchemyCatalog(BaseReader): """Uses SQLAlchemy to get the list of tables at some SQL URL These tables are presented as data entries in a catalog, but could then be loaded by any reader that implements SQLQuery. """ implements = {datatypes.Service} imports = {"sqlalchemy"} output_instance = "intake.readers.entry:Catalog" def _read(self, data, views=True, schema=None, **kwargs): import sqlalchemy engine = sqlalchemy.create_engine(data.url) meta = sqlalchemy.MetaData() meta.reflect(bind=engine, views=views, schema=schema) tables = list(meta.tables) entries = { name: DataDescription( "intake.readers.datatypes:SQLQuery", kwargs={"conn": data.url, "query": name}, ) for name in tables } return Catalog(data=entries) class StacCatalogReader(BaseReader): """Create a Catalog from a STAC endpoint or file https://stacspec.org/en """ # STAC organisation: Catalog->Item->Asset. Catalogs can reference Catalogs. # also have ItemCollection (from searching a Catalog) and CombinedAsset (multi-layer data) # Asset and CombinedAsset are datasets, the rest are Catalogs implements = {datatypes.STACJSON, datatypes.Literal} imports = {"pystac"} output_instance = "intake.readers.entry:Catalog" def _read( self, data, cls: str = "Catalog", signer: callable | None = None, prefer: tuple[str] | str | None = ("xarray", "numpy"), **kwargs, ): """ Parameters ---------- data: JSON-like STAC response, or the actual URL to fetch this from cls: the class of STAC object this should be, Catalog, Item, Asset, ItemCollection, Feature signer: if given, apply this function to assets and derived items, to add signature/ auth information to HTTP calls prefer: if given, select readers matching this spec from those that might apply to a given asset; e.g., "xarray" for those data that can be read into xarray. """ import pystac cls = getattr(pystac, cls) if isinstance(data, datatypes.JSONFile): self._stac = cls.from_file(data.url) else: self._stac = cls.from_dict(data.data) metadata = self._stac.to_dict() metadata.pop("links", None) self.metadata.update(metadata) cat = Catalog(metadata=self.metadata) items = [] # the following can be slow and could be deferred to lazy entries, if we can get # the names without details cheaply if isinstance(self._stac, pystac.Catalog): items = itertools.chain(self._stac.get_children(), self._stac.get_items()) elif isinstance(self._stac, pystac.ItemCollection): items = self._stac.items if hasattr(self._stac, "assets"): for key, value in self._stac.assets.items(): if signer: signer(value) try: reader = self._get_reader( value, signer=signer, prefer=prefer, metadata=self.metadata ).to_entry() cat[key] = reader except (ValueError, TypeError, StopIteration): pass for subcatalog in items: subcls = type(subcatalog).__name__ cat[subcatalog.id] = ReaderDescription( reader=self.qname(), kwargs=dict( { "cls": subcls, "data": datatypes.Literal(subcatalog.to_dict()), "signer": signer, "prefer": prefer, }, **kwargs, ), metadata=self.metadata, ) return cat @staticmethod def _get_reader(asset, signer=None, prefer=None, metadata=None): """ Assign intake driver for data I/O prefer: str | list[str] passed to .to_reader to inform what class of reader would be preferable, if any """ url = asset.href mime = asset.media_type if mime in ["", "null"]: mime = None # if mimetype not registered try rasterio driver storage_options = asset.extra_fields.get("table:storage_options", {}) if "credential" in storage_options: # MS-ABFS specific argument; look for MS identifier? storage_options["sas_token"] = storage_options["credential"] cls = datatypes.recommend(url, mime=mime, storage_options=storage_options, head=False) meta = asset.to_dict() if cls: data = cls[0](url=url, metadata=meta, storage_options=storage_options) else: raise ValueError if "stac-items" in meta.get("roles", []) or [] and isinstance(data, datatypes.Parquet): from intake.readers.readers import DaskGeoParquet data.metadata["signer"] = signer data.metadata["prefer"] = prefer return DaskGeoParquet(data) else: rr = None return data.to_reader(reader=rr, outtype=prefer, metadata=metadata) class StackBands(BaseReader): """Reimplementation of "StackBandsSource" from intake-stac Takes the subitems of a given collection and concatenates them using xarray on the given dimension. """ implements = {datatypes.STACJSON, datatypes.Literal} imports = {"pystac", "xarray"} output_instance = "xarray:Dataset" def _read(self, data, bands: list[str], concat_dim: str = "band", signer=None, **kw): """ Parameters ---------- data: STAC endpoint, file, or dict literal bands: band_id, name or common_name to select from contained items concat_dim: concat dimansion in the xarray sense Returns ------- Configured reader, so that you can persist it. Call .read() again to execute the read and concat operation. """ # this should be a separate reader for STACJSON, import pystac from pystac.extensions.eo import EOExtension cls = pystac.Item if isinstance(data, datatypes.JSONFile): self._stac = cls.from_file(data.url) else: self._stac = cls.from_dict(data.data) if signer: signer(self._stac) band_info = [band.to_dict() for band in EOExtension.ext(self._stac).bands] metadatas = {} titles = [] hrefs = [] types = [] assets = self._stac.assets for band in bands: # band can be band id, name or common_name if band in assets: info = next( ( b for b in band_info if band in [b.get(_) for _ in ["common_name", "name", "id"]] ), None, ) else: info = next( (b for b in band_info if band in [b.get(_) for _ in ["common_name", "name"]]), None, ) if info is not None: band = info.get("id", info.get("name")) if band not in assets or info is None: valid_band_names = [] for b in band_info: valid_band_names.append(b.get("id", b.get("name"))) valid_band_names.append(b.get("common_name")) raise ValueError( f"{band} not found in list of eo:bands in collection." f"Valid values: {sorted(list(set(valid_band_names)))}" ) asset = assets.get(band) metadatas[band] = asset.to_dict() titles.append(band) types.append(asset.media_type) hrefs.append(asset.href) unique_types = set(types) if len(unique_types) != 1: raise ValueError( f"Stacking failed: bands must have same type, multiple found: {unique_types}" ) reader = StacCatalogReader._get_reader(asset, signer=signer) reader.kwargs["dim"] = concat_dim reader.kwargs["data"].url = hrefs reader.kwargs.update(kw) return reader.read() class StacSearch(BaseReader): """ Get stac objects matching a search spec from a STAC endpoint """ implements = {datatypes.STACJSON} imports = {"pystac"} output_instance = "intake.readers.entry:Catalog" def __init__(self, metadata=None, **kwargs): super().__init__(metadata=metadata, **kwargs) def _read(self, data, query=None, **kwargs): """ Parameters ---------- query: See https://pystac-client.readthedocs.io/en/latest/api.html#item-search for a description of the available fields kwargs: passed to the resulting readers, e.g., for auth information """ import requests from pystac import ItemCollection req = requests.post(data.url + "/search", json=query) out = req.json() cat = Catalog(metadata=self.metadata) items = ItemCollection.from_dict(out).items for subcatalog in items: subcls = type(subcatalog).__name__ cat[subcatalog.id] = ReaderDescription( reader=StacCatalogReader.qname(), kwargs=dict( **{"cls": subcls, "data": datatypes.Literal(subcatalog.to_dict())}, **kwargs, ), ) return cat class STACIndex(BaseReader): """Searches stacindex.org for known public STAC data sources""" implements = {datatypes.Service} output_instance = Catalog.qname() imports = {"pystac"} def _read(self, *args, **kwargs): with fsspec.open("https://stacindex.org/api/catalogs") as f: data = json.load(f) cat = Catalog() for entry in data: if entry["isPrivate"]: continue if entry["isApi"]: cat[entry["slug"]] = StacSearch( data=datatypes.STACJSON(entry["url"]), metadata={ "title": entry["title"], "description": entry["summary"], "created": entry["created"], "updated": entry["updated"], }, ) else: cat[entry["slug"]] = StacCatalogReader( data=datatypes.STACJSON(entry["url"]), metadata={ "title": entry["title"], "description": entry["summary"], "created": entry["created"], "updated": entry["updated"], }, ) return cat class THREDDSCatalog(Catalog): """A catalog provided by a THREDDS service This subclass exists, just so we can indicate the possibility of using the server's search endpoint and xarray collection """ class THREDDSCatalogReader(BaseReader): """ Read from THREDDS endpoint https://www.unidata.ucar.edu/software/tds/ """ implements = {datatypes.THREDDSCatalog} output_instance = THREDDSCatalog.qname() imports = {"siphon", "xarray"} def _read(self, data, make="both", **kwargs): """ Parameters ---------- data: Service URL endpoint make: "both" | "dap" | "cdf" For each iterm, make an openDAP reader, netCDF reader, or both. If both, the entries will have the same name, with _DAP and _CDF appended. """ from siphon.catalog import TDSCatalog from intake.readers.readers import XArrayDatasetReader thr = TDSCatalog(data.url) cat = THREDDSCatalog(metadata=thr.metadata) for r in thr.catalog_refs.values(): cat[r.title] = THREDDSCatalogReader(datatypes.THREDDSCatalog(url=r.href)) for ds in thr.datasets.values(): if make == "both": cat[ds.name + "_DAP"] = XArrayDatasetReader( datatypes.Service(ds.access_urls["OpenDAP"]), engine="pydap" ) cat[ds.name + "_CDF"] = XArrayDatasetReader( datatypes.HDF5(ds.access_urls["HTTPServer"]), engine="h5netcdf" ) elif make == "dap": cat[ds.name] = XArrayDatasetReader( datatypes.Service(ds.access_urls["OpenDAP"]), engine="pydap" ) elif make == "cdf": cat[ds.name] = XArrayDatasetReader( datatypes.HDF5(ds.access_urls["HTTPServer"]), engine="h5netcdf" ) else: raise ValueError("parameter `make` must be one of both|dap|cdf, but got '%s'", make) return cat class HuggingfaceHubCatalog(BaseReader): """ Datasets from HuggingfaceHub To actually access datasets which are not public, you will need to supply and appropriate token. By default, you will be able to read any public/example data. Examples -------- >>> hf = intake.readers.catalogs.HuggingfaceHubCatalog().read() >>> hf['acronym_identification'].read() DatasetDict({ train: Dataset({ features: ['id', 'tokens', 'labels'], num_rows: 14006 }) validation: Dataset({ features: ['id', 'tokens', 'labels'], num_rows: 1717 }) test: Dataset({ features: ['id', 'tokens', 'labels'], num_rows: 1750 }) }) """ output_instance = "intake.readers.entry:Catalog" imports = {"datasets"} func = "huggingface_hub:list_datasets" def _read(self, *args, with_community_datasets: bool = False, **kwargs): """ Parameters ---------- with_community_datasets: If False, only includes official public data, and retrieves fewer entries. """ from intake.readers.datatypes import HuggingfaceDataset from intake.readers.readers import HuggingfaceReader import huggingface_hub datasets = huggingface_hub.list_datasets(full=False) if not with_community_datasets: datasets = [dataset for dataset in datasets if "/" not in dataset.id] entries = [ HuggingfaceReader(data=HuggingfaceDataset(d.id, metadata=d.__dict__)) for d in datasets ] cat = Catalog(entries=entries) cat.aliases = {d.id: e for d, e in zip(datasets, cat.entries)} return cat class SKLearnExamplesCatalog(BaseReader): """ Example datasets from sklearn.datasets https://scikit-learn.org/stable/datasets/toy_dataset.html Each entry has some specific parameters, please read the linked page. Note that the metadata is only available in the final dataset after .read(), not before. Examples -------- >>> import intake.readers.catalogs >>> cat = intake.readers.catalogs.SKLearnExamplesCatalog().read() >>> list(cat) ['breast_cancer', 'diabetes', 'digits', ...] >>> cat.olivetti_faces.read() downloading Olivetti faces from https://ndownloader.figshare.com/files/5976027 to TMP {'data': array([[0.30991736, 0.3677686 , 0.41735536, ..., 0.15289256, 0.16115703, 0.1570248 ], [0.45454547, 0.47107437, 0.5123967 , ..., 0.15289256, 0.15289256, 0.15289256], [0.3181818 , 0.40082645, 0.49173555, ..., 0.14049587, 0.14876033, ... """ output_instance = "intake.readers.entry:Catalog" imports = {"sklearn"} def _read(self, **kw): from intake.readers.readers import SKLearnExampleReader import sklearn.datasets names = [funcname[5:] for funcname in dir(sklearn.datasets) if funcname.startswith("load_")] names.extend( [funcname[6:] for funcname in dir(sklearn.datasets) if funcname.startswith("fetch_")] ) entries = [SKLearnExampleReader(name=name) for name in names] cat = Catalog(entries=entries) cat.aliases = {name: e for name, e in zip(names, cat.entries)} return cat class TorchDatasetsCatalog(BaseReader): """ Standard example PyTorch datasets Includes all the entries in packages torchvision, torchaudio, torchtext . The types of these data when read are all torch.utils.data:Dataset, but the contents are quite different. Examples -------- >>> cat = intake.readers.catalogs.TorchDatasetsCatalog(rootdir="here").read() >>> cat.RTE.read() (ShardingFilterIterDataPipe, ShardingFilterIterDataPipe, ShardingFilterIterDataPipe) """ # TODO: the load function can be used for a wide variety of local files output_instance = "intake.readers.entry:Catalog" imports = {"datasets"} def _read(self, rootdir: str, *args, **kwargs): """ Parameters ---------- rootdir: A local directory to store cached files Some datasets require further kwargs, such as subset (e.g., "train") or other selector. """ from intake.readers.readers import TorchDataset import importlib cat = Catalog() for name in ("vision", "audio", "text"): try: mod = importlib.import_module(f"torch{name}") for func in mod.datasets.__all__: f = getattr(mod.datasets, func) metadata = ( {"description": f.__doc__.split("\n", 1)[0], "text": f.__doc__} if f.__doc__ else {} ) metadata["section"] = name cat[func] = TorchDataset( modname=name, funcname=func, rootdir=rootdir, metadata=metadata ) except (ImportError, ModuleNotFoundError): pass return cat class TensorFlowDatasetsCatalog(BaseReader): """ Datasets from the TensorFlow public registry See https://github.com/tensorflow/datasets/tree/master/tensorflow_datasets for full decriptions. Data will be cached locally on load, and metadata is only fetched at that time. Examples -------- >>> tf = intake.readers.catalogs.TensorFlowDatasetsCatalog().read( >>> tf.xnli.read() Downloading and preparin ... Dataset xnli downloaded and prepared to <>>. Subsequent calls will reuse this data. Out[13]: ({Split('test'): >> import intake.readers.catalogs >>> cat = intake.readers.catalogs.EarthdataCatalogReader(temporal=("2002-01-01", "2002-01-02")).read() >>> reader = cat['C2723754864-GES_DISC'] >>> reader.read() Dimensions: (time: 2, lon: 3600, lat: 1800, nv: 2) Coordinates: * lon (lon) float32 -179.9 -179.9 ... 179.9 179.9 * lat (lat) float64 -89.95 -89.85 ... 89.85 89.95 * time (time) datetime64[ns] 2002-01-01 2002-01-02 Dimensions without coordinates: nv Data variables: precipitation (time, lon, lat) float32 dask.array precipitation_cnt (time, lon, lat) int8 dask.array ... """ output_instance = "intake.readers.entry:Catalog" imports = {"earthdata", "xarray"} func = "earthaccess:search_datasets" def _read(self, temporal=("1980-01-01", "2023-11-10"), **kwargs): cat = Catalog() dss = self._func(temporal=temporal, cloud_hosted=True, **kwargs) for ds in dss: cat[ds["meta"]["concept-id"]] = ReaderDescription( reader="intake.readers.catalogs:EarthdataReader", metadata=dict(ds), kwargs=dict( concept=ds["meta"]["concept-id"], temporal=temporal, cloud_hosted=True, **kwargs ), ) return cat ================================================ FILE: intake/readers/convert.py ================================================ """Convert between python representations of data By convention, functions here do not change the data, just how it is held. """ from __future__ import annotations import copy import re from urllib.parse import urljoin from functools import lru_cache from itertools import chain from intake import import_name, conf from intake.readers.datatypes import OpenAIService from intake.readers import BaseData, BaseReader, readers, LlamaServerReader, OpenAIReader from intake.readers.utils import all_to_one, subclasses, safe_dict class ImportsProperty: """Builds the .imports attribute from classes in the .instances class attribute""" def __get__(self, obj, cls): # this asserts that ALL in and out types should be importable cls.imports = set( _.split(":", 1)[0].split(".", 1)[0] for _ in chain(cls.instances, cls.instances.values()) if _ is not SameType ) return cls.imports class BaseConverter(BaseReader): """Converts from one object type to another Most often, subclasses call a single function on the data, but arbitrary complex transforms are possible. This is designed to be one step in a Pipeline. .run() will be called on the output object from the previous stage, subclasses will wither override that, or just provide a func=. """ instances: dict[str, str] = {} #: mapping from input types to output types def run(self, x, *args, **kwargs): """Execute a conversion stage on the output object from another stage Subclasses may override this """ func = import_name(self.func) return func(x, *args, **kwargs) def _read(self, *args, data=None, **kwargs): """Read the data Subclasses may override this if they wish to interact with the upstream reader/pipeline. """ if data is None: data = args[0] args = args[1:] if isinstance(data, BaseReader): data = data.read() return self.run(data, *args, **kwargs) class GenericFunc(BaseConverter): """Call given arbitrary function This could be a transform or anything; the caller should specify what the output_instance will be, since the class doesn't know. """ def _read(self, *args, data=None, func=None, data_kwarg=None, **kwargs): if data is not None and isinstance(data, BaseReader): data = data.read() if data is not None: if data_kwarg is None: return func(data, *args, **kwargs) else: kwargs[data_kwarg] = data return func(*args, **kwargs) return func(*args, **kwargs) class SameType: """Used to indicate that the output of a transform is the same as the input, which is arbitrary""" class DuckToPandas(BaseConverter): instances = {"duckdb:DuckDBPyRelation": "pandas:DataFrame"} func = "duckdb:DuckDBPyConnection.df" def run(self, x, *args, with_arrow=True, **kwargs): if with_arrow: import pandas as pd table = x.to_arrow_table() data = { col: pd.Series(pd.arrays.ArrowExtensionArray(val)) for col, val in zip(table.column_names, table.columns) } return pd.DataFrame(data) else: return x.df() class PandasToDuck(BaseConverter): """Save content of pandas dataframe to Duck internal storage Value of ``table`` can be used to point to attached database or output file. """ instances = {"pandas:DataFrame": "duckdb:DuckDBPyRelation"} func = "duckdb:df" def run( self, x, table: str, conn: dict | str | None = None, *args, comment: str = None, overwrite=True, **kwargs, ): # TODO: more options like metadata import duckdb from intake.readers.datatypes import SQLQuery duck = readers.DuckSQL._duck(None, conn=conn) out = duckdb.df(x, connection=duck) duck.register(view_name="temp_view", python_object=out) duck.sql( f"CREATE {'OR REPLACE' if overwrite else ''} " f"TABLE '{table}' AS SELECT * FROM 'temp_view';" ) if comment is not None: # https://duckdb.org/docs/stable/sql/data_types/ # literal_types.html#escape-string-literals comment = str(comment).replace("'", "''") duck.sql(f"COMMENT ON TABLE '{table}' IS '{comment}';") out = readers.DuckSQL(SQLQuery(conn=conn, query=f"SELECT * FROM '{table}'")) return out class DaskDFToPandas(BaseConverter): instances = { "dask.dataframe:DataFrame": "pandas:DataFrame", "dask_geopandas.core:GeoDataFrame": "geopandas:GeoDataFrame", "dask.array:Array": "numpy:ndarray", } func = "dask:compute" def run(self, x, *args, **kwargs): return self._func(x)[0] class PandasToGeopandas(BaseConverter): instances = {"pandas:DataFrame": "geopandas:GeoDataFrame"} func = "geopandas:GeoDataFrame" class XarrayToPandas(BaseConverter): instances = {"xarray:Dataset": "pandas:DataFrame"} func = "xarray:Dataset.to_dataframe" class PandasToXarray(BaseConverter): instances = {"pandas:DataFrame": "xarray:Dataset"} func = "xarray:Dataset.from_dataframe" class ToHvPlot(BaseConverter): instances = all_to_one( { "pandas:DataFrame", "dask.dataframe:DataFrame", "xarray:Dataset", "xarray:DataArray", }, "holoviews.core.layout:Composable", ) func = "hvplot:hvPlot" def run(self, data, explorer: bool = False, **kw): """For tabular data only, pass explorer=True to get an interactive GUI""" import hvplot if explorer: # this is actually a hvplot.ui:hvPlotExplorer and only allows tabular data return hvplot.explorer(data, **kw) return hvplot.hvPlot(data, **kw)() class RayToPandas(BaseConverter): instances = {"ray.data:Dataset": "pandas:DataFrame"} func = "ray.data:Dataset.to_pandas" class PandasToRay(BaseConverter): instances = {"pandas:DataFrame": "ray.data:Dataset"} func = "ray.data:from_pandas" class RayToDask(BaseConverter): instances = {"ray.data:Dataset": "dask.dataframe:DataFrame"} func = "ray.data:Dataset.to_dask" class DaskToRay(BaseConverter): instances = {"dask.dataframe:DataFrame": "ray.data:Dataset"} func = "ray.data:from_dask" class HuggingfaceToRay(BaseConverter): instances = {"datasets.arrow_dataset:Dataset": "ray.data:Dataset"} func = "ray.data:from_huggingface" class TorchToRay(BaseConverter): instances = {"torch.utils.data:Dataset": "ray.data:Dataset"} func = "ray.data:from_torch" class SparkDFToRay(BaseConverter): instances = {"pyspark.sql:DataFrame": "ray.data:Dataset"} func = "ray.data:from_spark" class RayToSpark(BaseConverter): instances = {"ray.data:Dataset": "pyspark.sql:DataFrame"} func = "ray.data:Dataset.to_spark" class TiledNodeToCatalog(BaseConverter): instances = {"tiled.client.node:Node": "intake.readers.entry:Catalog"} def run(self, x, **kw): # eager creation of entries from a node from intake.readers.datatypes import TiledDataset, TiledService from intake.readers.entry import Catalog from intake.readers.readers import TiledClient, TiledNode cat = Catalog() for k, client in x.items(): if type(client).__name__ == "Node": data = TiledService(url=client.uri) reader = TiledNode(data=data, metadata=client.item) cat[k] = reader else: data = TiledDataset(url=client.uri) reader = TiledClient( data, output_instance=f"{type(client).__module__}:{type(client).__name__}", metadata=client.item, ) cat[k] = reader return cat class TiledSearch(BaseConverter): """See https://blueskyproject.io/tiled/tutorials/search.html""" instances = {"tiled.client.node:Node": "tiled.client.node:Node"} def run(self, x, *arg, **kw): # TODO: expects instances of classes in tiled.queries, which must be pickled, but # could allow (name, args) or something else return x.search(*arg, **kw) class TileDBToNumpy(BaseConverter): instances = {"tiledb.libtiledb:Array": "numpy:ndarray"} def run(self, x, *args, **kwargs): # allow attribute selection here for when it wasn't included at read time? return x[:] class TileDBToPandas(BaseConverter): """Implemented only if an attribute was not already chosen.""" instances = {"tiledb.libtiledb:Array": "pandas:DataFrame"} func = "tiledb.libtiledb:Array.df" def run(self, x, *args, **kwargs): return x.df[:] class DaskArrayToTileDB(BaseConverter): # this is like output, and could return a datatypes.TileDB instead instances = {"dask.array:Array": "tiledb.libtiledb:Array"} func = "dask.array:to_tiledb" def run(self, x, uri, **kwargs): return self._func(x, uri, return_stored=True, **kwargs) class NumpyToTileDB(BaseConverter): # this could be considered an output converter, giving a datatypes.TileDB # instead of the array instance # alternatively, a datatypes.TileDB could be the *input* to the function instances = {"numpy:ndarray": "tiledb.libtiledb:Array"} func = "tiledb:from_numpy" def run(self, x, uri, **kwargs): return self._func(uri, x, **kwargs) class DeltaQueryToDask(BaseConverter): instances = {"deltalake:DeltaTable": "dask.dataframe:DataFrame"} func = "deltalake:DeltaTable.file_uris" def _read(self, reader, query, *args, **kwargs): import dask.dataframe as dd file_uris = reader.read().file_uris(query) return dd.read_parquet(file_uris, storage_options=reader.kwargs["data"].storage_options) class DeltaQueryToDaskGeopandas(BaseConverter): instances = {"deltalake:DeltaTable": "dask_geopandas:GeoDataFrame"} func = "deltalake:DeltaTable.file_uris" def _read(self, reader, query, *args, **kwargs): import dask_geopandas file_uris = reader.read().file_uris(query) return dask_geopandas.read_parquet( file_uris, storage_options=reader.kwargs["data"].storage_options ) class GeoDataFrameToSTACCatalog(BaseConverter): instances = {"geopandas:GeoDataFrame": "intake.readers.entry:Catalog"} func = "intake.readers.catalogs:StacCatalogReader" @classmethod def _un_arr(cls, data): # clean up dataframe import numpy as np if isinstance(data, dict): data = {k: cls._un_arr(v) for k, v in data.items()} elif isinstance(data, (list, np.ndarray)): data = [cls._un_arr(_) for _ in data] return data def read(self, data, *args, **kwargs): from intake.readers import Literal from intake.readers.catalogs import StacCatalogReader import stac_geoparquet # clean up numpy arrays->list and any assets that are just None data["assets"] = data.assets.apply( lambda x: {k: v for k, v in self._un_arr(x).items() if v} ) stac = stac_geoparquet.stac_geoparquet.to_item_collection(data) lit = Literal(stac.to_dict()) return StacCatalogReader( lit, signer=self.metadata.get("signer"), prefer=self.metadata.get("prefer"), cls="ItemCollection", metadata=self.metadata, ).read() class PandasToMetagraph(BaseConverter): instances = {"pd:DataFrame": "metagraph.wrappers.EdgeSet:PandasEdgeSet"} func = "metagraph.wrappers.EdgeSet:PandasEdgeSet" class NibabelToNumpy(BaseConverter): instances = {"nibabel.spatialimages:SpatialImage": "numpy:ndimage"} func = "nibabel.spatialimages:SpatialImage.get_fdata" class DicomToNumpy(BaseConverter): instances = {"pydicom.dataset:FileDataset": "numpy:ndarray"} func = "pydicom.dataset:FileDataset.pixel_array" def run(self, x, *args, **kwargs): return x.pixel_array class FITSToNumpy(BaseConverter): instances = {"astropy.io.fits:HDUList": "numpy:ndarray"} func = "astropy.io.fits:FitsHDU.data" def run(self, x, extension=None): """Get the array data of one FITS extension If hdu is None, find first extension containing data. """ if extension is None: found = False for extension, hdu in enumerate(x): if hdu.header.get("NAXIS", 0) > 0: found = True break if not found: raise ValueError("No data extensions") return x[extension].data class ASDFToNumpy(BaseConverter): instances = {"asdf:AsdfFile": "numpy:ndarray"} func = "asdf:AsdfFile." def run(self, x, tree_path: str | list[str], **kwargs): if isinstance(tree_path, str): tree_path = tree_path.split(".") for p in tree_path: x = x[p] return x[:] class PolarsLazy(BaseConverter): instances = {"polars:DataFrame": "polars:LazyFrame"} func = "polars:DataFrame.lazy" class PolarsEager(BaseConverter): instances = {"polars:LazyFrame": "polars:DataFrame"} func = "polars:LazyFrame.collect" # collect_async() ? class PolarsToPandas(BaseConverter): instances = {"polars:DataFrame": "pandas:DataFrame"} func = "polars:DataFrame.to_pandas" def run(self, x, *args, **kwargs): return x.to_pandas(*args, **kwargs) class PandasToPolars(BaseConverter): instances = {"pandas:DataFrame": "polars:DataFrame"} func = "polars:from_pandas" class DataFrameToMetadata(BaseConverter): instances = all_to_one( ["pandas:DataFrame", "dask.dataframe:DataFrame", "polars:DataFrame"], "builtins:dict" ) def run(self, x, *args, **kwargs): out = {"repr": repr(x), "shape": x.shape} # cf Repr, the output converter t = str(type(x)).lower() # TODO: perhaps can split this class into several # TODO: implement spark, daft, modin, ibis ... # Note that FileSizeReader can give file size on disk (if origin is files) if "pandas" in t: out["memory"] = x.memory_usage(deep=True).sum() out["schema"] = x.dtypes if hasattr(x, "dtypes") else x.dtype out["shape"] = x.shape elif "polars" in t: out["memory"] = x.estimated_size() out["shape"] = x.shape out["schema"] = x.schema elif "ray" in t: out["memory"] = x.size_bytes() out["shape"] = [x.count(), len(x.columns)] out["schema"] = safe_dict(x.schema) return safe_dict(out) class GGUFToLlamaCPPService(BaseConverter): instances = {"intake.readers.datatypes:GGUF": "intake.readers.datatypes:LlamaCPPService"} def run(self, x, **kwargs): return LlamaServerReader(x).read(**kwargs) class LLamaCPPServiceToOpenAIService(BaseConverter): instances = { "intake.readers.datatypes:LlamaCPPService": "intake.readers.datatypes:OpenAIService" } def run(self, x, options=None): url = urljoin(x.url, "/v1") service = OpenAIService(url=url, key="none", options=options) return service class OpenAIServiceToOpenAIClient(BaseConverter): instances = {"intake.readers.datatypes:OpenAIService": "openai:OpenAI"} def run(self, x): return OpenAIReader(x).read() def convert_class(data, out_type: str): """Get conversion class from given data to out_type This works on concrete data, not a datatype or reader instance. It returns the first match. out_type will match on regex, e.g., "pandas" would match "pandas:DataFrame" """ package = type(data).__module__.split(".", 1)[0] for cls in subclasses(BaseConverter): for intype, outtype in cls.instances.items(): if not re.findall(out_type, outtype): continue if intype.split(".", 1)[0] != package: continue thing = readers.import_name(intype) if isinstance(data, thing): return cls raise ValueError("Converter not found") def convert_classes(in_type: str): """Get available conversion classes for input type""" out_dict = [] package = in_type.split(":", 1)[0].split(".", 1)[0] for cls in subclasses(BaseConverter): for intype, outtype in cls.instances.items(): if "*" not in intype and intype.split(":", 1)[0].split(".", 1)[0] != package: continue if re.findall(intype.lower(), in_type.lower()) or re.findall( in_type.lower(), intype.lower() ): if outtype == SameType: outtype = intype out_dict.append((outtype, cls)) return out_dict class Pipeline(readers.BaseReader): """Holds a list of transforms/conversions to be enacted in sequence A transform on a pipeline makes a new pipeline with that transform added to the sequence of operations. """ from intake.readers.readers import BaseReader def __init__( self, steps: list[tuple[BaseReader, tuple, dict]], out_instances: list[str], output_instance=None, metadata=None, **kwargs, ): self.output_instances = [] prev = out_instances[0] for inst in out_instances: if inst is SameType: inst = prev prev = inst self.output_instances.append(inst) super().__init__( output_instance=output_instance or self.output_instances[-1], metadata=metadata, steps=steps, out_instances=self.output_instances, ) steps[-1][2].update(kwargs) @property def steps(self): return self.kwargs["steps"] def __call__(self, *args, **kwargs): return super().__call__( *args, steps=self.steps, out_instances=self.output_instances, metadata=self.metadata, **kwargs, ) def __repr__(self): start = "PipelineReader: \n" bits = [ f" {i}: {f.qname() if isinstance(f, BaseReader) else f.__name__}, {args} {kw} => {out}" for i, ((f, args, kw), out) in enumerate(zip(self.steps, self.output_instances)) ] return "\n".join([start] + bits) def output_doc(self): from intake import import_name out = import_name(self.output_instance) return out.__doc__ def doc(self): return self.doc_n(-1) def doc_n(self, n): """Documentation for the Nth step""" return self.steps[n][0].doc() def _read_stage_n(self, stage, discover=False, **kwargs): from intake.readers.readers import BaseReader func, arg, kw = self.steps[stage] kw2 = kw.copy() kw2.update(kwargs) for k, v in kw.items(): if isinstance(v, BaseReader): kw2[k] = v.read() else: kw2[k] = v arg = kw2.pop("args", arg) # TODO: these conditions can probably be combined if isinstance(func, type) and issubclass(func, BaseReader): if discover: return func(metadata=self.metadata).discover(*arg, **kw2) else: return func(metadata=self.metadata).read(*arg, **kw2) elif isinstance(func, BaseReader): if discover: return func.discover(*arg, **kw2) else: return func.read(*arg, **kw2) else: return func(*arg, **kw2) def _read(self, discover=False, **kwargs): data = None for i, step in enumerate(self.steps): kw = kwargs if i == len(self.steps) else {} if i: data = self._read_stage_n(i, data=data, **kw) else: data = self._read_stage_n(i, discover=discover, **kw) return data def apply(self, func, *arg, output_instance=None, **kwargs): """Add a pipeline stage applying function to the pipeline output so far""" from intake.readers.convert import GenericFunc kwargs["func"] = func return self.with_step((GenericFunc, arg, kwargs), output_instance or self.output_instance) def first_n_stages(self, n: int): """Truncate pipeline to the given stage If n is equal to the number of steps, this is a simple copy. """ # TODO: allow n=0 to get the basic reader? if n < 1 or n > len(self.steps): raise ValueError(f"n must be between {1} and {len(self.steps)}") kw = self.kwargs.copy() kw.update( dict( steps=self.steps[:n], out_instances=self.output_instances[:n], metadata=self.metadata, ) ) pipe = Pipeline( **kw, ) if n < len(self.steps): pipe._tok = (self.token, n) return pipe def discover(self, **kwargs): return self.read(discover=True) def with_step(self, step, out_instance): """A new pipeline like this one but with one more step""" if not isinstance(step, tuple): # must be a func - check? step = (step, (), {}) return Pipeline( steps=self.steps + [step], out_instances=self.output_instances + [out_instance], metadata=self.metadata, ) def read_stepwise(self, breakpoint=0): """Read with a wrapper class to allow executing one step at a time Parameters ---------- breakpoint: int At which stage of the pipeline to enter stepwise mode """ return PipelineExecution(self, breakpoint=breakpoint) class PipelineExecution: """Encapsulates a Pipeline, so you can step through it stepwise Interesting attributes to examine: - .data, the result of the most recent step (initially None) - .next, (i, step) the next step to perform. This is a copy, so you can edit the kwargs in-place without changing the original """ def __init__(self, pipeline, breakpoint=0): self.pipeline = pipeline self.data = None self.steps = iter(enumerate(pipeline.steps)) self.next = copy.copy(next(self.steps)) for _ in range(breakpoint): self.step() def __repr__(self): return f"Executing stage {self.next[0]} of pipeline\n{self.pipeline}" def cont(self): """Continue pipeline to the end without stopping again""" while True: out = self.step() if out is not self: return out def step(self, **kw): """Run one step of the pipeline If it is the last step, will return the result; otherwise will return self. """ i, step = self.next if i: self.data = self.pipeline._read_stage_n(i, data=self.data, **kw) else: self.data = self.pipeline._read_stage_n(i, **kw) try: self.next = next(self.steps) return self except StopIteration: return self.data def conversions_graph(avoid=None, allow_wildcard=True): avoid = avoid or conf["reader_avoid"] if isinstance(avoid, str): avoid = [avoid] import networkx graph = networkx.DiGraph() # transformers nodes = set( cls.output_instance for cls in subclasses(readers.BaseReader) if cls.output_instance and not any(re.findall(_.lower(), cls.qname().lower()) for _ in avoid) ) graph.add_nodes_from(nodes) for cls in subclasses(readers.BaseReader): if any(re.findall(_.lower(), cls.qname().lower()) for _ in avoid): continue if cls.output_instance: for impl in cls.implements: graph.add_node(cls.output_instance) graph.add_edge(impl.qname(), cls.output_instance, label=cls.qname()) for cls in subclasses(BaseConverter): if any(re.findall(_.lower(), cls.qname().lower()) for _ in avoid): continue for inttype, outtype in cls.instances.items(): if ( isinstance(outtype, str) and inttype != outtype and (allow_wildcard or "*" not in inttype) ): graph.add_nodes_from((inttype, outtype)) graph.add_edge(inttype, outtype, label=cls.qname()) return graph def plot_conversion_graph(filename) -> None: # TODO: return a PNG datatype or something else? import networkx as nx g = conversions_graph(allow_wildcard=False) a = nx.nx_agraph.to_agraph(g) # requires pygraphviz a.draw(filename, prog="fdp") @lru_cache() # clear cache if you import more things def path( start: str, end: str | tuple[str], cutoff: int = 5, avoid: tuple[str] | None = None ) -> list[list]: """Find possible conversion paths from start to end types Parameters ---------- start: data or reader qualified name to start with end: desired output type name; any match on any of the strings given cutoff: the maximum numer of steps to consider per path avoid: ignore all readers/converters with a name matching this Returns ------- A list of paths, where each item is a list of steps (starttype, endtype) for which there is a conversion class. """ import networkx as nx g = conversions_graph(avoid=avoid) alltypes = list(g) matchtypes = [_ for _ in alltypes if re.findall(start, _)] if not matchtypes: raise ValueError("type found no match: %s", start) start = matchtypes[0] if isinstance(end, str): end = (end,) matchtypes = [_ for _ in alltypes if any(re.findall(e, _) for e in end)] if not matchtypes: raise ValueError("outtype found no match: %s", end) end = matchtypes[0] return sorted(nx.all_simple_edge_paths(g, start, end, cutoff=cutoff), key=len) def auto_pipeline( url: str | BaseData, outtype: str | tuple[str] = "", storage_options: dict | None = None, avoid: list[str] | None = None, ) -> Pipeline: """Create pipeline from given URL to desired output type Will search for the shortest conversion path from the inferred data-type to the output. Parameters ---------- url: input data, usually a location/URL, but maybe a data instance outtype: pattern to match to possible output types storage_options: if url is a remote str, these are kwargs that fsspec may need to access it avoid: don't consider readers whose names match any of these strings """ from intake.readers.datatypes import recommend if isinstance(url, str): if storage_options: data = recommend(url, storage_options=storage_options)[0]( url=url, storage_options=storage_options ) else: data = recommend(url)[0](url=url) else: data = url if isinstance(data, BaseData): start = data.qname() steps = path(start, outtype, avoid=avoid) reader = data.to_reader(outtype=steps[0][0][1] if steps else outtype) if steps: for s in steps[0][1:]: reader = reader.transform[s[1]] elif isinstance(data, BaseReader): reader = data steps = path(data.output_instance, outtype, avoid=avoid) for s in steps[0]: reader = reader.transform[s[1]] return reader ================================================ FILE: intake/readers/datatypes.py ================================================ """Enumerates all the sorts of data that Intake knows about""" from __future__ import annotations import re from itertools import chain from functools import lru_cache as cache from typing import Any, Optional import fsspec from intake import import_name from intake.readers.utils import Tokenizable, subclasses # TODO: make "structure" possibilities an enum? # https://en.wikipedia.org/wiki/List_of_file_signatures class BaseData(Tokenizable): """Prototype dataset definition""" mimetypes: str = "" #: regex, MIME pattern to match filepattern: str = "" #: regex, file URLs to match; empty if relying on magic or contains structure: set[str] = set() #: informational tags for nature of data, e.g., "array" magic: set[ bytes | tuple ] = set() #: binary patterns, usually at the file head; each item identifies this data type contains: set[ str ] = set() #: if using a directory URL, an ls() on that path will contain these things def __init__(self, metadata: dict[str, Any] | None = None): self.metadata: dict[str, Any] = metadata or {} # arbitrary information @classmethod @cache def _filepattern(cls): return re.compile(cls.filepattern) @classmethod @cache def _mimetypes(cls): return re.compile(cls.mimetypes) @property def possible_readers(self): """List of reader classes for this type, grouped by importability""" from intake.readers.readers import recommend return recommend(self) @property def possible_outputs(self): """Map of importable readers to the expected output class of each""" readers = self.possible_readers["importable"] return {r: r.output_instance for r in readers} def to_reader_cls( self, outtype: tuple[str] | str | None = None, reader: tuple[str] | str | type | None = None ): if outtype and reader: raise ValueError if isinstance(reader, str): # exact match (no lowering) try: return import_name(reader) except (ImportError, ModuleNotFoundError): reader = (reader,) if isinstance(reader, tuple): for cls, out in self.possible_outputs.items(): # there shouldn't be many of these if any(re.findall(r.lower(), cls.qname().lower()) for r in reader): return cls if isinstance(reader, type): return reader elif outtype: if isinstance(outtype, str): outtype = (outtype,) for reader, out in self.possible_outputs.items(): # there shouldn't be many of these if out is not None and any( out == _ or re.findall(_.lower(), out.lower()) for _ in outtype ): return reader return next(iter(self.possible_readers["importable"])) def to_reader(self, outtype: str | None = None, reader: str | None = None, **kw): """Find an appropriate reader for this data If neither ``outtype`` or ``reader`` is passed, the first importable reader will be picked. See also ``.possible_outputs`` Parameters ---------- outtype: string to match against the output classes of potential readers reader: string to match against the class names of the readers """ return self.to_reader_cls(outtype, reader)(data=self, **kw) def to_entry(self): """Create DataDescription version of this, for placing in a Catalog""" from intake.readers.entry import DataDescription kw = {k: v for k, v in self.__dict__.items() if not k.startswith("_")} kw.pop("metadata") # this is always passed separately return DataDescription(datatype=self.qname(), kwargs=kw, metadata=self.metadata) def __repr__(self): d = {k: v for k, v in self.__dict__.items() if not k.startswith("_")} return f"{type(self).__name__}, {d}" def auto_pipeline(self, outtype: str | tuple[str]): """Find a pipeline to transform from this to the given output type""" from intake.readers.convert import auto_pipeline return auto_pipeline(self, outtype) class FileData(BaseData): """Datatypes loaded from files, local or remote""" def __init__(self, url, storage_options: dict | None = None, metadata: dict | None = None): self.url = url #: location of the file(s), should be str or list[str] self.storage_options = storage_options #: kwargs for a backend storage system super().__init__(metadata) class Service(BaseData): """Datatypes loaded from some service""" def __init__(self, url, options=None, metadata=None): self.url = url self.options = options or {} super().__init__(metadata=metadata) class Catalog(BaseData): """Datatypes that are groupings of other data""" structure = {"catalog"} class PMTiles(FileData): """single-file archive format for tiled image data""" filepattern = "pmtiles" magic = {b"PMTiles"} structure = {"image"} class DuckDB(FileData): """Columnar table DB format used exclusively by duckdb""" filepattern = "(duck?)db" magic = {(8, b"DUCK")} structure = {"table"} class Parquet(FileData): """Column-optimized binary format""" filepattern = "(parq|parquet)" mimetypes = "application/(vnd.apache.parquet|parquet|x-parquet)/" structure = {"table", "nested"} magic = {b"PAR1"} contains = {"_metadata", "parq", "parquet"} # a directory can be a dataset class CSV(FileData): """Human-readable tabular format, Comma Separated Values""" filepattern = "(csv$|txt$|tsv$)" mimetypes = "(text/csv|application/csv|application/vnd.ms-excel)" structure = {"table"} class CSVPattern(CSV): """Specialised version of CSV, with a path containing capturing fields Characteristically contains python-style format groups with {..} """ filepattern = ".*[{].*[}].*(csv$|txt$|tsv$)" class Text(FileData): """Any text file""" filepattern = "(txt$|text$|dat$|ascii$)" mimetypes = "text/.*" structure = {"sequence"} # some optional byte order marks # https://en.wikipedia.org/wiki/Byte_order_mark#Byte_order_marks_by_encoding magic = {b"\xEF\xBB\xBF", b"\xFE\xFF", b"\xFF\xFE", b"\x00\x00\xFE\xFF"} class XML(FileData): """Extensible Markup Language file""" filepattern = "xml[sx]?$" mimetypes = "(application|text)/xml" structure = {"nested"} magic = {b"Tiled")} class TiledDataset(Service): """Data access service for data-aware portals and data science tools""" structure = {"array", "table", "nested"} class TileDB(Service): """Service exposing versioned, chunked and potentially sparse arrays""" filepattern = "tiled://" # or a real URL, local or remote contains = {"__meta", "__schema"} structure = {"array", "table"} class IcebergDataset(JSONFile): """Indexed set of parquet files with servioning and diffs""" structure = {"tabular"} magic = {(None, b'"format-version":')} class DeltalakeTable(FileData): """Indexed set of parquet files with servioning and diffs""" # a directory by convention, but otherwise can't be distinguished contains = {"_delta_log"} structure = {"tabular"} class NumpyFile(FileData): """Simple array format""" # will also match .npz since it will be recognised as a ZIP archive magic = {b"\x93NUMPY"} filepattern = "(npy$|text$)" structure = {"array"} class RawBuffer(FileData): """A C or FORTRAN N-dimensional array buffer without metadata""" filepattern = "raw$" structure = {"array"} def __init__( self, url: str, dtype: str, storage_options: dict | None = None, metadata: dict | None = None, ): super().__init__(url, storage_options=storage_options, metadata=metadata) self.dtype = dtype # numpy-style class Literal(BaseData): """A value that can be embedded directly to YAML (text, dict, list)""" def __init__(self, data, metadata=None): self.data = data super().__init__(metadata=metadata) class Handle(JSONFile): """An identifier registered on handle registry See https://handle.net/ . May refer to a single file or a set of files """ filepattern = "hdl:" class Feather2(FileData): """Tabular format based on Arrow IPC""" magic = {b"ARROW1"} structure = {"tabular", "nested"} class Feather1(FileData): """Deprecated tabular format from the Arrow project""" magic = {b"FEA1"} structure = {"tabular", "nested"} class PythonSourceCode(FileData): """Source code file""" structure = {"code"} filepattern = "py$" class GDALRasterFile(FileData): """One of the filetpes at https://gdal.org/drivers/raster/index.html This class overlaps with some other types, so only use when necessary. These must be local paths or use GDAL's own virtual file system. """ structure = {"array"} class GDALVectorFile(FileData): """One of the filetypes at https://gdal.org/drivers/vector/index.html This class overlaps with some other types, so only use when necessary. These must be local paths or use GDAL's own virtual file system. """ structure = { "nested", "tabular", } # tabular when read by geopandas, could be called a conversion class HuggingfaceDataset(BaseData): """https://github.com/huggingface/datasets""" structure = {"nested", "text"} def __init__(self, name, split=None, metadata=None): super().__init__(metadata) self.name = name self.split = split class TFRecord(FileData): """Tensorflow record file, ready for machine learning""" structure = {"nested"} filepattern = "tfrec$" class KerasModel(FileData): """Keras model parameter set""" structure = {"model"} # complex filepattern = "pb$" # possibly protobuf class GGUF(FileData): """Trained model (see https://github.com/ggerganov/ggml/blob/master/docs/gguf.md)""" structure = {"model"} filepattern = "gguf$" magic = {b"GGUF"} class SafeTensors(FileData): """Trained model (see https://github.com/huggingface/safetensors?tab=readme-ov-file#format) """ # TODO: .bin sees to be an older pytorch-specific version of this structure = {"model"} filepattern = "safetensors$" magic = {(8, b"{")} class PickleFile(FileData): """Python pickle, arbitrary serialized object""" structure = set() class ModelConfig(FileData): """HuggingFace-style multi-file model directory Looks like a catalog of related models """ structure = {"model"} filepattern = "config.json" magic = {b'"model_type":'} class SKLearnPickleModel(PickleFile): """Trained model made by sklearn and saved as pickle""" comp_magic = { # These are a bit like datatypes making raw bytes/file object output (0, b"\x1f\x8b"): "gzip", (0, b"BZh"): "bzip2", (0, b"(\xc2\xb5/\xc3\xbd"): "zstd", (0, b"\xff\x06\x00\x00sNaPpY"): "sz", # stream framed format } container_magic = { # these are like datatypes making filesystems (257, b"ustar"): "tar", (0, b"PK"): "zip", } def recommend( url: str | None = None, mime: str | None = None, head: bool = True, contents: bool = False, storage_options=None, ignore: set[str] | None = None, ) -> set[BaseData]: """Show which Intake data types can apply to the given details Parameters ---------- url: str Location of data mime: str MIME type, usually "x/y" form head: bytes | bool | None A small number of bytes from the file head, for seeking magic bytes. If it is True, fetch these bytes from th given URL/storage_options and use them. If None, only fetch bytes if there is no match by mime type or path, if False, don't fetch at all. contents: bool | None Attempt to delve into URL to analyse constituent files. This can significantly slow your recommendation. storage_options: dict | None If passing a URL which might be a remote file, storage_options can be used by fsspec. ignore: set | None Don't include these in the output Returns ------- set of matching datatype classes. """ # TODO: more complex returns defining which type of match hit what, or some kind of score outs = ignore or set() out = [] if isinstance(url, (list, tuple)): url = url[0] if head is True and url: try: fs, url2 = fsspec.core.url_to_fs(url, **(storage_options or {})) mime = mime or fs.info(url2, refresh=True).get("ContentType", None) except (IOError, TypeError, AttributeError, ValueError): mime = mime or None try: fs, url2 = fsspec.core.url_to_fs(url, **(storage_options or {})) head = fs.cat_file(url2[0] if isinstance(url2, list) else url2, end=2**20) except (IOError, IndexError, ValueError): head = False else: fs = None if isinstance(head, bytes): # more specific first for cls in subclasses(BaseData): if cls in outs: continue for m in cls.magic: if isinstance(m, tuple): off, m = m if off is None: if re.findall(m, head): out.append(cls) outs.add(cls) break else: off = 0 if off is not None and head[off:].startswith(m): out.append(cls) outs.add(cls) break if mime: mime = mime.lower() for cls in subclasses(BaseData): if cls not in outs and cls.mimetypes and re.match(cls._mimetypes(), mime): out.append(cls) outs.add(cls) if url: poss = {} if fs is not None and fs.isdir(url): try: allfiles = fs.ls(url, detail=False) except IOError: allfiles = None else: allfiles = None files = set(subclasses(FileData)) bases = set(subclasses(BaseData)) - files # file types first, then other/services, more specific first for cls in chain(files, bases): if cls in outs: continue if cls.filepattern: find = re.search(cls._filepattern(), url.lower()) if find and not allfiles: # not a directory or empty directory poss[cls] = find.start() if cls.contains and allfiles: if any(re.search(c, a) for c in cls.contains for a in allfiles): poss[cls] = 0 out.extend(sorted(poss, key=lambda x: poss[x])) if contents and url: for ext in {".gz", ".gzip", ".bzip2", "bz2", ".zstd", ".tar", ".tgz"}: if url.endswith(ext): out.extend(recommend(url[: -len(ext)], head=False, ignore=outs)) if out: return out if head is None and url: return recommend(url, mime=mime, head=True, storage_options=storage_options) if isinstance(head, bytes): for (off, mag), comp in comp_magic.items(): if head[off:].startswith(mag): storage_options = (storage_options or {}).copy() storage_options["compression"] = comp out = recommend(url, storage_options=storage_options) if out: print("Update storage_options: ", storage_options) return out for (off, mag), comp in container_magic.items(): if head[off:].startswith(mag): prot = fsspec.core.split_protocol(url)[0] out = recommend(f"{comp}://*::{url}", storage_options={prot: storage_options}) if out: print("Update url: ", url, "\nstorage_options: ", storage_options) return out # TODO: if directory, look inside files? return [] ================================================ FILE: intake/readers/entry.py ================================================ """Description of the ways to load a data set These are the definitions as they would appear in a Catalog: they may have (unevaluated) user parameters and references to one-another, as well as other templated values. The rule is: when placing an entry in a catalog, it is converted to its constituent data and reader descriptions. When accessing an entry, it is re-instantiated as a reader. The entries within a catalog can be amended in-place (such as extracting user parameters or templating) and persisted as catalog files. """ from __future__ import annotations from collections.abc import Mapping from copy import copy from itertools import chain import re from typing import Any, Iterable import fsspec import yaml from intake import import_name from intake.readers.user_parameters import ( BaseUserParameter, SimpleUserParameter, set_values, ) from intake.readers.utils import ( Tokenizable, check_imports, extract_by_path, extract_by_value, merge_dicts, _is_tok, ) class DataDescription(Tokenizable): """Defines some data: class and arguments. This may be laoded in a number of ways A DataDescription normally resides in a Catalog, and can contain templated arguments. When there are user_parameters, these will also be applied to any reader that depends on this data. """ def __init__( self, datatype: str, kwargs: dict = None, metadata: dict = None, user_parameters: dict = None, ): self.datatype = datatype self.kwargs = kwargs or {} self.metadata = metadata or {} self.user_parameters = user_parameters or {} def __repr__(self): part = f"DataDescription type {self.datatype}\n kwargs {self.kwargs}" if self.user_parameters: part += f"\n parameters {self.user_parameters}" return part def to_data(self, user_parameters=None, **kwargs): cls = import_name(self.datatype) ups = self.user_parameters.copy() ups.update(user_parameters or {}) kw = self.get_kwargs(user_parameters=ups, **kwargs) return cls(**kw, metadata=self.metadata) def __call__(self, **kwargs): return self.to_data(**kwargs) def get_kwargs( self, user_parameters: dict[str | BaseUserParameter] | None = None, **kwargs ) -> dict[str, Any]: """Get set of kwargs for given reader, based on prescription, new args and user parameters Here, `user_parameters` is intended to come from the containing catalog. To provide values for a user parameter, include it by name in kwargs """ kw = self.kwargs.copy() kw.update(kwargs) up = self.user_parameters.copy() up.update(user_parameters or {}) kw = set_values(up, kw) return kw def extract_parameter( self, name: str, path: str | None = None, value: Any = None, cls: type = SimpleUserParameter, **kw, ): if not ((path is None) ^ (value is None)): raise ValueError if path is not None: kw, up = extract_by_path(path, cls, name, self.kwargs, **kw) else: kw, up = extract_by_value(value, cls, name, self.kwargs, **kw) self.kwargs = kw self.user_parameters[name] = up class ReaderDescription(Tokenizable): """ A serialisable description of a reader or pipeline This class is typically stored inside Catalogs, and can contain templated arguments which get evaluated at the time that it is accessed from a Catalog. """ def __init__( self, reader: str, kwargs: dict[str, Any] | None = None, user_parameters: dict[str | BaseUserParameter] | None = None, metadata: dict | None = None, output_instance: str | None = None, ): self.reader = reader self.kwargs = kwargs or dict[str, Any]() self.output_instance = output_instance self.user_parameters: dict[str | BaseUserParameter] = user_parameters or {} self.metadata = metadata or {} def check_imports(self): """Are the packages listed in the "imports" key of the metadata available?""" cls = import_name(self.reader) class_import = cls.check_imports() meta_imports = check_imports(*self.metadata.get("imports", set())) return class_import & meta_imports def get_kwargs(self, user_parameters=None, **kwargs) -> dict[str, Any]: """Get set of kwargs for given reader, based on prescription, new args and user parameters Here, `user_parameters` is intended to come from the containing catalog. To provide values for a user parameter, include it by name in kwargs """ kw = self.kwargs.copy() user_parameters = user_parameters or {} kw.update(kwargs) # make data instance up = user_parameters or {} # global/catalog if "data" in kw and isinstance(kw["data"], DataDescription): extra = kw["data"] up.update(kw["data"].user_parameters) kw_subset = {k: v for k, v in kwargs.items() if k in user_parameters or k in extra} kw["data"] = kw["data"].to_data(user_parameters=user_parameters, **kw_subset) else: extra = {} kw["output_instance"] = self.output_instance # now make reader up.update(user_parameters) kw = set_values(up, kw) return kw def extract_parameter(self, name: str, path=None, value=None, cls=SimpleUserParameter, **kw): """Creates new version of the description Creates new instance, since the token will in general change """ if not ((path is None) ^ (value is None)): raise ValueError if path is not None: kw, up = extract_by_path(path, cls, name, self.kwargs, **kw) else: kw, up = extract_by_value(value, cls, name, self.kwargs, **kw) self.kwargs = kw self.user_parameters[name] = up def to_reader(self, user_parameters=None, **kwargs): cls = import_name(self.reader) if "data" in kwargs and isinstance(kwargs["data"], DataDescription): # if not, is already a BaseData ups = kwargs["data"].user_parameters.copy() else: ups = {} ups.update(self.user_parameters) ups.update(user_parameters or {}) kw = self.get_kwargs(user_parameters=ups, **kwargs) return cls(metadata=self.metadata, **kw) def to_cat(self, name=None): """Create a Catalog containing only this entry""" cat = Catalog() cat.add_entry(self, name) return cat def __call__(self, user_parameters=None, **kwargs): return self.to_reader(user_parameters=user_parameters, **kwargs) @classmethod def from_dict(cls, data): # note that there should never be any embedded intake classes in kwargs, as they get pulled out # when any reader is added to to a catalog obj = super().from_dict(data) obj.user_parameters = { k: BaseUserParameter.from_dict(v) for k, v in data["user_parameters"].items() } return obj def __repr__(self): extra = f"\n parameters: {self.user_parameters}" if self.user_parameters else "" return ( f"Entry for reader: {self.reader}\n kwargs: {self.kwargs}\n" f" producing: {self.output_instance}" + extra ) class Catalog(Tokenizable): """A collection of data and reader descriptions.""" def __init__( self, entries: Iterable[ReaderDescription] | Mapping | None = None, aliases: dict[str, int] | None = None, data: Iterable[DataDescription] | Mapping = None, user_parameters: dict[str, BaseUserParameter] | None = None, parameter_overrides: dict[str, Any] | None = None, metadata: dict | None = None, ): self.version = 2 self.data: dict = data or {} self.aliases: dict = aliases or {} self.metadata: dict = metadata or {} self.user_parameters: dict[str, BaseUserParameter] = user_parameters or {} self._up_overrides: dict = parameter_overrides or {} if isinstance(entries, Mapping) or entries is None: self.entries = entries or {} else: self.entries = {} [self.add_entry(e) for e in entries] def add_entry( self, entry, name: str | None = None, clobber: bool = True, simplify: bool = False ): """Add entry/reader (and its requirements) in-place, with optional alias Parameters ---------- entry: instance of BaseData, BaseReader or their descriptions name: set the key value the iterm will be known as clobber: if False, will not overwrite an entry simplify: if True, checks if an equivalent entity already exists, and returns it's token if found. Such comparisons are relatively slow when you have >>100 entries. """ from intake.readers import BaseData, BaseReader from intake.readers.utils import find_funcs, replace_values if isinstance(entry, (BaseReader, BaseData)): entry = entry.to_entry() if simplify and entry in self: if name and name != entry.token: self.aliases[name] = entry.token return entry.token tokens = {} entry.kwargs = find_funcs(entry.kwargs, tokens) if name: entry._tok = name for tok, thing in reversed(tokens.items()): thing = thing.to_entry() if thing == entry: continue tok = self.add_entry(thing) if tok not in self: # did not add new one, because it's already there old_tok = next(iter(self._find_iter(thing))).token entry.kwargs = replace_values( entry.kwargs, "{data(%s)}" % tok, "{data(%s)}" % old_tok ) if isinstance(entry, ReaderDescription): if clobber is False and entry.token in self.entries: raise ValueError("Name {} exists in catalog, and clobber is False", entry.token) self.entries[entry.token] = entry elif isinstance(entry, DataDescription): if clobber is False and entry.token in self.data: raise ValueError("Name {} exists in catalog, and clobber is False", entry.token) self.data[entry.token] = entry else: raise ValueError if name and name != entry.token: self.aliases[name] = entry.token return entry.token def _ipython_key_completions_(self): return sorted(set(chain(self.aliases, self.data, self.entries))) def delete(self, name, recursive=False): """Remove named entity (data/entry) from catalog We do not check whether any other entity in the catalog refers *to* what is being deleted, so you can break other entries this way. Parameters ---------- recursive: bool Also removed data/entries references by the given one, and those they refer to in turn. """ if recursive: raise NotImplementedError del self[name] def extract_parameter( self, item: str, name: str, path: str | None = None, value: Any = None, cls=SimpleUserParameter, store_to: str | None = None, **kw, ): """ Descend into data & reader descriptions to create a user_parameter There are two ways to fund and replace values by a template: - if ``path`` is given, the kwargs will be walked to this location e.g., "field.0.special_value" -> kwargs["field"][0]["special_value"] - if ``value`` is given, all kwargs will be recursively walked, looking for values that equal that given. Matched values will be replaced by a template string like ``"{name}"``, and a user_parameter of class ``cls`` will be placed in the location given by ``store_to`` (could be "data", "catalog"). """ # TODO: if entity is "Catalog", extract over all entities; currently this will # cause a recursion loop entity = self.get_entity(item) entity.extract_parameter(name, path=path, value=value, cls=cls, **kw) if store_to is None: return elif store_to == "data" and isinstance(entity, ReaderDescription): entity.kwargs["data"].user_parameters[name] = entity.user_parameters.pop(name) else: self.move_parameter(item, store_to, name) def move_parameter(self, from_entity: str, to_entity: str, parameter_name: str) -> Catalog: """Move user-parameter from between entry/data `entity` is an alias name or entry/data token """ entity1 = self.get_entity(from_entity) entity2 = self.get_entity(to_entity) entity2.user_parameters[parameter_name] = entity1.user_parameters.pop(parameter_name) return self def promote_parameter_name(self, parameter_name: str, level: str = "cat") -> Catalog: """Find and promote given named parameter, assuming they are all identical parameter_name: the key string referring to the parameter level: cat | data If the parameter is found in a reader, it can be promoted to the data it depends on. Parameters in a data description can only be promoted to a catalog global. """ up = None ups = None if level not in ("cat", "data"): raise ValueError for entity in self.entries.values(): if parameter_name in entity.user_parameters and up is None: ups = entity.user_parameters[parameter_name] up = entity.user_parameters[parameter_name] entity0 = entity elif ( parameter_name in entity.user_parameters and up == entity.user_parameters[parameter_name] ): continue elif parameter_name in entity.user_parameters: ups[parameter_name] = up # rewind raise ValueError for entity in self.data.values(): if parameter_name in entity.user_parameters and up is None: assert level == "cat" ups = entity.user_parameters[parameter_name] up = entity.user_parameters[parameter_name] elif ( parameter_name in entity.user_parameters and up == entity.user_parameters[parameter_name] ): continue elif parameter_name in entity.user_parameters: ups[parameter_name] = up # rewind raise ValueError if level == "cat": self.user_parameters[parameter_name] = up else: entity0.kwargs["data"].user_parameters[parameter_name] = up return self def __getattr__(self, item): super().tab_completion_fixer(item) try: return self[item] except KeyError: pass except RecursionError as e: raise AttributeError from e raise AttributeError(item) def to_yaml_file(self, path: str, **storage_options): """Persist the state of this catalog as a YAML file storage_options: kwargs to pass to fsspec for opening the file to write """ # TODO: remove ['CATALOG_DIR', 'CATALOG_PATH', 'STORAGE_OPTIONS'] UPs? with fsspec.open(path, mode="wt", **storage_options) as stream: yaml.safe_dump(self.to_dict(), stream) @staticmethod def from_yaml_file(path: str, **kwargs): """Load YAML representation into a new Catalog instance storage_options: kwargs to pass to fsspec for opening the file to read; can pass as storage_options= or will pick up any unused kwargs for simplicity """ storage_options = kwargs.pop("storage_options", kwargs) of = fsspec.open(path, **storage_options) with of as stream: cat = Catalog.from_dict(yaml.safe_load(stream)) cat.user_parameters["CATALOG_PATH"] = path cat.user_parameters["CATALOG_DIR"] = of.fs.unstrip_protocol(of.fs._parent(path)) cat.user_parameters["STORAGE_OPTIONS"] = storage_options return cat @classmethod def from_entries(cls, data: dict, metadata=None): """Assemble catalog from a dict of entries""" cat = cls(metadata=metadata) for k, v in data.items(): cat[k] = v return cat @classmethod def from_dict(cls, data): """Assemble catalog from dict representation""" if data.get("version") != 2: raise ValueError("Not a V2 catalog") cat = cls() for key, clss in zip( ["entries", "data", "user_parameters"], [ReaderDescription, DataDescription, BaseUserParameter], ): for k, v in data[key].items(): if isinstance(v, dict): desc = clss.from_dict(v) desc._tok = k else: desc = v getattr(cat, key)[k] = desc cat.aliases = data["aliases"] cat.metadata = data["metadata"] return cat def get_entity(self, item: str): """Get the objects by reference Use this method if you want to change the catalog in-place item can be an entry in .aliases, in which case the original wil be returned, or a key in .entries, .user_parameters or .data. The entity in question is returned without processing. """ if item.lower() in ["cat", "catalog"]: return self # TODO: this can be simplified with `get(..) or` if item in self.aliases: item = self.aliases[item] if item in self.entries: return self.entries[item] elif item in self.data: return self.data[item] elif item in self.user_parameters: return self.user_parameters[item] else: raise KeyError(item) def get_aliases(self, entity: str): """Return those alias names that point to the given opaque key""" return {k for k, v in self.aliases.items() if v == entity} def search(self, expr) -> Catalog: """Make new catalog with a subset of this catalog The new catalog will have those entries which pass the filter `expr`, which is an instance of `intake.readers.search.BaseSearch` (i.e., has a method like `filter(entry) -> bool`). In the special case that expr is just a string, the `Text` search expression will be used. """ from intake.readers.search import Text if isinstance(expr, str): expr = Text(expr) cat = Catalog() for e, v in self.entries.items(): if expr.filter(v): cat.add_entry(v) aliases = self.get_aliases(e) cat.aliases.update({a: e for a in aliases}) return cat def __getitem__(self, item): ups = self.user_parameters.copy() kw = self._up_overrides.copy() if item in self.aliases: item = self.aliases[item] if item in self.entries: item = copy(self.entries[item]) # TODO: does not pass data's UPs to reader instantiation because data is still a str, # but could grab from self.data # ups.update(item.data.user_parameters) item = self._rehydrate(item) return item(user_parameters=ups, **(kw or {})) elif item in self.data: item = self.data[item] item = self._rehydrate(item) return item.to_data(user_parameters=ups, **(kw or {})) else: raise KeyError(item) def _rehydrate(self, val): """Recreate reader instances when accessed from this catalog, filling in refs and templates""" from intake.readers.entry import DataDescription, ReaderDescription if isinstance(val, dict): return {k: self._rehydrate(v) for k, v in val.items()} elif isinstance(val, str): m = re.match(r"{?data[(]([^)]+)[)]}?", val) if m: return self[m.groups()[0]] return val elif isinstance(val, bytes): return val elif isinstance(val, (tuple, set, list)): return type(val)(self._rehydrate(v) for v in val) elif isinstance(val, (DataDescription, ReaderDescription)): val2 = copy(val) val2.__dict__ = self._rehydrate(val.__dict__) return val2 return val def __delitem__(self, key): # remove alias, data or entry with no further actions if key in self.aliases: self.data.pop(self.aliases[key], None) self.entries.pop(self.aliases[key], None) for k, v in self.aliases.copy().items(): # remove alias pointing TO key if v == key: self.aliases.pop(k) self.aliases.pop(key, None) self.data.pop(key, None) self.entries.pop(key, None) def __delattr__(self, item): del self[item] def _find_iter(self, thing): if isinstance(thing, DataDescription): return (_ for _ in self.data.values() if thing.to_dict() == _.to_dict()) elif isinstance(thing, ReaderDescription): return (_ for _ in self.entries.values() if thing.to_dict() == _.to_dict()) else: return () def __contains__(self, thing): from intake.readers import BaseReader, BaseData if isinstance(thing, (BaseData, BaseReader)): thing = thing.to_entry() if hasattr(thing, "token"): thing0 = thing.token else: thing0 = thing easy = thing0 in self.data or thing0 in self.entries or thing0 in self.aliases return easy or any(self._find_iter(thing)) def __call__(self, **kwargs): """Set override values for any named user parameters Returns a new instance of Catalog with overrides set """ up_over = self._up_overrides.copy() up_over.update(kwargs) new = Catalog( entries=self.entries, aliases=self.aliases, data=self.data, user_parameters=self.user_parameters, parameter_overrides=up_over, metadata=self.metadata, ) return new def __iter__(self): return iter(self.aliases) def __len__(self) -> int: return len(self.aliases) def __dir__(self) -> Iterable[str]: return sorted(chain(object.__dir__(self), self.aliases)) def __add__(self, other: Catalog | DataDescription): if not isinstance(other, (Catalog, DataDescription)): raise TypeError if isinstance(other, (DataDescription, ReaderDescription)): other = Catalog(entries=[other]) return Catalog( entries=chain(self.entries.values(), other.entries.values()), aliases=merge_dicts(self.aliases, other.aliases), data=merge_dicts(self.data, other.data), user_parameters=merge_dicts(self.user_parameters, other.user_parameters), metadata=merge_dicts(self.metadata, other.metadata), ) def __iadd__(self, other: Catalog | ReaderDescription): if not isinstance(other, Catalog): other = Catalog([other]) self.entries.update(other.entries) self.aliases.update(other.aliases) self.user_parameters.update(other.user_parameters) self.metadata.update(other.metadata) return self def __repr__(self): aliases = set(self.aliases).union(e for e in self.entries if not _is_tok(e)) txt = f"""{type(self).__name__} data definitions: {len(self.data)} reader entries: {len(self.entries)} named datasets: {sorted(aliases)}""" if self.user_parameters: txt = txt + f"\n parameters: {sorted(self.user_parameters)}" return txt def __setitem__(self, name: str, entry): """Add the entry to this catalog with the given alias name If the entry is already in the catalog, this effectively just adds an alias. Any existing alias of the same name will be clobbered. """ self.add_entry(entry, name=name) def rename(self, old: str, new: str, clobber=True): """Change the alias of a dataset""" if not clobber and new in self.aliases: raise ValueError self.aliases[new] = self.aliases.pop(old) @property def name(self): if not re.match("^[0-9a-f]{16}$", self.token): return self.token else: return self.metadata.get("name", "unnamed") def give_name(self, tok: str, name: str, clobber=True): """Give an alias to a dataset tok: a key in the .entries dict """ if not clobber and name in self.aliases: raise ValueError if not isinstance(tok, str): tok = tok.token if tok not in self.entries: raise KeyError self.aliases[name] = tok alias = give_name # TODO: methods to split a pipeline into sequence of entries and to rejoin them ================================================ FILE: intake/readers/examples.py ================================================ """This module can contain examples of complex Intake use we wish to refer to""" import operator def ms_building_parquet(): """22k build polygon outlines in the US Virgin Islands""" import intake.readers import planetary_computer cat = intake.readers.datatypes.STACJSON( "https://planetarycomputer.microsoft.com/api/stac/v1" ).to_reader( reader="StacSearch", query={"collections": ["ms-buildings"]}, signer=planetary_computer.sign_inplace, prefer="Awkward", ) return cat.apply(operator.getitem, "USVirginIslands_32300213_2023-04-25").apply( operator.getitem, "data", output_instance="intake.readers.readers:AwkwardParquet", ) ms_us_virgin_cat = """ aliases: building_outlines: 1e150595b4343b5a data: ea539c01591e5e69: datatype: intake.readers.datatypes:STACJSON kwargs: metadata: {} storage_options: null url: https://planetarycomputer.microsoft.com/api/stac/v1 metadata: {} user_parameters: {} entries: 1e150595b4343b5a: kwargs: out_instances: - intake.readers.entry:Catalog - intake.readers.entry:Catalog - intake.readers.entry:Catalog steps: - - '{data(bd3879bda378781f)}' - [] - {} - - '{func(intake.readers.convert:GenericFunc)}' - - USVirginIslands_32300213_2023-04-25 - func: '{func(_operator:getitem)}' - - '{func(intake.readers.convert:GenericFunc)}' - - data - func: '{func(_operator:getitem)}' metadata: {} output_instance: intake.readers.entry:Catalog reader: intake.readers.convert:Pipeline user_parameters: {} bd3879bda378781f: kwargs: data: '{data(ea539c01591e5e69)}' prefer: Awkward query: collections: - ms-buildings signer: '{func(planetary_computer.sas:sign_inplace)}' metadata: {} output_instance: intake.readers.entry:Catalog reader: intake.readers.catalogs:StacSearch user_parameters: {} metadata: {} user_parameters: {} version: 2 """ def ms_delta_buildings(): # replicates https://planetarycomputer.microsoft.com/dataset/ms-buildings#Example-Notebook import intake.readers import planetary_computer cat = intake.readers.datatypes.STACJSON( "https://planetarycomputer.microsoft.com/api/stac/v1" ).to_reader(reader="StacCatalog", signer=planetary_computer.sign_inplace, prefer="Delta") return cat.apply(operator.getitem, "ms-buildings").apply( operator.getitem, "delta", output_instance="deltalake:DeltaTable" ) ================================================ FILE: intake/readers/importlist.py ================================================ """Imports made my intake when it itself is imported Since "plugins" are just subclasses of things like intake.readers.readers.BaseReader, importing them is enough for registration. To include imports from a package in the list of things to import, the canonical thing to do is include an entry under "intake.imports" in the package entrypoints; the value of each item will be imported. The following config keys define behaviour: - import_on_startup: if False, makes no automatic imports - entrypoints_block_list: if an entrypoints import has a top-level package name in this list, it will be skipped - import_extras: values in this list will be imported after processing entrypoints. This is a way to include imports without installing packages/entrypoints. """ from intake import conf, import_name, logger from importlib.metadata import entry_points def process_entries(): eps = entry_points() if hasattr(eps, "select"): # Python 3.10+ / importlib_metadata >= 3.9.0 specs = eps.select(group="intake.imports") else: specs = eps.get("intake.imports", []) for spec in specs: top_level = spec.value.split(":", 1)[0].split(".", 1)[0] bl = conf.get("import_block_list", []) if top_level in bl or spec.name in bl: logger.debug("Skipping import of %s", spec) continue try: import_name(spec.value) except Exception as e: logger.warning( "Importing %s as part of processing intake entrypoints failed\n(%s)", spec.value, e, ) for impname in conf["extra_imports"]: try: import_name(impname) except Exception as e: logger.warning( "Importing %s as part of processing intake extra_imports failed\n(%s)", impname, e, ) if conf["import_on_startup"]: process_entries() ================================================ FILE: intake/readers/metadata.py ================================================ """Some types and meanings of fields that can be expected in metadata dictionaries Metadata should be JSON-serializable. We may decide to have different recommended keys for data versus readers/pipelines For a possible schema we could decide to use, see https://specs.frictionlessdata.io/data-resource/ """ from __future__ import annotations from typing import List metadata_fields = { "description": (str, "one-line description of the dataset"), "text": (str, "long-form prose description of the dataset"), "timestamp": ( str, "most recent datum in the set, ISO format", ), # timespan would be in "data" as an extent "imports": (List[str], "top-level packages needed to read this"), "environment": (str, "YAML string or URL of a conda env spec"), # or requirements.txt "references": (List[str], "URLs with further information relating to this"), "repr": (str, "string form of output"), "data": ( dict, "any data-specific details, such as field types, missing values bounds or statistics", ), "history": ( List[dict], "Time-ordered list of operations done to get this data. Keys are ISO timestamps.", ), "datashape": (str, "if applicable, may have datashape, dtype(s), jsonschema or similar"), "thumbnail": (str, "url location of an image, ideally PNG format"), } ================================================ FILE: intake/readers/mixins.py ================================================ """Helpers for creating pipelines""" from __future__ import annotations import re from itertools import chain from intake import import_name from intake.readers.utils import Completable class PipelineMixin(Completable): """Make it possible to associate transforms with the given class""" def __getattr__(self, item): super().tab_completion_fixer(item) try: if item in dir(self.transform): return getattr(self.transform, item) if "Catalog" in self.output_instance: # a better way to mark this condition, perhaps the datatype's structure? out = self.read()[item] elif item in self._namespaces: out = self._namespaces[item] # the following can go very wrong - only allow via explicit opt-in? else: out = self.transform.__getattr__(item) # arbitrary method call except RecursionError as e: raise AttributeError(item) from e else: return out def __getitem__(self, item: str): from intake.readers.convert import Pipeline from intake.readers.transform import GetItem outtype = self.output_instance if "Catalog" in outtype: # a better way to mark this condition, perhaps the datatype's structure? # TODO: this prevents from doing a transform/convert on a cat, so must use # .transform for that return self.read()[item] if isinstance(self, Pipeline): return self.with_step((GetItem, (item,), {}), out_instance=outtype) return Pipeline( steps=[(self, (), {}), (GetItem, (item,), {})], out_instances=[self.output_instance, outtype], metadata=self.metadata, ) def __dir__(self): return sorted(chain(object.__dir__(self), dir(self.transform), self._namespaces)) @property def _namespaces(self): from intake.readers.namespaces import get_namespaces return get_namespaces(self) @classmethod def output_doc(cls): """Doc associated with output type""" out = import_name(cls.output_instance) return out.__doc__ def apply(self, func, *args, output_instance=None, **kwargs): """Make a pipeline by applying a function to this reader's output""" from intake.readers.convert import GenericFunc, Pipeline kwargs["func"] = func return Pipeline( steps=[(self, (), {}), (GenericFunc, args, kwargs)], out_instances=[ self.output_instance, output_instance or self.output_instance, ], metadata=self.metadata, ) @property def transform(self): from intake.readers.convert import convert_classes funcdict = convert_classes(self.output_instance) return Functioner(self, funcdict) class Functioner(Completable): """Find and apply transform functions to reader output""" def __init__(self, reader, funcdict): self.reader = reader self.funcdict = funcdict def _ipython_key_completions_(self): return [_[1].__name__ for _ in self.funcdict] def __getitem__(self, item): from intake.readers.convert import Pipeline from intake.readers.transform import GetItem found = False for out, func in self.funcdict: ###### if func.__name__ == item or re.findall(item, func): arg = () kw = {} found = True break if not found: func = GetItem arg = (item,) kw = {} if isinstance(self.reader, Pipeline): return self.reader.with_step((func, (), kw), out_instance=item) return Pipeline( steps=[(self.reader, (), {}), (func, arg, kw)], out_instances=[self.reader.output_instance, item], metadata=self.reader.metadata, ) def __repr__(self): import pprint return f"Transformers for {self.reader.output_instance}:\n{pprint.pformat(self.funcdict)}" def __call__(self, func, *args, output_instance=None, **kwargs): from intake.readers.convert import Pipeline if isinstance(self.reader, Pipeline): return self.reader.with_step((func, args, kwargs), out_instance=output_instance) # TODO: get output_instance from func, if possible return Pipeline( steps=[(self.reader, (), {}), (func, args, kwargs)], out_instances=[self.reader.output_instance, output_instance], metadata=self.reader.metadata, ) def methods(self): """Methods and attributes associated with the output_instance""" try: cls = import_name(self.reader.output_instance) dnames = (_ for _ in dir(cls) if not _.startswith("_")) except (ImportError, AttributeError): dnames = [] return dnames def __dir__(self): return list(sorted(set(chain((f.__name__ for (_, f) in self.funcdict), self.methods())))) def __getattr__(self, item): super().tab_completion_fixer(item) from intake.readers.convert import Pipeline from intake.readers.transform import Method out = [(outtype, func) for outtype, func in self.funcdict if func.__name__ == item] if not len(out): # TODO: import class being acted on, to see if attribute requested # really is available. Perhaps tie to config option. # TODO: exclude certain attributes that might be called during # a stack trace or other non-normal code, causing accidental # massive pipelines. E.g., dunders. For those, require `apply()`. outtype = self.reader.output_instance if item.startswith("_"): raise AttributeError(item) func = Method kw = {"method_name": item} else: outtype, func = out[0] kw = {} try: if isinstance(self.reader, Pipeline): out = self.reader.with_step((func, (), kw), out_instance=outtype) else: out = Pipeline( steps=[(self.reader, (), {}), (func, (), kw)], out_instances=[self.reader.output_instance, outtype], metadata=self.reader.metadata, ) except RecursionError as e: raise AttributeError from e else: return out ================================================ FILE: intake/readers/namespaces.py ================================================ """Add module accessors to pipelines, providing functions appropriate for its output The code here allow something like pipeline.np. to get completions from the numpy namespace, and apply to the pipeline. """ from __future__ import annotations import importlib.metadata import re from functools import lru_cache as cache from typing import Iterable from intake.readers.utils import Completable, subclasses class Namespace(Completable): """A set of functions as an accessor on a Reader, producing a Pipeline""" acts_on: tuple[str] = () #: types that this namespace is associated with imports: tuple[str] = () #: requires this top-level package def __init__(self, reader): self.reader = reader @classmethod @cache def _funcs(cls) -> Iterable[str]: if not cls.check_imports(): return [] # if self.reader.output_instance doesn't match self.acts_on cls.mod = importlib.import_module(cls.imports[0]) return [f for f in dir(cls.mod) if callable(getattr(cls.mod, f)) and not f.startswith("_")] def __dir__(self) -> Iterable[str]: # if self.reader.output_instance doesn't match self.acts_on: # return [] return self._funcs() def __getattr__(self, item): super().tab_completion_fixer(item) try: dir(self) func = getattr(self.mod, item) return FuncHolder(self.reader, func) except RecursionError as e: raise AttributeError from e def __repr__(self): return f"{self.imports} namespace" class FuncHolder: """Acts like a function to capture a call into a pipeline stage""" def __init__(self, reader, func): self.reader = reader self.func = func def __call__(self, *args, **kwargs): return self.reader.apply(self.func, **kwargs) class np(Namespace): acts_on = (".*",) # numpy works with a wide variety of objects imports = ("numpy",) class ak(Namespace): acts_on = "awkward:Array", "dask_awkward:Array" imports = ("awkward",) class xr(Namespace): acts_on = "xarray:DataArray", "xarray:Dataset" imports = ("xarray",) class pd(Namespace): acts_on = ("pandas:DataFrame", "pandas.Series") imports = ("pandas",) class pl(Namespace): acts_on = ("polars:DataFrame", "polars:Series", "polars:LazyFrame") imports = ("polars",) def get_namespaces(reader): """These namespaces are available on the reader""" out = {} for space in subclasses(Namespace): if any(re.match(act.lower(), reader.output_instance.lower()) for act in space.acts_on): out[space.__name__] = space(reader) return out ================================================ FILE: intake/readers/output.py ================================================ """Serialise and output data into persistent formats This is how to "export" data from Intake. By convention, functions here produce an instance of FileData (or other data type), which can then be used to produce new catalog entries. """ from __future__ import annotations import fsspec from intake.readers.convert import BaseConverter from intake.readers.datatypes import ( CSV, HDF5, PNG, CatalogFile, Feather2, NumpyFile, Parquet, Zarr, recommend, ) from intake.readers.utils import all_to_one # TODO: superclass for output, so they show up differently in any graph viz? # *most* things here produce a datatypes:BaseData, but not all class PandasToParquet(BaseConverter): instances = all_to_one( {"pandas:DataFrame", "dask.dataframe:DataFrame", "geopandas:GeoDataFrame"}, "intake.readers.datatypes:Parquet", ) def run(self, x, url, storage_options=None, metadata=None, **kwargs): x.to_parquet(url, storage_options=storage_options, **kwargs) return Parquet(url=url, storage_options=storage_options, metadata=metadata) class PandasToCSV(BaseConverter): instances = all_to_one( {"pandas:DataFrame", "dask.dataframe:DataFrame", "geopandas:GeoDataFrame"}, "intake.readers.datatypes:CSV", ) def run(self, x, url, storage_options=None, metadata=None, **kwargs): x.to_csv(url, storage_options=storage_options, **kwargs) return CSV(url=url, storage_options=storage_options, metadata=metadata) class PandasToHDF5(BaseConverter): instances = all_to_one( {"pandas:DataFrame", "dask.dataframe:DataFrame"}, "intake.readers.datatypes:HDF5", ) def run(self, x, url, table, storage_options=None, metadata=None, **kwargs): x.to_hdf(url, table, storage_options=storage_options, **kwargs) return HDF5(url=url, path=table, storage_options=storage_options, metadata=metadata) class PandasToFeather(BaseConverter): instances = all_to_one( {"pandas:DataFrame", "geopandas:GeoDataFrame"}, "intake.readers.datatypes:Feather2", ) def run(self, x, url, storage_options=None, metadata=None, **kwargs): # TODO: fsspec output x.to_feather(url, storage_options=storage_options, **kwargs) return Feather2(url=url, storage_options=storage_options, metadata=metadata) class XarrayToNetCDF(BaseConverter): instances = {"xarray:Dataset": "intake.readers.datatypes:HDF5"} def run(self, x, url, group="", metadata=None, **kwargs): x.to_netcdf(path=url, group=group or None, **kwargs) return HDF5(url, path=group, metadata=metadata) class XarrayToZarr(BaseConverter): instances = {"xarray:Dataset": "intake.readers.datatypes:Zarr"} func = "xarray:Dataset.to_zarr" def run(self, x, url, group="", storage_options=None, metadata=None, **kwargs): x.to_zarr(store=url, group=group or None, storage_options=storage_options, **kwargs) return Zarr(url, storage_options=storage_options, root=group, metadata=metadata) class DaskArrayToZarr(BaseConverter): instances = {"dask.array:Array": "intake.readers.datatypes:Zarr"} func = "xarray:Dataset.to_zarr" def run(self, x, url, group="", storage_options=None, metadata=None, **kwargs): x.to_zarr( store=url, component=group or None, storage_options=storage_options, **kwargs, ) return Zarr(url, storage_options=storage_options, root=group, metadata=metadata) class NumpyToNumpyFile(BaseConverter): """Save a single array into a single binary file""" instances = {"numpy:ndarray": "intake.readers.datatypes:NumpyFile"} func = "numpy:save" def run(self, x, path, *args, storage_options=None, metadata=None, **kwargs): if storage_options or "://" in path or "::" in path: with fsspec.open(path, **storage_options) as f: self._func(x, f) else: self._func(x, path) return NumpyFile(path, storage_options, metadata=metadata) class ToMatplotlib(BaseConverter): instances = all_to_one( {"pandas:DataFrame", "geopandas:GeoDataFrame", "xarray:Dataset"}, "matplotlib.pyplot:Figure", ) func = "matplotlib.pyplot:Figure" func_doc = "matplotlib.pyplot:plot" def run(self, x, **kwargs): fig = self._func() ax = fig.add_subplot(111) x.plot(ax=ax, **kwargs) return fig class MatplotlibToPNG(BaseConverter): """Take a matplotlib figure and save to PNG file This could be used to produce thumbnails if followed by FileByteReader; to use temporary storage rather than concrete files, can use memory: or caching filesystems """ instances = {"matplotlib.pyplot:Figure": "intake.readers.datatypes:PNG"} def run(self, x, url, metadata=None, storage_options=None, **kwargs): with fsspec.open(url, mode="wb", **(storage_options or {})) as f: x.savefig(f, format="png", **kwargs) return PNG(url=url, metadata=metadata, storage_options=storage_options) class GeopandasToFile(BaseConverter): """creates one of several output file types Uses url extension or explicit driver= kwarg """ instances = {"geopandas:GeoDataFrame": "intake.readers.datatypes:GeoJSON"} def run(self, x, url, metadata=None, **kwargs): x.to_file(url, **kwargs) return recommend(url)[0](url=url, metadata=metadata) class Repr(BaseConverter): """good for including "peek" at data in entries' metadata""" instances = {".*": "builtins:str"} func = "builtins:repr" class IPythonDisplay(BaseConverter): # maybe restrict to types known to render well? instances = {".*": "builtins:dict"} def run(self, x, **kwargs): """Produce ipython/jupyter compatible output without imports""" # does not consider _ipython_display_ types = { "html": "text/html", "svg": "image/svg+xml", "png": "image/png", "jpeg": "image/jpeg", "latex": "text/latex", "json": "application/json", "pretty": "text/plain", } for name, mime in types.items(): if hasattr(x, f"_repr_{name}_"): out = getattr(x, f"_repr_{name}_")() # out can be (data, metadata) for some calls # https://ipython.readthedocs.io/en/stable/config/integrating.html#metadata return {mime: out[0] if isinstance(out, tuple) else out} if hasattr(x, "_repr_mimebundle_"): return x._repr_mimebundle_() return {"text/plain": repr(x)} class CatalogToJson(BaseConverter): instances = {"intake.readers.entry:Catalog": "intake.readers.datatypes:CatalogFile"} func = "intake.readers.entry:Catalog.to_yaml_file" def run(self, x, url, metadata=None, storage_options=None, **kwargs): if storage_options: kwargs.update(storage_options) x.to_yaml_file(url, **kwargs) return CatalogFile(url=url, storage_options=storage_options, metadata=metadata) ================================================ FILE: intake/readers/readers.py ================================================ """Classes for reading data into a python objects""" from __future__ import annotations import inspect import itertools import json import os import re from functools import lru_cache import fsspec from fsspec.callbacks import _DEFAULT_CALLBACK as DEFAULT_CALLBACK import intake.readers.datatypes from intake import import_name, logger from intake.readers import datatypes from intake.readers.mixins import PipelineMixin from intake.readers.utils import Tokenizable, subclasses, port_in_use, find_free_port from intake.utils import is_fsspec_url class BaseReader(Tokenizable, PipelineMixin): imports: set[str] = set() #: top-level packages required to use this implements: set[datatypes.BaseData] = set() #: datatype(s) this applies to optional_imports: set[str] = set() #: packages that might be required by some options func: str = "builtins:NotImplementedError" #: function name for loading data func_doc: str = None #: docstring origin if not from func output_instance: str = None #: type the reader produces other_funcs: set[str] = set() #: function names to recognise when matching user calls def __init__( self, *args, metadata: dict | None = None, output_instance: str | None = None, **kwargs, ): if ( self.implements and (args and not isinstance(args[0], datatypes.BaseData)) and not any(isinstance(_, datatypes.BaseData) for _ in kwargs.values()) ): # reader requires data input, but not given - guess if len(self.implements) == 1: d_cls = list(self.implements)[0] sig = inspect.signature(d_cls.__init__).parameters kw2 = {} for k, v in kwargs.copy().items(): if k in sig: kw2[k] = kwargs.pop(k) args = (d_cls(*args, **kw2),) else: raise NotImplementedError( "Guessing the data type not supported for a reader implementing multiple data " "classes. Please Instantiate the data type directly." ) self.kwargs = kwargs if args: self.kwargs["args"] = args met = {} for a in itertools.chain( reversed(kwargs.get("args", [])), reversed(kwargs.values()), reversed(args) ): if isinstance(a, datatypes.BaseData): met.update(a.metadata) met.update(metadata or {}) self.metadata = met if output_instance: self.output_instance = output_instance def __repr__(self): return f"{type(self).__name__} reader producing {self.output_instance}" def __call__(self, *args, **kwargs): """New version of this instance with altered arguments""" kw = self.kwargs.copy() kw.update(kwargs) if args: kw["args"] = args return type(self)(**kw) @classmethod def doc(cls): """Doc associated with loading function""" f = cls.func_doc or cls.func if isinstance(f, str): f = import_name(f) upstream = f.__doc__ if f is not NotImplementedError else "" sig = str(inspect.signature(cls._read)) doc = cls._read.__doc__ return "\n\n".join(_ for _ in [cls.qname(), cls.__doc__, sig, doc, upstream] if _) def discover(self, **kwargs): """Part of the data The intent is to return a minimal dataset, but for some readers and conditions this may be up to the whole of the data. Output type is the same as for read(). """ return self.read(**kwargs) @property def _func(self): """Import and replace .func, if it is a string""" if isinstance(self.func, str): return import_name(self.func) return self.func def read(self, *args, **kwargs): """Produce data artefact Any of the arguments encoded in the data instance can be overridden. Output type is given by the .output_instance attribute """ logger.debug("Reading %s", self) kw = self.kwargs.copy() kw.update(kwargs) args = kw.pop("args", ()) or args return self._read(*args, **kw) def _read(self, *args, **kwargs): """This is the method subclasses will tend to override""" raise NotImplementedError def to_entry(self): """Create an entry version of this, ready to be inserted into a Catalog""" from intake.readers.entry import ReaderDescription return ReaderDescription( reader=self.qname(), kwargs=self.kwargs, output_instance=self.output_instance, metadata=self.metadata, ) def to_cat(self, name=None): """Create a Catalog containing on this reader""" return self.to_entry().to_cat(name) @property def data(self): """The BaseData this reader depends on, if it has one""" data = self.kwargs.get("data") if data is None: args = self.kwargs.get("args", ()) if not (args): raise ValueError("Cloud not find a data entity in this reader") data = args[0] if not isinstance(data, datatypes.BaseData): raise ValueError("Data argument isn't a BaseData") return data def to_reader(self, outtype: tuple[str] | str | None = None, reader: str | None = None, **kw): """Make a different reader for the data used by this reader""" return self.data.to_reader(outtype=outtype, reader=reader, metadata=self.metadata, **kw) def auto_pipeline(self, outtype: str | tuple[str], avoid: list[str] | None = None): from intake import auto_pipeline return auto_pipeline(self, outtype=outtype, avoid=avoid) class FileReader(BaseReader): """Convenience superclass for readers of files""" url_arg = "url" other_urls = {} # if we have other_funcs, may have different url_args for each storage_options = False def _read(self, data, **kw): kw[self.url_arg] = data.url if self.storage_options and data.storage_options: kw["storage_options"] = data.storage_options return self._func(**kw) class OpenFilesReader(FileReader): url_arg = "urlpath" implements = {datatypes.FileData} func = "fsspec:open_files" output_instance = "fsspec.core:OpenFiles" class PanelImageViewer(FileReader): output_instance = "panel.pane:Image" implements = {datatypes.PNG, datatypes.JPEG} func = "panel.pane:Image" url_arg = "object" class FileByteReader(FileReader): """The contents of file(s) as bytes""" output_instance = "builtin:bytes" implements = {datatypes.FileData} def discover(self, data=None, **kwargs): data = data or self.kwargs["data"] with fsspec.open(data.url, mode="rb", **(data.storage_options or {})) as f: return f.read() def _read(self, data, **kwargs): out = [] for of in fsspec.open_files(data.url, mode="rb", **(data.storage_options or {})): with of as f: out.append(f.read()) return b"".join(out) class FileTextReader(FileReader): """The contents of file(s) as str""" output_instance = "builtins:str" implements = {datatypes.FileData} def discover(self, data=None, encoding=None, **kwargs): data = data or self.kwargs["data"] if encoding: data.storage_options["encoding"] = encoding with fsspec.open(data.url, mode="rt", **(data.storage_options or {})) as f: return f.read() def _read(self, data, encoding=None, **kwargs): out = [] if encoding: data.storage_options["encoding"] = encoding for of in fsspec.open_files(data.url, mode="rt", **(data.storage_options or {})): with of as f: out.append(f.read()) return "".join(out) class FileSizeReader(FileReader): output_instance = "builtins:int" implements = {datatypes.FileData} def _read(self, data, **kw): fs, path = fsspec.core.url_to_fs(data.url, **(data.storage_options or {})) path = fs.expand_path(path) # or use fs.du with deep return sum(fs.info(p)["size"] for p in path) class Pandas(FileReader): imports = {"pandas"} output_instance = "pandas:DataFrame" storage_options = True class PandasParquet(Pandas): implements = {datatypes.Parquet} optional_imports = {"fastparquet", "pyarrow"} func = "pandas:read_parquet" url_arg = "path" class PandasFeather(Pandas): implements = {datatypes.Feather2, datatypes.Feather1} imports = {"pandas", "pyarrow"} func = "pandas:read_feather" url_arg = "path" class PandasORC(Pandas): implements = {datatypes.ORC} imports = {"pandas", "pyarrow"} func = "pandas:read_orc" url_arg = "path" class PandasExcel(Pandas): implements = {datatypes.Excel} imports = {"pandas", "openpyxl"} func = "pandas:read_excel" url_arg = "io" class PandasSQLAlchemy(BaseReader): implements = {datatypes.SQLQuery} func = "pandas:read_sql" imports = {"sqlalchemy", "pandas"} output_instance = "pandas:DataFrame" def discover(self, **kwargs): if "chunksize" not in kwargs: kwargs["chunksize"] = 10 return next(iter(self.read(**kwargs))) def _read(self, data, **kwargs): read_sql = import_name(self.func) return read_sql(sql=data.query, con=data.conn, **kwargs) class DaskDF(FileReader): imports = {"dask", "pandas"} output_instance = "dask.dataframe:DataFrame" storage_options = True def discover(self, **kwargs): return self.read().head() class DaskParquet(DaskDF): implements = {datatypes.Parquet} optional_imports = {"fastparquet", "pyarrow"} func = "dask.dataframe:read_parquet" url_arg = "path" class DaskGeoParquet(DaskParquet): imports = {"dask", "geopandas"} func = "dask_geopandas:read_parquet" output_instance = "dask_geopandas.core:GeoDataFrame" class DaskHDF(DaskDF): implements = {datatypes.HDF5} optional_imports = {"h5py"} func = "dask.dataframe:read_hdf" url_arg = "pattern" def _read(self, data, **kw): return self._func(data.url, key=data.path, **kw) class DaskJSON(DaskDF): implements = {datatypes.JSONFile} func = "dask.dataframe:read_json" url_arg = "url_path" class DaskDeltaLake(DaskDF): implements = {datatypes.DeltalakeTable} imports = {"dask_deltatable"} func = "dask_deltatable:read_deltalake" url_arg = "path" class DaskSQL(BaseReader): implements = {datatypes.SQLQuery} imports = {"dask", "pandas", "sqlalchemy"} func = "dask.dataframe:read_sql" def _read(self, data, index_col, **kw): """Dask requires `index_col` to partition the dataframe on.""" return self._func(data.quary, data.conn, index_col, **kw) class DaskNPYStack(FileReader): """Requires a directory with .npy files and an "info" pickle file""" # TODO: single npy file, or stack without info (which can be read from any one file) implements = {datatypes.NumpyFile} imports = {"dask", "numpy"} func = "dask.array:from_npy_stack" output_instance = "dask.array:Array" url_arg = "dirname" class DaskZarr(FileReader): implements = {datatypes.Zarr} imports = {"dask", "zarr"} output_instance = "dask.array:Array" func = "dask.array:from_zarr" def _read(self, data, **kwargs): return self._func( url=data.url, component=data.root or None, storage_options=data.storage_options, **kwargs, ) class NumpyZarr(FileReader): implements = {datatypes.Zarr} imports = {"zarr"} output_instance = "numpy:ndarray" func = "zarr:open" def _read(self, data, **kwargs): return self._func(data.url, storage_options=data.storage_options, path=data.root, **kwargs)[ : ] class DuckDB(BaseReader): imports = {"duckdb"} output_instance = "duckdb:DuckDBPyRelation" # can be converted to pandas with .df func_doc = "duckdb:query" implements = {datatypes.SQLQuery} _dd = {} # hold the engines, so results are still valid def discover(self, **kwargs): return self.read().limit(10) @classmethod def _duck(cls, data, conn=None): import duckdb conn = getattr(data, "conn", conn) or {} # only SQL type normally has this if str(conn) not in cls._dd: # TODO: separate engine creation and caching? if isinstance(conn, str): # https://duckdb.org/docs/extensions/ if conn.startswith("sqlite:"): duckdb.connect(":default:").execute("INSTALL sqlite;LOAD sqlite;") conn1 = re.sub("^sqlite3?:/{0,3}", "", conn) conn1 = {"database": conn1} d = duckdb.connect(**conn1) elif conn.startswith("postgres"): d = duckdb.connect() d.execute("INSTALL postgres;LOAD postgres;") # extra params possible here https://duckdb.org/docs/extensions/postgres_scanner#usage d.execute(f"CALL postgres_attach('{conn}');") else: d = duckdb.connect(conn) else: d = duckdb.connect(**conn) cls._dd[str(conn)] = d # connection must be cached for results to be usable d = cls._dd[str(conn)] if isinstance(data, datatypes.FileData) and "://" in data.url: d.execute("INSTALL httpfs;LOAD httpfs;") return d class DuckParquet(DuckDB, FileReader): implements = {datatypes.Parquet} def _read(self, data, **kwargs): return self._duck(data).query(f"SELECT * FROM read_parquet('{data.url}')") class DuckCSV(DuckDB, FileReader): implements = {datatypes.CSV} def _read(self, data, **kwargs): return self._duck(data).query(f"SELECT * FROM read_csv_auto('{data.url}')") class DuckJSON(DuckDB, FileReader): implements = {datatypes.JSONFile} def _read(self, data, **kwargs): return self._duck(data).query(f"SELECT * FROM read_json_auto('{data.url}')") class DuckSQL(DuckDB): implements = {datatypes.SQLQuery} def _read(self, data, **kwargs): words = len(data.query.split()) q = data.query if words > 1 else f"SELECT * FROM {data.query}" return self._duck(data).query(q) class SparkDataFrame(FileReader): imports = {"pyspark"} func = "pyspark.sql:SparkSession.builder.getOrCreate" func_doc = "pyspark.sql:SparkSession.read" output_instance = "pyspark.sql:DataFrame" def discover(self, **kwargs): return self.read(**kwargs).limit(10) class SparkCSV(SparkDataFrame): implements = {datatypes.CSV} def _read(self, data, **kwargs): return self._func().read.csv(data.url, **kwargs) class SparkParquet(SparkDataFrame): implements = {datatypes.Parquet} def _read(self, data, **kwargs): return self._func().read.parquet(data.url, **kwargs) class SparkText(SparkDataFrame): implements = {datatypes.Text} def _read(self, data, **kwargs): return self._func().read.text(data.url, **kwargs) class SparkDeltaLake(SparkDataFrame): implements = {datatypes.DeltalakeTable} imports = {"pyspark", "delta-spark"} def _read(self, data, **kw): # see https://docs.delta.io/latest/quick-start.html#python for config return self._func().read.format("delta").load(data.url, **kw) class HuggingfaceReader(BaseReader): imports = {"datasets"} implements = {datatypes.HuggingfaceDataset} func = "datasets:load_dataset" output_instance = "datasets.arrow_dataset:Dataset" def _read(self, data, *args, **kwargs): return self._func(data.name, split=data.split, **kwargs) class SKLearnExampleReader(BaseReader): func = "sklearn:datasets" imports = {"sklearn"} output_instance = "sklearn.utils:Bunch" def _read(self, name, **kw): import sklearn.datasets loader = getattr(sklearn.datasets, f"load_{name}", None) or getattr( sklearn.datasets, f"fetch_{name}" ) return loader() class LlamaServerReader(BaseReader): """Create llama.cpp server using local pretrained model file The read() method allows you to pass arguments directly to the llama.cpp server as kwargs. Common arguments are host: (str) hostname for the the server to listen on, default: 127.0.0.1 port: (int) port number for the server to listen on, default: 0, which means first free port system_prompt_file: (uri) Special handling here to support fsspec uri where the file is cached to disk first Additional kwargs not passed to llama.cpp startup_timeout: (int) time in seconds to wait for server to respond to a health check before failing, default 60 callback: fsspec.callbacks.Callback derived instance progress indicator during model download, default None Any remaining kwargs are passed as '-- ' to llama.cpp. Where '_' is replaced with '-'. For options that do not take arguments, like `--verbose` the value should be None or the empty string "". The following short-name arguments are supported as kwargs, they will be transformed to long-form when passed to llama.cpp. If a short-name option is missing it is best to use the long-form. """ output_instance = "intake.readers.datatypes:LlamaCPPService" implements = {datatypes.GGUF} imports = {"requests"} _short_kwargs = { "v": "verbose", "s": "seed", "t": "threads", "tb": "threads-draft", "tbd": "threads-batch-draft", "ps": "p-split", "lcs": "lookup-cache-static", "lcd": "lookup-cache-dynamic", "c": "ctx-size", "n": "predict", "b": "batch-size", "ub": "ubatch-size", "fa": "flash-attn", "p": "prompt", "f": "file", "bf": "binary-file", "e": "escape", "ptc": "prompt-token-count", "r": "reverse-prompt", "sp": "special", "cnv": "conversation", "l": "logit-bias", "j": "json-schema", "gan": "grp-attn-n", "gaw": "grp-attn-w", "dkvc": "dump-kv-cache", "nkvo": "no-ko-offload", "ctk": "cache-type-k", "ctv": "cache-type-v", "dt": "defrag-thold", "np": "parallel", "ns": "sequences", "cb": "cont-batching", "ngl": "gpu-layers", "ngld": "gpu-layers-draft", "sm": "split-mode", "ts": "tensor-split", "mg": "main-gpu", "md": "model-draft", "o": "output", "sps": "slot-prompt-similarity", "ld": "logdir", } @classmethod def _short_kwargs_docs(cls): return "\n".join(f" -{k:4s}-> --{v}" for k, v in cls._short_kwargs.items()) @classmethod @lru_cache() def _find_executable(cls): import shutil # executables were renamed in https://github.com/ggerganov/llama.cpp/pull/7809 path = shutil.which("llama-server") if path is None: # fallback on old name path = shutil.which("server") return path @classmethod def check_imports(cls): imports = super().check_imports() path = cls._find_executable() return imports & (path is not None) def _local_model_path(self, data, callback=DEFAULT_CALLBACK): import os from fsspec.core import split_protocol from intake.catalog.default import user_data_dir protocol, _ = split_protocol(data.url) if protocol is None: # no protocol means local path return data.url storage_options = {} if data.storage_options is None else data.storage_options cache_location = os.path.join(user_data_dir(), "llama.cpp") options = { protocol: storage_options, "simplecache": {"cache_storage": cache_location}, } fs, path = fsspec.core.url_to_fs(f"simplecache::{data.url}", **options) cached_fn = fs._check_file(path) if cached_fn: return cached_fn sha = fs._mapper(path) cached_fn = os.path.join(fs.storage[-1], sha) fs.fs.get_file(path, cached_fn, callback=callback) return cached_fn def _read(self, data, log_file="llama-cpp.log", **kwargs): startup_timeout = kwargs.pop("startup_timeout", 60) callback = kwargs.pop("callback", DEFAULT_CALLBACK) port = kwargs.pop("port", 0) host = kwargs.pop("host", "127.0.0.1") if port == 0: port = find_free_port() URL = f"http://{host}:{port}" if port_in_use(host, port): raise RuntimeError(f"{URL} in use.") import requests import subprocess import atexit f = open(log_file, "wb") server_path = self._find_executable() path = self._local_model_path(data, callback=callback) cmd = [server_path, "-m", path, "--host", host, "--port", str(port), "--log-disable"] for k, v in kwargs.items(): if k in self._short_kwargs: k = self._short_kwargs[k] k = k.replace("_", "-") if not k.startswith("-"): k = f"--{k}" if k == "--system-prompt-file": path = fsspec.open_local(f"simplecache::{v}") cmd.extend([str(k), path]) elif v not in [None, ""]: cmd.extend([str(k), str(v)]) else: cmd.append(str(k)) P = subprocess.Popen(cmd, stdout=f, stderr=f) import time t0 = time.time() while True: try: res = requests.get(f"{URL}/health") if res.ok: break except requests.ConnectionError: pass elapsed = time.time() - t0 if (P.poll() is not None) or (elapsed > startup_timeout): raise RuntimeError( f"Could not start {server_path}. See {log_file} for more details." ) atexit.register(P.terminate) return intake.readers.datatypes.LlamaCPPService( url=URL, options={"Process": P, "log_file": log_file} ) LlamaServerReader.__doc__ += f"\n{LlamaServerReader._short_kwargs_docs()}" class LlamaCPPCompletion(BaseReader): implements = {datatypes.LlamaCPPService} imports = {"requests"} output_instance = "builtins:str" def _read(self, data, prompt: str = "", *args, **kwargs): import requests r = requests.post( f"{data.url}/completion", json={"prompt": prompt, **kwargs}, headers={"Content-Type": "application/json"}, ) return r.json()["content"] class LlamaCPPEmbedding(BaseReader): implements = {datatypes.LlamaCPPService} imports = {"requests"} output_instance = "builtins:str" def _read(self, data, prompt: str = "", *args, **kwargs): import requests r = requests.post( f"{data.url}/embedding", json={"content": prompt, **kwargs}, headers={"Content-Type": "application/json"}, ) return r.json()["embedding"] class OpenAIReader(BaseReader): implements = {datatypes.OpenAIService} imports = {"openai"} output_instance = "openai:OpenAI" def _read(self, data, **kwargs): import openai client = openai.Client(api_key=data.key, base_url=data.url, **kwargs) return client class OpenAICompletion(BaseReader): implements = {datatypes.OpenAIService} imports = {"requests"} output_instance = "builtins:dict" # related high-volume endpoints, assistant, embeddings def _read(self, data, messages: list[dict], *args, model="gtp-3.5-turbo", **kwargs): import requests url = f"{data.url}/v1/chat/completions" options = data.options.copy() options.update(kwargs) r = requests.get( url, headers={"Content-Type": "application/json", "Authorization": f"Bearer {data.key}"}, json=dict(messages=messages, **options), ) return r.json()["choices"][0]["message"] class TorchDataset(BaseReader): output_instance = "torch.utils.data:Dataset" def _read(self, modname, funcname, rootdir, **kw): import importlib mod = importlib.import_module(f"torch{modname}") func = getattr(mod.datasets, funcname) try: return func(rootdir, download=True) except TypeError: return func(rootdir) class TFPublicDataset(BaseReader): # contains ({split: tensorflow.data.Dataset}, data_info) by default output_instance = "builtins:tuple" func = "tensorflow_datasets:load" def _read(self, name, *args, **kwargs): return self._func(name, download=True, with_info=True, **kwargs) class TFTextreader(FileReader): imports = {"tensorflow"} implements = {datatypes.Text} func = "tensorflow.data:TextLineDataset" output_instance = "tensorflow.data:Dataset" url_arg = "filenames" class TFORC(FileReader): imports = {"tensorflow_io"} implements = {datatypes.ORC} func = "tensorflow_io:IODataset.from_orc" url_arg = "filename" output_instance = "tensorflow.data:Dataset" class TFSQL(BaseReader): imports = {"tensorflow_io"} implements = {datatypes.SQLQuery} func = "tensorflow_io:experimental.IODataset.from_sql" output_instance = "tensorflow.data:Dataset" def _read(self, data, **kwargs): return self._func(endpoint=data.conn, query=data.query, **kwargs) class KerasImageReader(FileReader): imports = {"keras"} implements = {datatypes.PNG, datatypes.JPEG} # others func = "keras.utils:image_dataset_from_directory" output_instance = "tensorflow.data:Dataset" url_arg = "directory" class KerasText(FileReader): imports = {"keras"} implements = {datatypes.Text} func = "keras.utils:text_dataset_from_directory" output_instance = "tensorflow.data:Dataset" url_arg = "directory" class KerasAudio(FileReader): imports = {"keras"} implements = {datatypes.WAV} func = "keras.utils:audio_dataset_from_directory" output_instance = "tensorflow.data:Dataset" url_arg = "directory" class KerasModelReader(FileReader): imports = {"keras"} implements = {datatypes.KerasModel} func = "tensorflow.keras.models:load_model" url_arg = "filepath" output_instance = "keras.engine.training:Model" class TFRecordReader(FileReader): imports = {"tensorflow"} implements = {datatypes.TFRecord} func = "tensorflow.data:TFRecordDataset" output_instance = "tensorflow.data:TFRecordDataset" url_arg = "filenames" class SKLearnModelReader(FileReader): # https://scikit-learn.org/stable/model_persistence.html # recommends skops, which seems little used imports = {"sklearn"} implements = {datatypes.SKLearnPickleModel} func = "pickle:load" output_instance = "sklearn.base:BaseEstimator" def _read(self, data, **kw): with fsspec.open(data.url, **(data.storage_options or {})) as f: return self._func(f) class Awkward(FileReader): imports = {"awkward"} output_instance = "awkward:Array" storage_options = True class AwkwardParquet(Awkward): implements = {datatypes.Parquet} imports = {"awkward", "pyarrow"} func = "awkward:from_parquet" url_arg = "path" def discover(self, **kwargs): kwargs["row_groups"] = [0] return self.read(**kwargs) class DaskAwkwardParquet(AwkwardParquet): imports = {"dask_awkward", "pyarrow", "dask"} func = "dask_awkward:from_parquet" output_instance = "dask_awkward:Array" def discover(self, **kwargs): return self.read(**kwargs).partitions[0] class AwkwardJSON(Awkward): implements = {datatypes.JSONFile} func = "awkward:from_json" url_arg = "source" class AwkwardAVRO(Awkward): implements = {datatypes.AVRO} func = "awkward:from_avro_file" url_arg = "file" class DaskAwkwardJSON(Awkward): imports = {"dask_awkward", "dask"} func = "dask_awkward:from_json" output_instance = "dask_awkward:Array" url_arg = "source" def discover(self, **kwargs): return self.read(**kwargs).partitions[0] class HandleToUrlReader(BaseReader): """Dereference handle (hdl:) identifiers See handle.net for a description of the registry. """ implements = {datatypes.Handle} func = "requests:get" imports = {"requests", "aiohttp"} output_instance = datatypes.BaseData.qname() @classmethod def _extract(cls, meta, base): h = fsspec.filesystem("http") if "URL_ORIGINAL_DATA" in meta: # file url = re.findall('href="(.*?)"', meta["URL_ORIGINAL_DATA"]["value"])[0] elif "HAS_PARTS" in meta: # dataset ids = meta["HAS_PARTS"]["value"].split(";") rr = h.cat([f"{base}/{u.lstrip('hdl:/')}" for u in ids]) rr2 = [{i["type"]: i["data"] for i in json.loads(r)["values"]} for r in rr.values()] url = [cls._extract(r2, base) for r2 in rr2] return url def _read(self, data, base="https://hdl.handle.net/api/handles", **kwargs): h = fsspec.filesystem("http") r = h.cat(f"{base}/{data.url.lstrip('hdl:/')}") j = json.loads(r) meta = {i["type"]: i["data"] for i in j["values"]} url = self._extract(meta, base) # TODO: we can assume HDF->xarray here? cls = datatypes.recommend(url[0] if isinstance(url, list) else url)[0] return cls(url=url, metadata=meta) class PandasCSV(Pandas): implements = {datatypes.CSV} func = "pandas:read_csv" url_arg = "filepath_or_buffer" def discover(self, **kw): kw["nrows"] = 10 kw.pop("skipfooter", None) kw.pop("chunksize", None) return self.read(**kw) class PandasHDF5(Pandas): implements = {datatypes.HDF5} func = "pandas:read_hdf" imports = {"pandas", "pytables"} def _read(self, data, **kw): if data.storage_options: # or fsspec-like with fsspec.open(data.url, "rb", **data.storage_options) as f: self._func(f, data.path, **kw) return self._func(data.url, **kw) class DaskCSV(DaskDF): implements = {datatypes.CSV} func = "dask.dataframe:read_csv" url_arg = "urlpath" class DaskText(FileReader): imports = {"dask"} implements = {datatypes.Text} func = "dask.bag:read_text" output_instance = "dask.bag.core:Bag" storage_options = True url_arg = "urlpath" def discover(self, n=10, **kwargs): return self.read().take(n) class DaskCSVPattern(DaskCSV): """Apply categorical data extraction to a set of CSV paths using dask Paths are of the form "proto://path/{field}/measurement_{date:%Y-%m-%d}.csv", where the format-like fields will be captured as columns in the output. """ implements = {datatypes.CSVPattern} def _read(self, data, **kw): from pandas.api.types import CategoricalDtype from intake.readers.utils import pattern_to_glob from intake.source.utils import reverse_formats url = pattern_to_glob(data.url) df = self._func(url, storage_options=data.storage_options, include_path_column=True, **kw) paths = sorted(df["path"].cat.categories) column_by_field = { field: df["path"] .cat.codes.map(dict(enumerate(values))) .astype(CategoricalDtype(set(values))) for field, values in reverse_formats(data.url, paths).items() } return df.assign(**column_by_field).drop(columns=["path"]) class Polars(FileReader): imports = {"polars"} output_instance = "polars:LazyFrame" url_arg = "source" def discover(self, **kwargs): # https://pola-rs.github.io/polars/py-polars/html/reference/ # lazyframe/api/polars.LazyFrame.fetch.html return self.read().fetch() class PolarsDeltaLake(Polars): implements = {datatypes.DeltalakeTable} func = "polars:scan_delta" class PolarsAvro(Polars): implements = {datatypes.AVRO} func = "polars:read_avro" output_instance = "polars:DataFrame" # i.e., not lazy class PolarsFeather(Polars): implements = {datatypes.Feather2} func = "polars:scan_ipc" class PolarsParquet(Polars): implements = {datatypes.Parquet} func = "polars:scan_parquet" class PolarsCSV(Polars): implements = {datatypes.CSV} func = "polars:scan_csv" class PolarsJSON(Polars): implements = {datatypes.JSONFile} func = "polars:scan_ndjson" class PolarsIceberg(Polars): imports = {"polars", "pyiceberg"} implements = {datatypes.IcebergDataset} func = "polars:scan_iceberg" class PolarsExcel(Polars): implements = {datatypes.Excel} func = "polars:read_excel" output_instance = "polars:DataFrame" # i.e., not lazy class Ray(FileReader): # https://docs.ray.io/en/latest/data/creating-datasets.html#supported-file-formats imports = {"ray"} output_instance = "ray.data:Dataset" url_arg = "paths" def discover(self, **kwargs): return self.read(**kwargs).limit(10) def _read(self, data, **kw): if ( data.url.startswith("s3://") and data.storage_options and data.storage_options.get("anon") ): data = type(data)(url=f"s3://anonymous@{data.url[5:]}") # TODO: other auth parameters, key/secret, token # apparently, creating an S3FileSystem here is also allowed return super()._read(data, **kw) class RayParquet(Ray): implements = {datatypes.Parquet} func = "ray.data:read_parquet" class RayCSV(Ray): implements = {datatypes.CSV} func = "ray.data:read_csv" class RayJSON(Ray): implements = {datatypes.JSONFile} func = "ray.data:read_json" class RayText(Ray): implements = {datatypes.Text} func = "ray.data:read_text" class RayBinary(Ray): implements = {datatypes.FileData} func = "ray.data:read_binary_files" class RayDeltaLake(Ray): implements = {datatypes.DeltalakeTable} imports = {"deltaray"} func = "deltaray:read_delta" url_arg = "table_uri" class DeltaReader(FileReader): implements = {datatypes.Parquet, datatypes.DeltalakeTable} imports = {"deltalake"} func = "deltalake:DeltaTable" url_arg = "table_uri" storage_options = True output_instance = "deltalake:DeltaTable" class TiledNode(BaseReader): implements = {datatypes.TiledService} imports = {"tiled"} output_instance = "tiled.client.node:Node" func = "tiled.client:from_uri" def _read(self, data, **kwargs): opts = data.options.copy() opts.update(kwargs) return self._func(data.url, **opts) class TiledClient(BaseReader): # returns dask/normal x/array/dataframe implements = {datatypes.TiledDataset} output_instance = "tiled.client.base:BaseClient" def _read(self, data, as_client=True, dask=False, **kwargs): from tiled.client import from_uri opts = data.options.copy() opts.update(kwargs) if dask: opts["structure_clients"] = "dask" client = from_uri(data.url, **opts) if as_client: return client else: return client.read() class TileDBReader(BaseReader): imports = {"tiledb"} implements = {datatypes.TileDB} output_instance = "tiledb.libtiledb.Array" func = "tiledb:open" def _read(self, data, attribute=None, **kwargs): return self._func(data.url, attr=attribute, config=data.options, **kwargs) class TileDBDaskReader(BaseReader): imports = {"tiledb", "dask"} func = "dask.array:from_tiledb" implements = {datatypes.TileDB} output_instance = "dask.array:Array" def _read(self, data, attribute=None, **kwargs): return self._func(data.url, attribute=attribute, config=data.options, **kwargs) class PythonModule(BaseReader): output_instance = "builtins:module" implements = {datatypes.PythonSourceCode} def _read(self, data, module_name=None, **kwargs): from types import ModuleType if module_name is None: module_name = data.url.rsplit("/", 1)[-1].split(".", 1)[0] with fsspec.open(data.url, "rt", **(data.storage_options or {})) as f: mod = ModuleType(module_name) exec(f.read(), mod.__dict__) return mod class SKImageReader(FileReader): output_instance = "numpy:ndarray" imports = {"scikit-image"} implements = {datatypes.PNG, datatypes.TIFF, datatypes.JPEG} func = "skimage.io:imread" url_arg = "fname" class NumpyText(FileReader): output_instance = "numpy:ndarray" implements = {datatypes.Text} imports = {"numpy"} func = "numpy:loadtxt" def _read(self, data, **kw): if data.storage_options or "://" in data.url or "::" in data.url: with fsspec.open(data.url, **(data.storage_options or {})) as f: return self._func(f, **kw) return self._func(data.url, **kw) class NumpyReader(NumpyText): func = "numpy:load" implements = {datatypes.NumpyFile} class CupyNumpyReader(NumpyText): output_instance = "cupy:ndarray" implements = {datatypes.NumpyFile} imports = {"cupy"} func = "cupy:loadtxt" class CupyTextReader(CupyNumpyReader): implements = {datatypes.Text} func = "numpy:loadtxt" class XArrayDatasetReader(FileReader): output_instance = "xarray:Dataset" imports = {"xarray"} optional_imports = {"zarr", "h5netcdf", "cfgrib", "scipy", "tiledb"} # and others # DAP is not a file but an API, maybe should be separate implements = { datatypes.NetCDF3, datatypes.HDF5, datatypes.GRIB2, datatypes.IcechunkRepo, datatypes.Zarr, datatypes.OpenDAP, datatypes.TileDB, } # xarray also reads from images and tabular data func = "xarray:open_mfdataset" other_funcs = {"xarray:open_dataset"} other_urls = {"xarray:open_dataset": "filename_or_obj"} url_arg = "paths" def _read(self, data, open_local=False, **kw): from xarray import open_dataset, open_mfdataset if "engine" not in kw: if isinstance(data, (datatypes.Zarr, datatypes.IcechunkRepo)): kw["engine"] = "zarr" if data.root and "group" not in kw: kw["group"] = data.root elif isinstance(data, datatypes.TileDB): kw["engine"] = "tiledb" if data.options: kw.setdefault("backend_kwargs", {})["config"] = data.options elif isinstance(data, datatypes.NetCDF3): kw["engine"] = "scipy" elif isinstance(data, datatypes.HDF5): kw["engine"] = "h5netcdf" if kw.get("engine", "") in ["zarr", "kerchunk"] and data.storage_options: kw.setdefault("backend_kwargs", {})["storage_options"] = data.storage_options if isinstance(data, (datatypes.HDF5, datatypes.NetCDF3)): kw.setdefault("engine", "h5netcdf") if getattr(data, "path", False): kw["group"] = data.path if isinstance(data, datatypes.IcechunkRepo): import icechunk url = f"{data.url}_storage" if "storage" not in data.url else data.url store_cls = getattr(icechunk, url) store = store_cls(**(data.storage_options or {})) repo = icechunk.Repository.open(store) session = repo.readonly_session(data.ref) zarr_store = session.store kw.get("backend_kwargs", {}).pop("storage_options", None) kw.setdefault("backend_kwargs", {})["consolidated"] = False return open_dataset(zarr_store, **kw) auth = kw.pop("auth", "") if isinstance(data, datatypes.OpenDAP) and auth and kw.get("engine", "") == "pydap": import requests if isinstance(auth, str): setup = intake.import_name(f"pydap.cas.{auth}:setup_session") un = kw.pop("username", os.getenv("DAP_USER")) pw = kw.pop("password", os.getenv("DAP_PASSWORD")) session = setup(un, pw, check_url=data.url) else: session = requests.Session() session.auth = auth return open_dataset(data.url, session=session, **kw) if isinstance(data.url, (tuple, set, list)) and len(data.url) == 1: return open_dataset(data.url[0], **kw) elif (isinstance(data.url, (tuple, set, list)) and len(data.url) > 1) or ( isinstance(data.url, str) and "*" in data.url ): if isinstance(data, (datatypes.Zarr, datatypes.OpenDAP)): ofs = data.url elif open_local: ofs = fsspec.open_local(data.url, **(data.storage_options or {})) elif (isinstance(data.url, str) and is_fsspec_url(data.url)) or is_fsspec_url( data.url[0] ): ofs = [ _.open() for _ in fsspec.open_files(data.url, **(data.storage_options or {})) ] else: ofs = data.url return open_mfdataset(ofs, **kw) else: if ( isinstance(data, datatypes.FileData) and is_fsspec_url(data.url) and not isinstance(data, datatypes.Zarr) ): # special case, because xarray would assume a DAP endpoint if open_local: f = fsspec.open_local(data.url, **(data.storage_options or {})) return open_dataset(f, **kw) else: f = fsspec.open(data.url, **(data.storage_options or {})).open() return open_dataset(f, **kw) return open_dataset(data.url, **kw) class XArrayPatternReader(XArrayDatasetReader): """Same as XarrayDatasetReader, but recognises file patterns If you use a URL like "/path/file_{value}_.nc". The template may include specifiers like ":d" to determine the type of the values inferred. This reader supports all the same filetypes as XArrayDatasetReader. The read step may be accelerated by providing arguments like ``parallel=True`` and ``combine_attrs="override" - see the xr.open_mfdataset documentation. Note: this method determined the ``concat_dim`` and ``combine`` arguments, so passing these will raise an exception. """ # should we have an explicit pattern type data input? def _read(self, data, open_local=False, pattern=None, **kw): import pandas as pd from intake.readers.utils import pattern_to_glob from intake.source.utils import reverse_formats if isinstance(data.url, str): url = pattern_to_glob(data.url) fs, _, paths = fsspec.get_fs_token_paths(url, **(data.storage_options or {})) val_dict = reverse_formats(data.url, paths) else: paths = data.url val_dict = reverse_formats(pattern, data.url) indices = [pd.Index(v, name=k) for k, v in val_dict.items()] data2 = type(data)(url=paths, storage_options=data.storage_options, metadata=data.metadata) if "concat_dim" in kw: ccm = kw.pop("concat_dim") ccm = [ccm] if isinstance(ccm, str) else ccm for ind, cd in zip(indices, ccm): ind.name = cd kw.setdefault("combine", "nested") return super()._read(data2, concat_dim=indices, open_local=open_local, **kw) class RasterIOXarrayReader(FileReader): output_instance = "xarray:Dataset" imports = {"rioxarray"} implements = {datatypes.TIFF, datatypes.GDALRasterFile} func = "rioxarray:open_rasterio" url_arg = "filename" def _read(self, data, concat_kwargs=None, **kwargs): import xarray as xr from rioxarray import open_rasterio concat_kwargs = concat_kwargs or { k: kwargs.pop(k) for k in {"dim", "data_vars", "coords", "compat", "position", "join"} if k in kwargs } ofs = fsspec.open_files(data.url, **(data.storage_options or {})) bits = [open_rasterio(of.open(), **kwargs) for of in ofs] if len(bits) == 1: return bits else: # requires dim= in kwargs return xr.concat(bits, **concat_kwargs) class GeoPandasReader(FileReader): # TODO: geopandas also supports postGIS output_instance = "geopandas:GeoDataFrame" imports = {"geopandas"} implements = { datatypes.GeoJSON, datatypes.CSV, datatypes.SQLite, datatypes.Shapefile, datatypes.GDALVectorFile, datatypes.GeoPackage, datatypes.FlatGeoBuf, } func = "geopandas:read_file" url_arg = "filename" def _read(self, data, with_fsspec=None, **kwargs): import geopandas if with_fsspec is None: with_fsspec = ( ("://" in data.url and "!" not in data.url) or "::" in data.url or data.storage_options ) if with_fsspec: with fsspec.open(data.url, **(data.storage_options or {})) as f: return geopandas.read_file(f, **kwargs) return geopandas.read_file(data.url, **kwargs) class GeoPandasTabular(FileReader): output_instance = "geopandas:GeoDataFrame" imports = {"geopandas", "pyarrow"} implements = {datatypes.Parquet, datatypes.Feather2} func = "geopandas:read_parquet" other_funcs = {"geopandas:read_feather"} url_arg = "path" def _read(self, data, **kwargs): import geopandas if "://" in data.url or "::" in data.url: f = fsspec.open(data.url, **(data.storage_options or {})).open() else: f = data.url if isinstance(data, datatypes.Parquet): return geopandas.read_parquet(f, **kwargs) elif isinstance(data, datatypes.Feather2): return geopandas.read_feather(f, **kwargs) else: raise ValueError class ScipyMatlabReader(FileReader): output_instance = "numpy:ndarray" implements = {datatypes.MatlabArray} imports = {"scipy"} func = "scipy.io:loadmat" def _read(self, data, **kwargs): return self._func(data.path, appendmat=False, **kwargs)[data.variable] class ScipyMatrixMarketReader(FileReader): output_instance = "scipy.sparse:coo_matrix" # numpy-like implements = {datatypes.MatrixMarket} imports = {"scipy"} func = "scipy.io:mmread" def _read(self, data, **kw): with fsspec.open(data.url, **data.storage_options) as f: return self._func(f) class NibabelNiftiReader(FileReader): output_instance = "nibabel.spatialimages:SpatialImage" implements = {datatypes.Nifti} # and other medical image types imports = {"nibabel"} func = "nibabel:load" url_arg = "filename" def _read(self, data, **kw): with fsspec.open(data.url, **(data.storage_options or {})) as f: return self._func(f, **kw) class FITSReader(FileReader): output_instance = "astropy.io.fits:HDUList" implements = {datatypes.FITS} imports = {"astropy"} func = "astropy.io.fits:open" def _read(self, data, **kw): if data.storage_options: kw.pop("use_fsspec") kw.pop("fsspec_kwargs") return self._func(data.url, use_fsspec=True, fsspec_kwargs=data.storage_options, **kw) return self._func(data.url, **kw) class ASDFReader(FileReader): implements = {datatypes.ASDF} imports = {"asdf"} func = "asdf:open" output_instance = "asdf:AsdfFile" def _read(self, data, **kw): if data.storage_options or "://" in data.url or "::" in data.url: # want the file to stay open, since array access is lazy by default f = fsspec.open(data.url, **(data.storage_options or {})).open() return self._func(f, **kw) return self._func(data.url, **kw) class DicomReader(FileReader): output_instance = "pydicom.dataset:FileDataset" implements = {datatypes.DICOM} imports = {"pydicom"} func = "pydicom:read_file" url_arg = "fp" # can be file-like storage_options = True def _read(self, data, **kw): with fsspec.open(data.url, **(data.storage_options or {})) as f: return self._func(f, **kw) class Condition(BaseReader): def _read( self, if_true, if_false, condition: callable[[BaseReader, ...], bool] | bool, **kwargs ): if isinstance(condition, bool): cond = condition elif isinstance(condition, BaseReader): cond = condition.read() else: cond = condition(**kwargs) if cond: return if_true.read() if isinstance(if_true, BaseReader) else if_true else: return if_false.read() if isinstance(if_false, BaseReader) else if_false class PMTileReader(BaseReader): implements = {datatypes.PMTiles} func = "pmtiles.reader:Reader" output_instance = "pmtiles.reader:Reader" def _read(self, data): import pmtiles.reader if "://" in data.url or "::" in data.url: f = fsspec.open(data.url, **(data.storage_options or {})).open() def get_bytes(offset, length): f.seek(offset) return f.read(length) else: f = open(data.url) get_bytes = pmtiles.reader.MmapSource(f) return self._func(get_bytes) class FileExistsReader(BaseReader): implements = {datatypes.FileData} func = "fsspec.core:url_to_fs" output_instance = "builtins:bool" def _read(self, data, *args, **kwargs): try: fs, path = fsspec.core.url_to_fs(data.url, **(data.storage_options or {})) except FileNotFoundError: return False return fs.exists(path) class YAMLCatalogReader(FileReader): implements = {datatypes.YAMLFile, datatypes.YAMLFile} func = "intake.readers.entry:Catalog.from_yaml_file" url_arg = "path" storage_options = True output_instance = "intake.readers.entry:Catalog" class PrometheusMetricReader(BaseReader): implements = {datatypes.Prometheus} imports = {"prometheus_api_client"} output_instance = "typing:Iterator" func = "prometheus_api_client:custom_query" other_funcs = {"prometheus_api_client:get_metric_range_data"} def _read(self, data: datatypes.Prometheus, *args, **kwargs): from prometheus_api_client import PrometheusConnect from prometheus_api_client.utils import parse_datetime prom = PrometheusConnect(url=data.url, **(data.options or {})) if data.query: # this is a catalog, should be separate reader? return prom.custom_query(data.query, **kwargs) if not data.metric: return prom.all_metrics(**kwargs) start_time = parse_datetime(data.start_time) if data.start_time else parse_datetime("1900") end_time = parse_datetime(data.end_time) if data.end_time else parse_datetime("now") return prom.get_metric_range_data( data.metric, label_config=data.labels, start_time=start_time, end_time=end_time, **kwargs, ) class Retry(BaseReader): """Retry (part of) a pipeline until it returns without exception Retries the whole of the selected pipeline; an exception will start at the beginning. """ def _read( self, data, max_tries=10, allowed_exceptions=(Exception,), backoff0=0.1, backoff_factor=1.3, start_stage=None, **kw, ): """ Parameters ---------- data: intake pipeline/reader max_tries: number of attempts that can be made allowed_exceptions: tuple of Exceptions we will try again for; others will raise start_stage: if given, index of pipeline member stage to start from for retries (else all); may be negative from the most recent previous stage (-1). """ import time if isinstance(allowed_exceptions, (list, set)): allowed_exceptions = tuple(allowed_exceptions) reader = data if isinstance(data, BaseReader) else data.reader if start_stage and start_stage < 0: start_stage = len(reader.steps) + start_stage if start_stage: data = reader.first_n_stages(start_stage).read() else: data = None start_stage = 0 for j in range(max_tries): try: for i in range(start_stage, len(reader.steps)): if i == 0: data = reader._read_stage_n(stage=0) else: data = reader._read_stage_n(stage=1, data=data) except allowed_exceptions: if j == max_tries: raise time.sleep(backoff0 * backoff_factor**i) return data def recommend(data): """Show which readers claim to support the given data instance or a superclass The ordering is more specific readers first """ seen = set() out = {"importable": [], "not_importable": []} data = type(data) if not isinstance(data, type) else data for datacls in data.mro(): for cls in subclasses(BaseReader): if any(datacls == imp for imp in cls.implements): if cls not in seen: seen.add(cls) if cls.check_imports(): out["importable"].append(cls) else: out["not_importable"].append(cls) return out def reader_from_call(func: str, *args, join_lines=False, **kwargs) -> BaseReader: """Attempt to construct a reader instance by finding one that matches the function call Fails for readers that don't define a func, probably because it depends on the file type or needs a dynamic instance to be a method of. Parameters ---------- func: callable | str If a callable, pass args and kwargs as you would have done to execute the function. If a string, it should look like ``"func(arg1, args2, kwarg1, **kw)"``, i.e., a normal python call but as a string. In the latter case, args and kwargs are ignored """ import re from itertools import chain if isinstance(func, str): if join_lines: func = func.replace("\n", "") frame = inspect.currentframe().f_back match = re.match("^([^(]+=)?([^(]+)[(](.*)[)][^)]?$", func) if match: groups = match.groups() else: raise ValueError func = eval(groups[1], frame.f_globals, frame.f_locals) args, kwargs = eval( f"""(lambda *args, **kwargs: (args, kwargs))({groups[2]})""", frame.f_globals, frame.f_locals, ) package = func.__module__.split(".", 1)[0] found = False for cls in subclasses(BaseReader): if cls.check_imports() and any( f.split(":", 1)[0].split(".", 1)[0] == package for f in ({cls.func} | cls.other_funcs) ): ffs = [f for f in ({cls.func} | cls.other_funcs) if import_name(f) == func] if ffs: found = cls func_name = ffs[0] break if not found: raise ValueError("Function not found in the set of readers") pars = inspect.signature(func).parameters kw = dict(zip(pars, args), **kwargs) data_kw = {} if issubclass(cls, FileReader): data_kw["storage_options"] = kw.pop("storage_options", None) data_kw["url"] = kw.pop(getattr(cls, "other_urls", {}).get(func_name, cls.url_arg)) datacls = None if len(cls.implements) == 1: datacls = next(iter(cls.implements)) elif getattr(cls, "url_arg", None): clss = datatypes.recommend(data_kw["url"], storage_options=data_kw["storage_options"]) clss2 = [c for c in clss if c in cls.implements] if clss: datacls = next(iter(chain(clss2, clss))) if datacls: datacls = datacls(**data_kw) if data_kw["storage_options"] is None: del data_kw["storage_options"] cls = cls(datacls, **kwargs) else: url = data_kw.pop("url") cls = cls(url, **data_kw) return cls ================================================ FILE: intake/readers/search.py ================================================ """Find datasets meeting some complex criteria""" from __future__ import annotations from intake.readers.entry import ReaderDescription # some inspiration https://blueskyproject.io/tiled/reference/queries.html class SearchBase: """Prototype for a single term in a search expression The method `filter()` is meant to be overridden in subclasses. """ def filter(self, entry: ReaderDescription) -> bool: """Does the given ReaderDescription entry match the query?""" # should not raise: an exception counts as False return True def __or__(self, other): return Or(self, other) def __and__(self, other): return And(self, other) def __inv__(self): return Not(self) class Or(SearchBase): def __init__(self, first: SearchBase, second: SearchBase): self.first = first self.second = second def filter(self, entry: ReaderDescription) -> bool: return self.first.filter(entry) or self.second.filter(entry) class And(SearchBase): def __init__(self, first: SearchBase, second: SearchBase): self.first = first self.second = second def filter(self, entry: ReaderDescription) -> bool: return self.first.filter(entry) and self.second.filter(entry) class Not(SearchBase): def __init__(self, first: SearchBase): self.first = first def filter(self, entry: ReaderDescription) -> bool: return not self.first.filter(entry) class Any(SearchBase): def __init__(self, *terms: tuple[SearchBase, ...]): self.terms = terms def filter(self, entry: ReaderDescription) -> bool: return any(t.filter(entry) for t in self.terms) class All(SearchBase): def __init__(self, *terms: tuple[SearchBase, ...]): self.terms = terms def filter(self, entry: ReaderDescription) -> bool: return all(t.filter(entry) for t in self.terms) class Text(SearchBase): """Search for given string anywhere in the text repr of an entry.""" def __init__(self, text: str): self.text = text def filter(self, entry: ReaderDescription) -> bool: return self.text in str(entry) class Importable(SearchBase): """Check if the packages listed in "imports" field exist in the environment This only checks for top-level packages, and does not actually import anything. If any package is not found, the filter fails. """ def filter(self, entry: ReaderDescription) -> bool: return entry.check_imports() class EnvironmentSatisfied(SearchBase): """Compare the "environment" metadata field to the current information in conda The value of "environment" can be a URL to load from (and environment.yml file), or literal YAML text. See https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html for specs. The filter passes if the environment is satisfied, i.e., the packages and versions in the current environment are allowed by the spec. """ def filter(self, entry: ReaderDescription) -> bool: env = entry.metadata.get("environment") if not env: # no env restrictions means a pass return True return self._is_consistent(env) @staticmethod def _is_consistent(env, output=False): # TODO: this is quite slow, should cache? import fsspec import os import subprocess import tempfile import shlex fn = tempfile.mktemp(suffix=".yaml") try: if "dependencies:" not in env: with fsspec.open(env, "rt") as f: env = f.read() with open(fn, "wt") as f: f.write(env) cmd = shlex.split(f"conda compare {fn}") kw = {} if output else dict(stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) out = subprocess.check_call(cmd, **kw) return out == 0 except Exception: return False finally: if os.path.exists(fn): os.remove(fn) ================================================ FILE: intake/readers/tests/__init__.py ================================================ ================================================ FILE: intake/readers/tests/cats/__init__.py ================================================ ================================================ FILE: intake/readers/tests/cats/stac_data/1.0.0/catalog/catalog.json ================================================ { "type": "Catalog", "id": "test", "stac_version": "1.0.0", "description": "test catalog", "links": [ { "rel": "child", "href": "./child-catalog.json", "type": "application/json" }, { "rel": "root", "href": "./catalog.json", "type": "application/json" } ], "stac_extensions": [] } ================================================ FILE: intake/readers/tests/cats/stac_data/1.0.0/catalog/child-catalog.json ================================================ { "type": "Catalog", "id": "test", "stac_version": "1.0.0", "description": "child catalog", "links": [ { "rel": "root", "href": "./catalog.json", "type": "application/json" } ], "stac_extensions": [] } ================================================ FILE: intake/readers/tests/cats/stac_data/1.0.0/collection/collection.json ================================================ { "id": "simple-collection", "type": "Collection", "stac_extensions": [ "https://stac-extensions.github.io/eo/v1.0.0/schema.json", "https://stac-extensions.github.io/view/v1.0.0/schema.json" ], "stac_version": "1.0.0", "description": "A simple collection demonstrating core catalog fields with links to a couple of items", "title": "Simple Example Collection", "providers": [ { "name": "Remote Data, Inc", "description": "Producers of awesome spatiotemporal assets", "roles": ["producer", "processor"], "url": "http://remotedata.io" } ], "extent": { "spatial": { "bbox": [ [ 172.91173669923782, 1.3438851951615003, 172.95469614953714, 1.3690476620161975 ] ] }, "temporal": { "interval": [["2020-12-11T22:38:32.125Z", "2020-12-14T18:02:31.437Z"]] } }, "license": "CC-BY-4.0", "summaries": { "platform": ["cool_sat1", "cool_sat2"], "constellation": ["ion"], "instruments": ["cool_sensor_v1", "cool_sensor_v2"], "gsd": { "minimum": 0.512, "maximum": 0.66 }, "eo:cloud_cover": { "minimum": 1.2, "maximum": 1.2 }, "proj:epsg": { "minimum": 32659, "maximum": 32659 }, "view:sun_elevation": { "minimum": 54.9, "maximum": 54.9 }, "view:off_nadir": { "minimum": 3.8, "maximum": 3.8 }, "view:sun_azimuth": { "minimum": 135.7, "maximum": 135.7 } }, "links": [ { "rel": "root", "href": "./collection.json", "type": "application/json" }, { "rel": "item", "href": "./simple-item.json", "type": "application/geo+json", "title": "Simple Item" } ] } ================================================ FILE: intake/readers/tests/cats/stac_data/1.0.0/collection/simple-item.json ================================================ { "stac_version": "1.0.0", "stac_extensions": [ "https://stac-extensions.github.io/projection/v1.0.0/schema.json", "https://stac-extensions.github.io/eo/v1.0.0/schema.json" ], "type": "Feature", "id": "S2B_MSIL2A_20171227T160459_N0212_R054_T17QLA_20201014T165101", "bbox": [ 172.91173669923782, 1.3438851951615003, 172.95469614953714, 1.3690476620161975 ], "geometry": { "coordinates": [ [ [-82.89978, 18.98277161], [-81.85693, 18.99053787], [-81.85202, 17.99825755], [-82.888855, 17.99092482], [-82.89978, 18.98277161] ] ], "type": "Polygon" }, "properties": { "datetime": "2017-12-27T16:04:59.027000Z" }, "collection": "simple-collection", "links": [ { "rel": "collection", "href": "./collection.json", "type": "application/json", "title": "Simple Example Collection" }, { "rel": "root", "href": "./collection.json", "type": "application/json" }, { "rel": "parent", "href": "./collection.json", "type": "application/json" } ], "assets": { "B02": { "href": "https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/17/Q/LA/2017/12/27/S2B_MSIL2A_20171227T160459_N0212_R054_T17QLA_20201014T165101.SAFE/GRANULE/L2A_T17QLA_A004227_20171227T160750/IMG_DATA/R10m/T17QLA_20171227T160459_B02_10m.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "title": "Band 2 - Blue", "eo:bands": [ { "name": "B02", "common_name": "blue", "description": "Band 2 - Blue", "center_wavelength": 0.49, "full_width_half_max": 0.098 } ], "gsd": 10, "proj:shape": [10980, 10980], "proj:bbox": [300000, 1990200, 409800, 2100000], "proj:transform": [10, 0, 300000, 0, -10, 2100000], "roles": ["data"] }, "B03": { "href": "https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/17/Q/LA/2017/12/27/S2B_MSIL2A_20171227T160459_N0212_R054_T17QLA_20201014T165101.SAFE/GRANULE/L2A_T17QLA_A004227_20171227T160750/IMG_DATA/R10m/T17QLA_20171227T160459_B03_10m.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "title": "Band 3 - Green", "eo:bands": [ { "name": "B03", "common_name": "green", "description": "Band 3 - Green", "center_wavelength": 0.56, "full_width_half_max": 0.045 } ], "gsd": 10, "proj:shape": [10980, 10980], "proj:bbox": [300000, 1990200, 409800, 2100000], "proj:transform": [10, 0, 300000, 0, -10, 2100000], "roles": ["data"] } } } ================================================ FILE: intake/readers/tests/cats/stac_data/1.0.0/collection/zarr-collection.json ================================================ { "type": "Collection", "id": "daymet-daily-hi", "stac_version": "1.0.0", "description": "{{ collection.description }}", "links": [ { "rel": "license", "href": "https://science.nasa.gov/earth-science/earth-science-data/data-information-policy" } ], "stac_extensions": [ "https://stac-extensions.github.io/datacube/v2.0.0/schema.json" ], "cube:dimensions": { "time": { "type": "temporal", "description": "24-hour day based on local time", "extent": ["1980-01-01T12:00:00Z", "2020-12-30T12:00:00Z"] }, "x": { "type": "spatial", "axis": "x", "description": "x coordinate of projection", "extent": [-5802250.0, -5519250.0], "step": 1000.0, "reference_system": { "$schema": "https://proj.org/schemas/v0.2/projjson.schema.json", "type": "ProjectedCRS", "name": "undefined", "base_crs": { "name": "undefined", "datum": { "type": "GeodeticReferenceFrame", "name": "undefined", "ellipsoid": { "name": "undefined", "semi_major_axis": 6378137, "inverse_flattening": 298.257223563 } }, "coordinate_system": { "subtype": "ellipsoidal", "axis": [ { "name": "Longitude", "abbreviation": "lon", "direction": "east", "unit": "degree" }, { "name": "Latitude", "abbreviation": "lat", "direction": "north", "unit": "degree" } ] } }, "conversion": { "name": "unknown", "method": { "name": "Lambert Conic Conformal (2SP)", "id": { "authority": "EPSG", "code": 9802 } }, "parameters": [ { "name": "Latitude of 1st standard parallel", "value": 25, "unit": "degree", "id": { "authority": "EPSG", "code": 8823 } }, { "name": "Latitude of 2nd standard parallel", "value": 60, "unit": "degree", "id": { "authority": "EPSG", "code": 8824 } }, { "name": "Latitude of false origin", "value": 42.5, "unit": "degree", "id": { "authority": "EPSG", "code": 8821 } }, { "name": "Longitude of false origin", "value": -100, "unit": "degree", "id": { "authority": "EPSG", "code": 8822 } }, { "name": "Easting at false origin", "value": 0, "unit": "metre", "id": { "authority": "EPSG", "code": 8826 } }, { "name": "Northing at false origin", "value": 0, "unit": "metre", "id": { "authority": "EPSG", "code": 8827 } } ] }, "coordinate_system": { "subtype": "Cartesian", "axis": [ { "name": "Easting", "abbreviation": "E", "direction": "east", "unit": "metre" }, { "name": "Northing", "abbreviation": "N", "direction": "north", "unit": "metre" } ] } } }, "y": { "type": "spatial", "axis": "y", "description": "y coordinate of projection", "extent": [-622000.0, -39000.0], "step": -1000.0, "reference_system": { "$schema": "https://proj.org/schemas/v0.2/projjson.schema.json", "type": "ProjectedCRS", "name": "undefined", "base_crs": { "name": "undefined", "datum": { "type": "GeodeticReferenceFrame", "name": "undefined", "ellipsoid": { "name": "undefined", "semi_major_axis": 6378137, "inverse_flattening": 298.257223563 } }, "coordinate_system": { "subtype": "ellipsoidal", "axis": [ { "name": "Longitude", "abbreviation": "lon", "direction": "east", "unit": "degree" }, { "name": "Latitude", "abbreviation": "lat", "direction": "north", "unit": "degree" } ] } }, "conversion": { "name": "unknown", "method": { "name": "Lambert Conic Conformal (2SP)", "id": { "authority": "EPSG", "code": 9802 } }, "parameters": [ { "name": "Latitude of 1st standard parallel", "value": 25, "unit": "degree", "id": { "authority": "EPSG", "code": 8823 } }, { "name": "Latitude of 2nd standard parallel", "value": 60, "unit": "degree", "id": { "authority": "EPSG", "code": 8824 } }, { "name": "Latitude of false origin", "value": 42.5, "unit": "degree", "id": { "authority": "EPSG", "code": 8821 } }, { "name": "Longitude of false origin", "value": -100, "unit": "degree", "id": { "authority": "EPSG", "code": 8822 } }, { "name": "Easting at false origin", "value": 0, "unit": "metre", "id": { "authority": "EPSG", "code": 8826 } }, { "name": "Northing at false origin", "value": 0, "unit": "metre", "id": { "authority": "EPSG", "code": 8827 } } ] }, "coordinate_system": { "subtype": "Cartesian", "axis": [ { "name": "Easting", "abbreviation": "E", "direction": "east", "unit": "metre" }, { "name": "Northing", "abbreviation": "N", "direction": "north", "unit": "metre" } ] } } }, "nv": { "type": "count", "description": "Size of the 'time_bnds' variable.", "values": [0, 1] } }, "cube:variables": { "dayl": { "type": "data", "description": "daylength", "dimensions": ["time", "y", "x"], "unit": "s", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean", "grid_mapping": "lambert_conformal_conic", "long_name": "daylength", "units": "s" } }, "lambert_conformal_conic": { "type": "data", "dimensions": [], "shape": [], "attrs": { "false_easting": 0.0, "false_northing": 0.0, "grid_mapping_name": "lambert_conformal_conic", "inverse_flattening": 298.257223563, "latitude_of_projection_origin": 42.5, "longitude_of_central_meridian": -100.0, "semi_major_axis": 6378137.0, "standard_parallel": [25.0, 60.0] } }, "lat": { "type": "auxiliary", "description": "latitude coordinate", "dimensions": ["y", "x"], "unit": "degrees_north", "shape": [584, 284], "chunks": [584, 284], "attrs": { "long_name": "latitude coordinate", "standard_name": "latitude", "units": "degrees_north" } }, "lon": { "type": "auxiliary", "description": "longitude coordinate", "dimensions": ["y", "x"], "unit": "degrees_east", "shape": [584, 284], "chunks": [584, 284], "attrs": { "long_name": "longitude coordinate", "standard_name": "longitude", "units": "degrees_east" } }, "prcp": { "type": "data", "description": "daily total precipitation", "dimensions": ["time", "y", "x"], "unit": "mm/day", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: sum", "grid_mapping": "lambert_conformal_conic", "long_name": "daily total precipitation", "units": "mm/day" } }, "srad": { "type": "data", "description": "daylight average incident shortwave radiation", "dimensions": ["time", "y", "x"], "unit": "W/m2", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: mean", "grid_mapping": "lambert_conformal_conic", "long_name": "daylight average incident shortwave radiation", "units": "W/m2" } }, "swe": { "type": "data", "description": "snow water equivalent", "dimensions": ["time", "y", "x"], "unit": "kg/m2", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: mean", "grid_mapping": "lambert_conformal_conic", "long_name": "snow water equivalent", "units": "kg/m2" } }, "time_bnds": { "type": "data", "dimensions": ["time", "nv"], "shape": [14965, 2], "chunks": [365, 2], "attrs": {} }, "tmax": { "type": "data", "description": "daily maximum temperature", "dimensions": ["time", "y", "x"], "unit": "degrees C", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: maximum", "grid_mapping": "lambert_conformal_conic", "long_name": "daily maximum temperature", "units": "degrees C" } }, "tmin": { "type": "data", "description": "daily minimum temperature", "dimensions": ["time", "y", "x"], "unit": "degrees C", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: minimum", "grid_mapping": "lambert_conformal_conic", "long_name": "daily minimum temperature", "units": "degrees C" } }, "vp": { "type": "data", "description": "daily average vapor pressure", "dimensions": ["time", "y", "x"], "unit": "Pa", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: mean", "grid_mapping": "lambert_conformal_conic", "long_name": "daily average vapor pressure", "units": "Pa" } }, "yearday": { "type": "data", "description": "day of year (DOY) starting with day 1 on January 1st", "dimensions": ["time"], "shape": [14965], "chunks": [365], "attrs": { "long_name": "day of year (DOY) starting with day 1 on January 1st" } } }, "title": "Daymet Daily Hawaii", "keywords": [ "Daymet", "Hawaii", "Temperature", "Precipitation", "Vapor Pressure", "Weather" ], "providers": [ { "name": "Microsoft", "roles": ["host", "processor"], "url": "https://planetarycomputer.microsoft.com" }, { "name": "ORNL DAAC", "roles": ["producer"], "url": "https://doi.org/10.3334/ORNLDAAC/1840" } ], "assets": { "zarr-https": { "href": "https://daymeteuwest.blob.core.windows.net/daymet-zarr/daily/hi.zarr", "type": "application/vnd+zarr", "title": "Daily Hawaii Daymet HTTPS Zarr root", "description": "HTTPS URI of the daily Hawaii Daymet Zarr Group on Azure Blob Storage.", "xarray:open_kwargs": { "consolidated": true }, "roles": ["data", "zarr", "https"] }, "zarr-abfs": { "href": "abfs://daymet-zarr/daily/hi.zarr", "type": "application/vnd+zarr", "title": "Daily Hawaii Daymet Azure Blob File System Zarr root", "description": "Azure Blob File System of the daily Hawaii Daymet Zarr Group on Azure Blob Storage for use with adlfs.", "xarray:storage_options": { "account_name": "daymeteuwest" }, "xarray:open_kwargs": { "consolidated": true }, "roles": ["data", "zarr", "abfs"] }, "thumbnail": { "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/daymet-daily-hi.png", "type": "image/png", "title": "Daymet daily Hawaii map thumbnail", "roles": ["thumbnail"] } }, "msft:short_description": "Daily surface weather data on a 1-km grid for Hawaii", "msft:storage_account": "daymeteuwest", "msft:container": "daymet-zarr", "msft:group_id": "daymet", "msft:group_keys": ["daily", "hawaii"], "extent": { "spatial": { "bbox": [[-160.3056, 17.9539, -154.772, 23.5186]] }, "temporal": { "interval": [["1980-01-01T12:00:00Z", "2020-12-30T12:00:00Z"]] } }, "license": "proprietary" } ================================================ FILE: intake/readers/tests/cats/stac_data/1.0.0/item/zarr-item.json ================================================ { "type": "Feature", "stac_version": "1.0.0", "id": "daymet-daily-hi", "properties": { "cube:dimensions": { "time": { "type": "temporal", "description": "24-hour day based on local time", "extent": ["1980-01-01T12:00:00Z", "2020-12-30T12:00:00Z"] }, "x": { "type": "spatial", "axis": "x", "description": "x coordinate of projection", "extent": [-5802250.0, -5519250.0], "step": 1000.0, "reference_system": { "$schema": "https://proj.org/schemas/v0.2/projjson.schema.json", "type": "ProjectedCRS", "name": "undefined", "base_crs": { "name": "undefined", "datum": { "type": "GeodeticReferenceFrame", "name": "undefined", "ellipsoid": { "name": "undefined", "semi_major_axis": 6378137, "inverse_flattening": 298.257223563 } }, "coordinate_system": { "subtype": "ellipsoidal", "axis": [ { "name": "Longitude", "abbreviation": "lon", "direction": "east", "unit": "degree" }, { "name": "Latitude", "abbreviation": "lat", "direction": "north", "unit": "degree" } ] } }, "conversion": { "name": "unknown", "method": { "name": "Lambert Conic Conformal (2SP)", "id": { "authority": "EPSG", "code": 9802 } }, "parameters": [ { "name": "Latitude of 1st standard parallel", "value": 25, "unit": "degree", "id": { "authority": "EPSG", "code": 8823 } }, { "name": "Latitude of 2nd standard parallel", "value": 60, "unit": "degree", "id": { "authority": "EPSG", "code": 8824 } }, { "name": "Latitude of false origin", "value": 42.5, "unit": "degree", "id": { "authority": "EPSG", "code": 8821 } }, { "name": "Longitude of false origin", "value": -100, "unit": "degree", "id": { "authority": "EPSG", "code": 8822 } }, { "name": "Easting at false origin", "value": 0, "unit": "metre", "id": { "authority": "EPSG", "code": 8826 } }, { "name": "Northing at false origin", "value": 0, "unit": "metre", "id": { "authority": "EPSG", "code": 8827 } } ] }, "coordinate_system": { "subtype": "Cartesian", "axis": [ { "name": "Easting", "abbreviation": "E", "direction": "east", "unit": "metre" }, { "name": "Northing", "abbreviation": "N", "direction": "north", "unit": "metre" } ] } } }, "y": { "type": "spatial", "axis": "y", "description": "y coordinate of projection", "extent": [-622000.0, -39000.0], "step": -1000.0, "reference_system": { "$schema": "https://proj.org/schemas/v0.2/projjson.schema.json", "type": "ProjectedCRS", "name": "undefined", "base_crs": { "name": "undefined", "datum": { "type": "GeodeticReferenceFrame", "name": "undefined", "ellipsoid": { "name": "undefined", "semi_major_axis": 6378137, "inverse_flattening": 298.257223563 } }, "coordinate_system": { "subtype": "ellipsoidal", "axis": [ { "name": "Longitude", "abbreviation": "lon", "direction": "east", "unit": "degree" }, { "name": "Latitude", "abbreviation": "lat", "direction": "north", "unit": "degree" } ] } }, "conversion": { "name": "unknown", "method": { "name": "Lambert Conic Conformal (2SP)", "id": { "authority": "EPSG", "code": 9802 } }, "parameters": [ { "name": "Latitude of 1st standard parallel", "value": 25, "unit": "degree", "id": { "authority": "EPSG", "code": 8823 } }, { "name": "Latitude of 2nd standard parallel", "value": 60, "unit": "degree", "id": { "authority": "EPSG", "code": 8824 } }, { "name": "Latitude of false origin", "value": 42.5, "unit": "degree", "id": { "authority": "EPSG", "code": 8821 } }, { "name": "Longitude of false origin", "value": -100, "unit": "degree", "id": { "authority": "EPSG", "code": 8822 } }, { "name": "Easting at false origin", "value": 0, "unit": "metre", "id": { "authority": "EPSG", "code": 8826 } }, { "name": "Northing at false origin", "value": 0, "unit": "metre", "id": { "authority": "EPSG", "code": 8827 } } ] }, "coordinate_system": { "subtype": "Cartesian", "axis": [ { "name": "Easting", "abbreviation": "E", "direction": "east", "unit": "metre" }, { "name": "Northing", "abbreviation": "N", "direction": "north", "unit": "metre" } ] } } }, "nv": { "type": "count", "description": "Size of the 'time_bnds' variable.", "values": [0, 1] } }, "cube:variables": { "dayl": { "type": "data", "description": "daylength", "dimensions": ["time", "y", "x"], "unit": "s", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean", "grid_mapping": "lambert_conformal_conic", "long_name": "daylength", "units": "s" } }, "lambert_conformal_conic": { "type": "data", "dimensions": [], "shape": [], "attrs": { "false_easting": 0.0, "false_northing": 0.0, "grid_mapping_name": "lambert_conformal_conic", "inverse_flattening": 298.257223563, "latitude_of_projection_origin": 42.5, "longitude_of_central_meridian": -100.0, "semi_major_axis": 6378137.0, "standard_parallel": [25.0, 60.0] } }, "lat": { "type": "auxiliary", "description": "latitude coordinate", "dimensions": ["y", "x"], "unit": "degrees_north", "shape": [584, 284], "chunks": [584, 284], "attrs": { "long_name": "latitude coordinate", "standard_name": "latitude", "units": "degrees_north" } }, "lon": { "type": "auxiliary", "description": "longitude coordinate", "dimensions": ["y", "x"], "unit": "degrees_east", "shape": [584, 284], "chunks": [584, 284], "attrs": { "long_name": "longitude coordinate", "standard_name": "longitude", "units": "degrees_east" } }, "prcp": { "type": "data", "description": "daily total precipitation", "dimensions": ["time", "y", "x"], "unit": "mm/day", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: sum", "grid_mapping": "lambert_conformal_conic", "long_name": "daily total precipitation", "units": "mm/day" } }, "srad": { "type": "data", "description": "daylight average incident shortwave radiation", "dimensions": ["time", "y", "x"], "unit": "W/m2", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: mean", "grid_mapping": "lambert_conformal_conic", "long_name": "daylight average incident shortwave radiation", "units": "W/m2" } }, "swe": { "type": "data", "description": "snow water equivalent", "dimensions": ["time", "y", "x"], "unit": "kg/m2", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: mean", "grid_mapping": "lambert_conformal_conic", "long_name": "snow water equivalent", "units": "kg/m2" } }, "time_bnds": { "type": "data", "dimensions": ["time", "nv"], "shape": [14965, 2], "chunks": [365, 2], "attrs": {} }, "tmax": { "type": "data", "description": "daily maximum temperature", "dimensions": ["time", "y", "x"], "unit": "degrees C", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: maximum", "grid_mapping": "lambert_conformal_conic", "long_name": "daily maximum temperature", "units": "degrees C" } }, "tmin": { "type": "data", "description": "daily minimum temperature", "dimensions": ["time", "y", "x"], "unit": "degrees C", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: minimum", "grid_mapping": "lambert_conformal_conic", "long_name": "daily minimum temperature", "units": "degrees C" } }, "vp": { "type": "data", "description": "daily average vapor pressure", "dimensions": ["time", "y", "x"], "unit": "Pa", "shape": [14965, 584, 284], "chunks": [365, 584, 284], "attrs": { "cell_methods": "area: mean time: mean", "grid_mapping": "lambert_conformal_conic", "long_name": "daily average vapor pressure", "units": "Pa" } }, "yearday": { "type": "data", "description": "day of year (DOY) starting with day 1 on January 1st", "dimensions": ["time"], "shape": [14965], "chunks": [365], "attrs": { "long_name": "day of year (DOY) starting with day 1 on January 1st" } } }, "start_datetime": "1980-01-01T12:00:00Z", "end_datetime": "2020-12-30T12:00:00Z", "datetime": null }, "geometry": { "type": "Polygon", "coordinates": [ [ [-154.7780670634169, 17.960033949329812], [-154.7780670634169, 23.51232608231902], [-160.2988400944475, 23.51232608231902], [-160.2988400944475, 17.960033949329812], [-154.7780670634169, 17.960033949329812] ] ] }, "links": [], "assets": { "zarr-https": { "href": "https://daymeteuwest.blob.core.windows.net/daymet-zarr/daily/hi.zarr", "type": "application/vnd+zarr", "title": "Daily Hawaii Daymet HTTPS Zarr root", "description": "HTTPS URI of the daily Hawaii Daymet Zarr Group on Azure Blob Storage.", "xarray:open_kwargs": { "consolidated": true }, "roles": ["data", "zarr", "https"] }, "zarr-abfs": { "href": "abfs://daymet-zarr/daily/hi.zarr", "type": "application/vnd+zarr", "title": "Daily Hawaii Daymet Azure Blob File System Zarr root", "description": "Azure Blob File System of the daily Hawaii Daymet Zarr Group on Azure Blob Storage for use with adlfs.", "xarray:storage_options": { "account_name": "daymeteuwest" }, "xarray:open_kwargs": { "consolidated": true }, "roles": ["data", "zarr", "abfs"] }, "thumbnail": { "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/daymet-daily-hi.png", "type": "image/png", "title": "Daymet daily Hawaii map thumbnail" } }, "bbox": [ -160.2988400944475, 17.960033949329812, -154.7780670634169, 23.51232608231902 ], "stac_extensions": [ "https://stac-extensions.github.io/datacube/v2.0.0/schema.json" ] } ================================================ FILE: intake/readers/tests/cats/stac_data/1.0.0/itemcollection/example-search.json ================================================ { "id": "mysearchresults", "stac_version": "1.0.0-beta.2", "stac_extensions": ["single-file-stac"], "description": "A bunch of results from a search", "type": "FeatureCollection", "features": [ { "stac_version": "1.0.0-beta.2", "stac_extensions": [ "https://stac-extensions.github.io/projection/v1.0.0/schema.json", "https://stac-extensions.github.io/view/v1.0.0/schema.json" ], "type": "Feature", "id": "LC80370332018039LGN00", "collection": "landsat-8-l1", "bbox": [-112.21054, 37.83042, -109.4992, 39.95532], "geometry": { "type": "Polygon", "coordinates": [ [ [-111.6768167850251, 39.952817693022276], [-109.5010938553632, 39.55607811527241], [-110.03573868784865, 37.83172334507642], [-112.20846353249907, 38.236456540046845], [-111.6768167850251, 39.952817693022276] ] ] }, "properties": { "datetime": "2018-02-08T18:02:15.719478+00:00", "view:sun_azimuth": 152.63804142, "view:sun_elevation": 31.82216637, "proj:epsg": 32612 }, "assets": { "index": { "type": "text/html", "title": "HTML index page", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/index.html" }, "thumbnail": { "title": "Thumbnail image", "type": "image/jpeg", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_thumb_large.jpg" }, "B1": { "type": "image/tiff; application=geotiff", "title": "Band 1 (coastal)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B1.TIF" }, "B2": { "type": "image/tiff; application=geotiff", "title": "Band 2 (blue)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B2.TIF" }, "B3": { "type": "image/tiff; application=geotiff", "title": "Band 3 (green)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B3.TIF" }, "B4": { "type": "image/tiff; application=geotiff", "title": "Band 4 (red)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B4.TIF" }, "B5": { "type": "image/tiff; application=geotiff", "title": "Band 5 (nir)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B5.TIF" }, "B6": { "type": "image/tiff; application=geotiff", "title": "Band 6 (swir16)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B6.TIF" }, "B7": { "type": "image/tiff; application=geotiff", "title": "Band 7 (swir22)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B7.TIF" }, "B8": { "type": "image/tiff; application=geotiff", "title": "Band 8 (pan)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B8.TIF" }, "B9": { "type": "image/tiff; application=geotiff", "title": "Band 9 (cirrus)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B9.TIF" }, "B10": { "type": "image/tiff; application=geotiff", "title": "Band 10 (lwir)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B10.TIF" }, "B11": { "type": "image/tiff; application=geotiff", "title": "Band 11 (lwir)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_B11.TIF" }, "ANG": { "title": "Angle coefficients file", "type": "text/plain", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_ANG.txt" }, "MTL": { "title": "original metadata file", "type": "text/plain", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_MTL.txt" }, "BQA": { "title": "Band quality data", "type": "image/tiff; application=geotiff", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/037/033/LC08_L1TP_037033_20180208_20180221_01_T1/LC08_L1TP_037033_20180208_20180221_01_T1_BQA.TIF" } }, "links": [] }, { "stac_version": "1.0.0", "stac_extensions": ["projection", "view"], "type": "Feature", "id": "LC80340332018034LGN00", "collection": "landsat-8-l1", "bbox": [-107.6044, 37.8096, -104.86884, 39.97508], "geometry": { "type": "Polygon", "coordinates": [ [ [-107.03912158283073, 39.975078807631036], [-104.87161559271382, 39.548160703908025], [-105.43927721248009, 37.81075859503169], [-107.60423259994965, 38.24485405534073], [-107.03912158283073, 39.975078807631036] ] ] }, "properties": { "datetime": "2018-02-03T17:43:44Z", "view:sun_azimuth": 153.39513457, "view:sun_elevation": 30.41894816, "proj:epsg": 32613 }, "assets": { "index": { "type": "text/html", "title": "HTML index page", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/index.html" }, "thumbnail": { "title": "Thumbnail image", "type": "image/jpeg", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_thumb_large.jpg" }, "B1": { "type": "image/tiff; application=geotiff", "title": "Band 1 (coastal)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B1.TIF" }, "B2": { "type": "image/tiff; application=geotiff", "title": "Band 2 (blue)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B2.TIF" }, "B3": { "type": "image/tiff; application=geotiff", "title": "Band 3 (green)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B3.TIF" }, "B4": { "type": "image/tiff; application=geotiff", "title": "Band 4 (red)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B4.TIF" }, "B5": { "type": "image/tiff; application=geotiff", "title": "Band 5 (nir)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B5.TIF" }, "B6": { "type": "image/tiff; application=geotiff", "title": "Band 6 (swir16)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B6.TIF" }, "B7": { "type": "image/tiff; application=geotiff", "title": "Band 7 (swir22)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B7.TIF" }, "B8": { "type": "image/tiff; application=geotiff", "title": "Band 8 (pan)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B8.TIF" }, "B9": { "type": "image/tiff; application=geotiff", "title": "Band 9 (cirrus)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B9.TIF" }, "B10": { "type": "image/tiff; application=geotiff", "title": "Band 10 (lwir)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B10.TIF" }, "B11": { "type": "image/tiff; application=geotiff", "title": "Band 11 (lwir)", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_B11.TIF" }, "ANG": { "title": "Angle coefficients file", "type": "text/plain", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_ANG.txt" }, "MTL": { "title": "original metadata file", "type": "text/plain", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_MTL.txt" }, "BQA": { "title": "Band quality data", "type": "image/tiff; application=geotiff", "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/034/033/LC08_L1TP_034033_20180203_20180220_01_T1/LC08_L1TP_034033_20180203_20180220_01_T1_BQA.TIF" } }, "links": [] } ], "collections": [ { "id": "landsat-8-l1", "title": "Landsat 8 L1", "description": "Landat 8 imagery radiometrically calibrated and orthorectified using gound points and Digital Elevation Model (DEM) data to correct relief displacement.", "keywords": ["landsat", "earth observation", "usgs"], "stac_version": "1.0.0-beta.2", "stac_extensions": ["item_assets"], "extent": { "spatial": { "bbox": [[-180, -90, 180, 90]] }, "temporal": { "interval": [["2013-06-01T00:00:00Z", null]] } }, "providers": [ { "name": "USGS", "roles": ["producer"], "url": "https://landsat.usgs.gov/" }, { "name": "Planet Labs", "roles": ["processor"], "url": "https://github.com/landsat-pds/landsat_ingestor" }, { "name": "AWS", "roles": ["host"], "url": "https://landsatonaws.com/" }, { "name": "Development Seed", "roles": ["processor"], "url": "https://github.com/sat-utils/sat-api" } ], "license": "PDDL-1.0", "summaries": { "gsd": [15], "platform": ["landsat-8"], "instruments": ["oli", "tirs"], "view:off_nadir": [0] }, "item_assets": { "index": { "type": "text/html", "title": "HTML index page" }, "thumbnail": { "title": "Thumbnail image", "type": "image/jpeg" }, "B1": { "type": "image/tiff; application=geotiff", "title": "Band 1 (coastal)" }, "B2": { "type": "image/tiff; application=geotiff", "title": "Band 2 (blue)" }, "B3": { "type": "image/tiff; application=geotiff", "title": "Band 3 (green)" }, "B4": { "type": "image/tiff; application=geotiff", "title": "Band 4 (red)" }, "B5": { "type": "image/tiff; application=geotiff", "title": "Band 5 (nir)" }, "B6": { "type": "image/tiff; application=geotiff", "title": "Band 6 (swir16)" }, "B7": { "type": "image/tiff; application=geotiff", "title": "Band 7 (swir22)" }, "B8": { "type": "image/tiff; application=geotiff", "title": "Band 8 (pan)" }, "B9": { "type": "image/tiff; application=geotiff", "title": "Band 9 (cirrus)" }, "B10": { "type": "image/tiff; application=geotiff", "title": "Band 10 (lwir)" }, "B11": { "type": "image/tiff; application=geotiff", "title": "Band 11 (lwir)" }, "ANG": { "title": "Angle coefficients file", "type": "text/plain" }, "MTL": { "title": "original metadata file", "type": "text/plain" }, "BQA": { "title": "Band quality data", "type": "image/tiff; application=geotiff" } }, "links": [ { "rel": "self", "href": "./example-search.json" } ] } ], "links": [] } ================================================ FILE: intake/readers/tests/cats/stac_data/1.0.0beta2/earthsearch/readme.md ================================================ Generated with: ```python import satsearch import pystac import json bbox = [35.48, -3.24, 35.58, -3.14] dates = '2020-07-01/2020-08-15' URL='https://earth-search.aws.element84.com/v0' results = satsearch.Search.search(url=URL, collections=['sentinel-s2-l2a-cogs'], datetime=dates, bbox=bbox, sort=['-properties.datetime']) # 18 items found items = results.items() print(len(items)) items.save('single-file-stac.json') # validation returns empty list import json from pystac.validation import validate_dict with open('single-file-stac.json') as f: js = json.load(f) print(validate_dict(js)) cat = pystac.read_file('single-file-stac.json') ``` ================================================ FILE: intake/readers/tests/cats/stac_data/1.0.0beta2/earthsearch/single-file-stac.json ================================================ { "id": "STAC", "description": "Single file STAC", "stac_version": "1.0.0-beta.2", "stac_extensions": ["single-file-stac"], "type": "FeatureCollection", "features": [ { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2A_36MYB_20200814_0_L2A", "bbox": [ 34.79870551983084, -3.7056906919566326, 35.755863403460744, -2.7110273448887328 ], "geometry": { "type": "Polygon", "coordinates": [ [ [34.80044299251402, -3.7056906919566326], [34.79870551983084, -2.712838434794184], [35.755863403460744, -2.7110273448887328], [35.58561393371855, -3.4903485137854644], [35.53750206413227, -3.703878577458925], [34.80044299251402, -3.7056906919566326] ] ] }, "properties": { "datetime": "2020-08-14T08:11:00Z", "platform": "sentinel-2a", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "view:off_nadir": 0, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2A_MSIL2A_20200814T074621_N0214_R135_T36MYB_20200814T103139", "sentinel:data_coverage": 85.74, "eo:cloud_cover": 47.78, "sentinel:valid_cloud_cover": true, "created": "2020-08-17T19:57:08.648Z", "updated": "2020-08-17T19:57:08.648Z", "sentinel:utm_zone": 36 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/8/14/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/14/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/14/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_36MYB_20200814_0_L2A" }, { "rel": "derived_from", "href": "s3://cirrus-v0-data-1qm7gekzjucbq/sentinel-s2-l2a/36/M/YB/2020/8/S2A_36MYB_20200814_0_L2A/S2A_36MYB_20200814_0_L2A.json", "title": "Source STAC Item", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2A_36MYB_20200811_0_L2A", "bbox": [ 35.3528545470006, -3.704390328989478, 35.78841174435426, -2.7109586089556537 ], "geometry": { "type": "Polygon", "coordinates": [ [ [35.3528545470006, -3.704390328989478], [35.3871744235407, -3.550087222076425], [35.57403108911051, -2.7114299346410236], [35.785723330760796, -2.7109586089556537], [35.78841174435426, -3.7031212858685087], [35.3528545470006, -3.704390328989478] ] ] }, "properties": { "datetime": "2020-08-11T08:01:05Z", "platform": "sentinel-2a", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "view:off_nadir": 0, "proj:epsg": 32736, "sentinel:utm_zone": 36, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2A_MSIL2A_20200811T073621_N0214_R092_T36MYB_20200811T114649", "sentinel:data_coverage": 32.62, "eo:cloud_cover": 12.78, "sentinel:valid_cloud_cover": true, "created": "2020-09-19T02:55:16.877Z", "updated": "2020-09-19T02:55:16.877Z" }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/8/11/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/11/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/11/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_36MYB_20200811_0_L2A" }, { "rel": "canonical", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200811_0_L2A/S2A_36MYB_20200811_0_L2A.json", "type": "application/json" }, { "title": "Source STAC Item", "rel": "derived_from", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items/S2A_36MYB_20200811_0_L2A", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2B_36MYB_20200809_0_L2A", "bbox": [ 34.79870551983084, -3.7056906919566326, 35.76700629249285, -2.7110017811328926 ], "geometry": { "type": "Polygon", "coordinates": [ [ [34.80044299251402, -3.7056906919566326], [34.79870551983084, -2.712838434794184], [35.76700629249285, -2.7110017811328926], [35.59534033727834, -3.4950217349500936], [35.54865807009975, -3.7038464213259266], [34.80044299251402, -3.7056906919566326] ] ] }, "properties": { "datetime": "2020-08-09T08:10:57Z", "platform": "sentinel-2b", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "view:off_nadir": 0, "proj:epsg": 32736, "sentinel:utm_zone": 36, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2B_MSIL2A_20200809T074619_N0214_R135_T36MYB_20200809T103816", "sentinel:data_coverage": 86.85, "eo:cloud_cover": 77.28, "sentinel:valid_cloud_cover": true, "created": "2020-08-18T10:57:54.124Z", "updated": "2020-08-18T10:57:54.124Z" }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/8/9/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/9/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/9/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36MYB_20200809_0_L2A" }, { "title": "Source STAC Item", "rel": "derived_from", "href": "s3://cirrus-v0-data-1qm7gekzjucbq/sentinel-s2-l2a/36/M/YB/2020/8/S2B_36MYB_20200809_0_L2A/S2B_36MYB_20200809_0_L2A.json", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2B_36MYB_20200806_0_L2A", "bbox": [ 35.361851558698646, -3.704366288842677, 35.78841174435426, -2.7109586089556537 ], "geometry": { "type": "Polygon", "coordinates": [ [ [35.361851558698646, -3.704366288842677], [35.40381967148886, -3.5125664894255837], [35.582659278647895, -2.7114114515918657], [35.785723330760796, -2.7109586089556537], [35.78841174435426, -3.7031212858685087], [35.361851558698646, -3.704366288842677] ] ] }, "properties": { "datetime": "2020-08-06T08:01:02Z", "platform": "sentinel-2b", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "view:off_nadir": 0, "proj:epsg": 32736, "sentinel:utm_zone": 36, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2B_MSIL2A_20200806T073619_N0214_R092_T36MYB_20200806T113023", "sentinel:data_coverage": 31.77, "eo:cloud_cover": 0, "sentinel:valid_cloud_cover": true, "created": "2020-08-17T20:39:17.063Z", "updated": "2020-08-17T20:39:17.063Z" }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/8/6/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/6/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/6/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36MYB_20200806_0_L2A" }, { "title": "Source STAC Item", "rel": "derived_from", "href": "s3://cirrus-v0-data-1qm7gekzjucbq/sentinel-s2-l2a/36/M/YB/2020/8/S2B_36MYB_20200806_0_L2A/S2B_36MYB_20200806_0_L2A.json", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2A_36MYB_20200804_0_L2A", "bbox": [ 34.79870551983084, -3.7056906919566326, 35.757300644517535, -2.7110240533940337 ], "geometry": { "type": "Polygon", "coordinates": [ [ [34.80044299251402, -3.7056906919566326], [34.79870551983084, -2.712838434794184], [35.757300644517535, -2.7110240533940337], [35.593185658988496, -3.4635764446186035], [35.539121505336155, -3.7038739183195086], [34.80044299251402, -3.7056906919566326] ] ] }, "properties": { "datetime": "2020-08-04T08:11:00Z", "platform": "sentinel-2a", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "view:off_nadir": 0, "proj:epsg": 32736, "sentinel:utm_zone": 36, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2A_MSIL2A_20200804T074621_N0214_R135_T36MYB_20200804T104333", "sentinel:data_coverage": 85.89, "eo:cloud_cover": 75.37, "sentinel:valid_cloud_cover": true, "created": "2020-08-17T21:33:14.530Z", "updated": "2020-08-17T21:33:14.530Z" }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/8/4/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/4/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/4/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_36MYB_20200804_0_L2A" }, { "title": "Source STAC Item", "rel": "derived_from", "href": "s3://cirrus-v0-data-1qm7gekzjucbq/sentinel-s2-l2a/36/M/YB/2020/8/S2A_36MYB_20200804_0_L2A/S2A_36MYB_20200804_0_L2A.json", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2A_36MYB_20200801_0_L2A", "bbox": [ 35.354473584331565, -3.704386009664254, 35.78841174435426, -2.7109586089556537 ], "geometry": { "type": "Polygon", "coordinates": [ [ [35.354473584331565, -3.704386009664254], [35.3944358874427, -3.5225127312927675], [35.53486937287048, -2.8896897890758986], [35.574751456777825, -2.7114283938558765], [35.785723330760796, -2.7109586089556537], [35.78841174435426, -3.7031212858685087], [35.354473584331565, -3.704386009664254] ] ] }, "properties": { "datetime": "2020-08-01T08:01:05Z", "platform": "sentinel-2a", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "view:off_nadir": 0, "proj:epsg": 32736, "sentinel:utm_zone": 36, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2A_MSIL2A_20200801T073621_N0214_R092_T36MYB_20200801T102406", "sentinel:data_coverage": 32.57, "eo:cloud_cover": 62.8, "sentinel:valid_cloud_cover": true, "created": "2020-08-18T10:54:09.447Z", "updated": "2020-08-18T10:54:09.447Z" }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/8/1/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/1/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/8/1/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_36MYB_20200801_0_L2A" }, { "title": "Source STAC Item", "rel": "derived_from", "href": "s3://cirrus-v0-data-1qm7gekzjucbq/sentinel-s2-l2a/36/M/YB/2020/8/S2A_36MYB_20200801_0_L2A/S2A_36MYB_20200801_0_L2A.json", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2B_36MYB_20200730_0_L2A", "bbox": [ 34.79870551983084, -3.7056906919566326, 35.7687137359457, -2.7109978548485385 ], "geometry": { "type": "Polygon", "coordinates": [ [ [34.80044299251402, -3.7056906919566326], [34.79870551983084, -2.712838434794184], [35.7687137359457, -2.7109978548485385], [35.602303822311306, -3.471108470253178], [35.5503674728783, -3.703841481678485], [34.80044299251402, -3.7056906919566326] ] ] }, "properties": { "datetime": "2020-07-30T08:10:57Z", "platform": "sentinel-2b", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "view:off_nadir": 0, "proj:epsg": 32736, "sentinel:utm_zone": 36, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2B_MSIL2A_20200730T074619_N0214_R135_T36MYB_20200730T111202", "sentinel:data_coverage": 87.03, "eo:cloud_cover": 5.89, "sentinel:valid_cloud_cover": true, "created": "2020-08-18T09:45:24.058Z", "updated": "2020-08-18T09:45:24.058Z" }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/30/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/30/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/30/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36MYB_20200730_0_L2A" }, { "title": "Source STAC Item", "rel": "derived_from", "href": "s3://cirrus-v0-data-1qm7gekzjucbq/sentinel-s2-l2a/36/M/YB/2020/7/S2B_36MYB_20200730_0_L2A/S2B_36MYB_20200730_0_L2A.json", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2B_36MYB_20200727_0_L2A", "bbox": [ 35.364731107030785, -3.704358575275874, 35.78841174435426, -2.7109586089556537 ], "geometry": { "type": "Polygon", "coordinates": [ [ [35.364731107030785, -3.704358575275874], [35.40440599150377, -3.5241264913281642], [35.58517559334435, -2.711406049575985], [35.785723330760796, -2.7109586089556537], [35.78841174435426, -3.7031212858685087], [35.364731107030785, -3.704358575275874] ] ] }, "properties": { "datetime": "2020-07-27T08:01:01Z", "platform": "sentinel-2b", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "view:off_nadir": 0, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2B_MSIL2A_20200727T073619_N0214_R092_T36MYB_20200727T122549", "sentinel:data_coverage": 31.47, "eo:cloud_cover": 49.45, "sentinel:valid_cloud_cover": true, "created": "2020-08-31T18:24:19.293Z", "updated": "2020-08-31T18:24:19.293Z", "sentinel:utm_zone": 36 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/27/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/27/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/27/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36MYB_20200727_0_L2A" }, { "rel": "canonical", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200727_0_L2A/S2B_36MYB_20200727_0_L2A.json", "type": "application/json" }, { "rel": "derived_from", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items/S2B_36MYB_20200727_0_L2A", "title": "Source STAC Item", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2A_36MYB_20200725_0_L2A", "bbox": [ 34.79870551983084, -3.7056906919566326, 35.759906699226775, -2.7110180807693265 ], "geometry": { "type": "Polygon", "coordinates": [ [ [34.80044299251402, -3.7056906919566326], [34.79870551983084, -2.712838434794184], [35.759906699226775, -2.7110180807693265], [35.58973859625803, -3.4867218329449443], [35.54119098027402, -3.703867960107399], [34.80044299251402, -3.7056906919566326] ] ] }, "properties": { "datetime": "2020-07-25T08:10:59Z", "platform": "sentinel-2a", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "view:off_nadir": 0, "proj:epsg": 32736, "sentinel:utm_zone": 36, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2A_MSIL2A_20200725T074621_N0214_R135_T36MYB_20200725T111115", "sentinel:data_coverage": 86.12, "eo:cloud_cover": 1.01, "sentinel:valid_cloud_cover": true, "created": "2020-08-18T10:54:56.621Z", "updated": "2020-08-18T10:54:56.621Z" }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/25/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/25/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/25/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_36MYB_20200725_0_L2A" }, { "title": "Source STAC Item", "rel": "derived_from", "href": "s3://cirrus-v0-data-1qm7gekzjucbq/sentinel-s2-l2a/36/M/YB/2020/7/S2A_36MYB_20200725_0_L2A/S2A_36MYB_20200725_0_L2A.json", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2A_36MYB_20200722_0_L2A", "bbox": [ 35.356453603399366, -3.704380723262607, 35.78841174435426, -2.7109586089556537 ], "geometry": { "type": "Polygon", "coordinates": [ [ [35.356453603399366, -3.704380723262607], [35.39652214425136, -3.52401250696652], [35.57744628705123, -2.7114226260972907], [35.785723330760796, -2.7109586089556537], [35.78841174435426, -3.7031212858685087], [35.356453603399366, -3.704380723262607] ] ] }, "properties": { "datetime": "2020-07-22T08:01:04Z", "platform": "sentinel-2a", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "data_coverage": 32.26, "view:off_nadir": 0, "eo:cloud_cover": 69.73, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2A_MSIL2A_20200722T073621_N0214_R092_T36MYB_20200722T105841", "created": "2020-09-19T01:08:35.436Z", "updated": "2020-09-19T01:08:35.436Z", "sentinel:valid_cloud_cover": true, "sentinel:utm_zone": 36, "sentinel:data_coverage": 32.26 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/22/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/22/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/22/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_36MYB_20200722_0_L2A" }, { "rel": "canonical", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200722_0_L2A/S2A_36MYB_20200722_0_L2A.json", "type": "application/json" }, { "rel": "derived_from", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items/S2A_36MYB_20200722_0_L2A", "title": "Source STAC Item", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2B_36MYB_20200720_0_L2A", "bbox": [ 34.79870551983084, -3.7056906919566326, 35.77185877492271, -2.710990616463335 ], "geometry": { "type": "Polygon", "coordinates": [ [ [34.80044299251402, -3.7056906919566326], [34.79870551983084, -2.712838434794184], [35.77185877492271, -2.710990616463335], [35.610236308119404, -3.449883225671188], [35.5534261567553, -3.7038326347645305], [34.80044299251402, -3.7056906919566326] ] ] }, "properties": { "datetime": "2020-07-20T08:10:56Z", "platform": "sentinel-2b", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "data_coverage": 87.34, "view:off_nadir": 0, "eo:cloud_cover": 14.84, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2B_MSIL2A_20200720T074619_N0214_R135_T36MYB_20200720T112901", "created": "2020-09-19T10:48:10.455Z", "updated": "2020-09-19T10:48:10.455Z", "sentinel:valid_cloud_cover": true, "sentinel:utm_zone": 36, "sentinel:data_coverage": 87.34 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/20/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/20/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/20/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36MYB_20200720_0_L2A" }, { "rel": "canonical", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200720_0_L2A/S2B_36MYB_20200720_0_L2A.json", "type": "application/json" }, { "rel": "derived_from", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items/S2B_36MYB_20200720_0_L2A", "title": "Source STAC Item", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2B_36MYB_20200717_0_L2A", "bbox": [ 35.36833009312432, -3.704348921313967, 35.78841174435426, -2.7109586089556537 ], "geometry": { "type": "Polygon", "coordinates": [ [ [35.36833009312432, -3.704348921313967], [35.407819165099255, -3.5245820359904383], [35.588411026975066, -2.7113990960324563], [35.785723330760796, -2.7109586089556537], [35.78841174435426, -3.7031212858685087], [35.36833009312432, -3.704348921313967] ] ] }, "properties": { "datetime": "2020-07-17T08:01:00Z", "platform": "sentinel-2b", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "data_coverage": 31.12, "view:off_nadir": 0, "eo:cloud_cover": 99.94, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2B_MSIL2A_20200717T073619_N0214_R092_T36MYB_20200718T133139", "created": "2020-08-28T01:04:04.443Z", "updated": "2020-08-28T01:04:04.443Z", "sentinel:valid_cloud_cover": true, "sentinel:utm_zone": 36, "sentinel:data_coverage": 31.12 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/17/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/17/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/17/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36MYB_20200717_0_L2A" }, { "rel": "canonical", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200717_0_L2A/S2B_36MYB_20200717_0_L2A.json", "type": "application/json" }, { "rel": "derived_from", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items/S2B_36MYB_20200717_0_L2A", "title": "Source STAC Item", "type": "application/json" }, { "rel": "derived_from", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items/S2B_36MYB_20200717_0_L2A", "title": "Source STAC Item", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2A_36MYB_20200715_0_L2A", "bbox": [ 34.79870551983084, -3.7056906919566326, 35.762423439929975, -2.7110123074852157 ], "geometry": { "type": "Polygon", "coordinates": [ [ [34.80044299251402, -3.7056906919566326], [34.79870551983084, -2.712838434794184], [35.762423439929975, -2.7110123074852157], [35.58950948158126, -3.500550130276924], [35.54379972231785, -3.7038604423827857], [34.80044299251402, -3.7056906919566326] ] ] }, "properties": { "datetime": "2020-07-15T08:10:59Z", "platform": "sentinel-2a", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "data_coverage": 86.39, "view:off_nadir": 0, "eo:cloud_cover": 98.19, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2A_MSIL2A_20200715T074621_N0214_R135_T36MYB_20200715T120435", "created": "2020-08-27T22:44:05.523Z", "updated": "2020-08-27T22:44:05.523Z", "sentinel:valid_cloud_cover": true, "sentinel:utm_zone": 36, "sentinel:data_coverage": 86.39 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/15/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/15/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/15/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_36MYB_20200715_0_L2A" }, { "rel": "canonical", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200715_0_L2A/S2A_36MYB_20200715_0_L2A.json", "type": "application/json" }, { "rel": "derived_from", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items/S2A_36MYB_20200715_0_L2A", "title": "Source STAC Item", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2A_36MYB_20200712_0_L2A", "bbox": [ 35.36185349129008, -3.7043662836689135, 35.78841174435426, -2.7109586089556537 ], "geometry": { "type": "Polygon", "coordinates": [ [ [35.36185349129008, -3.7043662836689135], [35.527401739826665, -2.9456144416642576], [35.579424700604456, -2.711418387850691], [35.785723330760796, -2.7109586089556537], [35.78841174435426, -3.7031212858685087], [35.36185349129008, -3.7043662836689135] ] ] }, "properties": { "datetime": "2020-07-12T08:01:03Z", "platform": "sentinel-2a", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "data_coverage": 31.92, "view:off_nadir": 0, "eo:cloud_cover": 12.02, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2A_MSIL2A_20200712T073621_N0214_R092_T36MYB_20200712T110016", "created": "2020-08-18T09:03:03.858Z", "updated": "2020-08-18T09:03:03.858Z", "sentinel:valid_cloud_cover": true, "sentinel:utm_zone": 36, "sentinel:data_coverage": 31.92 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/12/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/12/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/12/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_36MYB_20200712_0_L2A" }, { "rel": "derived_from", "href": "s3://cirrus-v0-data-1qm7gekzjucbq/sentinel-s2-l2a/36/M/YB/2020/7/S2A_36MYB_20200712_0_L2A/S2A_36MYB_20200712_0_L2A.json", "title": "Source STAC Item", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2B_36MYB_20200710_0_L2A", "bbox": [ 34.79870551983084, -3.7056906919566326, 35.77024131589214, -2.7109943401109917 ], "geometry": { "type": "Polygon", "coordinates": [ [ [34.80044299251402, -3.7056906919566326], [34.79870551983084, -2.712838434794184], [35.77024131589214, -2.7109943401109917], [35.60554409132611, -3.4622467746776224], [35.55171700875178, -3.7038375795975336], [34.80044299251402, -3.7056906919566326] ] ] }, "properties": { "datetime": "2020-07-10T08:10:56Z", "platform": "sentinel-2b", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "data_coverage": 87.17, "view:off_nadir": 0, "eo:cloud_cover": 10.17, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2B_MSIL2A_20200710T074619_N0214_R135_T36MYB_20200710T115611", "created": "2020-08-18T12:01:03.380Z", "updated": "2020-08-18T12:01:03.380Z", "sentinel:valid_cloud_cover": true, "sentinel:utm_zone": 36, "sentinel:data_coverage": 87.17 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/10/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/10/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/10/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36MYB_20200710_0_L2A" }, { "rel": "derived_from", "href": "s3://cirrus-v0-data-1qm7gekzjucbq/sentinel-s2-l2a/36/M/YB/2020/7/S2B_36MYB_20200710_0_L2A/S2B_36MYB_20200710_0_L2A.json", "title": "Source STAC Item", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2B_36MYB_20200707_0_L2A", "bbox": [ 35.363651831328085, -3.704361467477506, 35.78841174435426, -2.7109586089556537 ], "geometry": { "type": "Polygon", "coordinates": [ [ [35.363651831328085, -3.704361467477506], [35.58391733787192, -2.7114087514517324], [35.785723330760796, -2.7109586089556537], [35.78841174435426, -3.7031212858685087], [35.363651831328085, -3.704361467477506] ] ] }, "properties": { "datetime": "2020-07-07T08:01:01Z", "platform": "sentinel-2b", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "data_coverage": 31.59, "view:off_nadir": 0, "eo:cloud_cover": 19.61, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2B_MSIL2A_20200707T073619_N0214_R092_T36MYB_20200707T113322", "created": "2020-09-18T22:19:41.465Z", "updated": "2020-09-18T22:19:41.465Z", "sentinel:valid_cloud_cover": true, "sentinel:utm_zone": 36, "sentinel:data_coverage": 31.59 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/7/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/7/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/7/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36MYB_20200707_0_L2A" }, { "rel": "canonical", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2B_36MYB_20200707_0_L2A/S2B_36MYB_20200707_0_L2A.json", "type": "application/json" }, { "rel": "derived_from", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items/S2B_36MYB_20200707_0_L2A", "title": "Source STAC Item", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2A_36MYB_20200705_0_L2A", "bbox": [ 34.79870551983084, -3.7056906919566326, 35.762781954089746, -2.7110114846428153 ], "geometry": { "type": "Polygon", "coordinates": [ [ [34.80044299251402, -3.7056906919566326], [34.79870551983084, -2.712838434794184], [35.762781954089746, -2.7110114846428153], [35.543081537096796, -3.703862512777141], [34.80044299251402, -3.7056906919566326] ] ] }, "properties": { "datetime": "2020-07-05T08:10:58Z", "platform": "sentinel-2a", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "view:off_nadir": 0, "eo:cloud_cover": 12.46, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2A_MSIL2A_20200705T074621_N0214_R135_T36MYB_20200705T111345", "sentinel:data_coverage": 86.27, "created": "2020-08-27T21:35:09.718Z", "updated": "2020-08-27T21:35:09.718Z", "sentinel:valid_cloud_cover": true, "sentinel:utm_zone": 36 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/5/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/5/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/5/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_36MYB_20200705_0_L2A" }, { "rel": "canonical", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200705_0_L2A/S2A_36MYB_20200705_0_L2A.json", "type": "application/json" }, { "title": "Source STAC Item", "rel": "derived_from", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items/S2A_36MYB_20200705_0_L2A", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] }, { "type": "Feature", "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "view", "proj"], "id": "S2A_36MYB_20200702_0_L2A", "bbox": [ 35.356093300381986, -3.704381685556869, 35.78841174435426, -2.7109586089556537 ], "geometry": { "type": "Polygon", "coordinates": [ [ [35.356093300381986, -3.704381685556869], [35.39966829397743, -3.506566766775803], [35.57726786230264, -2.7114230081670594], [35.785723330760796, -2.7109586089556537], [35.78841174435426, -3.7031212858685087], [35.356093300381986, -3.704381685556869] ] ] }, "properties": { "datetime": "2020-07-02T08:01:04Z", "platform": "sentinel-2a", "constellation": "sentinel-2", "instruments": ["msi"], "gsd": 10, "data_coverage": 32.37, "view:off_nadir": 0, "eo:cloud_cover": 55.53, "proj:epsg": 32736, "sentinel:latitude_band": "M", "sentinel:grid_square": "YB", "sentinel:sequence": "0", "sentinel:product_id": "S2A_MSIL2A_20200702T073621_N0214_R092_T36MYB_20200702T105954", "created": "2020-08-27T21:10:40.692Z", "updated": "2020-08-27T21:10:40.692Z", "sentinel:valid_cloud_cover": true, "sentinel:utm_zone": 36, "sentinel:data_coverage": 32.37 }, "collection": "sentinel-s2-l2a-cogs", "assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "href": "https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/36/M/YB/2020/7/2/0/preview.jpg" }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/L2A_PVI.tif", "proj:shape": [343, 343], "proj:transform": [320, 0, 699960, 0, -320, 9700000, 0, 0, 1] }, "info": { "title": "Original JSON metadata", "type": "application/json", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/2/0/tileInfo.json" }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "href": "https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/36/M/YB/2020/7/2/0/metadata.xml" }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/TCI.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B01.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B02.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B03.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B04.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B05.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B06.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B07.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B08.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B8A.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B09.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B11.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/B12.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/AOT.tif", "proj:shape": [1830, 1830], "proj:transform": [60, 0, 699960, 0, -60, 9700000, 0, 0, 1] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/WVP.tif", "proj:shape": [10980, 10980], "proj:transform": [10, 0, 699960, 0, -10, 9700000, 0, 0, 1] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/SCL.tif", "proj:shape": [5490, 5490], "proj:transform": [20, 0, 699960, 0, -20, 9700000, 0, 0, 1] } }, "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_36MYB_20200702_0_L2A" }, { "rel": "canonical", "href": "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/M/YB/2020/7/S2A_36MYB_20200702_0_L2A/S2A_36MYB_20200702_0_L2A.json", "type": "application/json" }, { "rel": "derived_from", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items/S2A_36MYB_20200702_0_L2A", "title": "Source STAC Item", "type": "application/json" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "collection", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" } ] } ], "collections": [ { "id": "sentinel-s2-l2a-cogs", "stac_version": "1.0.0-beta.2", "description": "Sentinel-2a and Sentinel-2b imagery, processed to Level 2A (Surface Reflectance) and converted to Cloud-Optimized GeoTIFFs", "links": [ { "rel": "self", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs" }, { "rel": "license", "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice" }, { "rel": "about", "href": "https://github.com/stac-utils/stac-sentinel" }, { "rel": "parent", "href": "https://earth-search.aws.element84.com/v0/" }, { "rel": "root", "href": "https://earth-search.aws.element84.com/v0/" }, { "rel": "items", "href": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items" } ], "stac_extensions": ["item-assets"], "title": "Sentinel 2 L2A COGs", "keywords": ["sentinel", "earth observation", "esa"], "providers": [ { "name": "ESA", "roles": ["producer"], "url": "https://earth.esa.int/web/guest/home" }, { "name": "Sinergise", "roles": ["processor"], "url": "https://registry.opendata.aws/sentinel-2/" }, { "name": "AWS", "roles": ["host"], "url": "http://sentinel-pds.s3-website.eu-central-1.amazonaws.com/" }, { "name": "Element 84", "roles": ["processor"], "url": "https://element84.com" } ], "summaries": { "platform": ["sentinel-2a", "sentinel-2b"], "constellation": ["sentinel-2"], "instruments": ["msi"], "gsd": [10], "view:off_nadir": [0] }, "item_assets": { "thumbnail": { "title": "Thumbnail", "type": "image/png", "roles": ["thumbnail"] }, "overview": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["overview"], "gsd": 10, "eo:bands": [ { "name": "B04", "common_name": "red", "center_wavelength": 0.6645, "full_width_half_max": 0.038 }, { "name": "B03", "common_name": "green", "center_wavelength": 0.56, "full_width_half_max": 0.045 }, { "name": "B02", "common_name": "blue", "center_wavelength": 0.4966, "full_width_half_max": 0.098 } ] }, "info": { "title": "Original JSON metadata", "type": "application/json", "roles": ["metadata"] }, "metadata": { "title": "Original XML metadata", "type": "application/xml", "roles": ["metadata"] }, "visual": { "title": "True color image", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["overview"], "gsd": 10, "eo:bands": [ { "name": "B04", "common_name": "red", "center_wavelength": 0.6645, "full_width_half_max": 0.038 }, { "name": "B03", "common_name": "green", "center_wavelength": 0.56, "full_width_half_max": 0.045 }, { "name": "B02", "common_name": "blue", "center_wavelength": 0.4966, "full_width_half_max": 0.098 } ] }, "B01": { "title": "Band 1 (coastal)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 60, "eo:bands": [ { "name": "B01", "common_name": "coastal", "center_wavelength": 0.4439, "full_width_half_max": 0.027 } ] }, "B02": { "title": "Band 2 (blue)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 10, "eo:bands": [ { "name": "B02", "common_name": "blue", "center_wavelength": 0.4966, "full_width_half_max": 0.098 } ] }, "B03": { "title": "Band 3 (green)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 10, "eo:bands": [ { "name": "B03", "common_name": "green", "center_wavelength": 0.56, "full_width_half_max": 0.045 } ] }, "B04": { "title": "Band 4 (red)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 10, "eo:bands": [ { "name": "B04", "common_name": "red", "center_wavelength": 0.6645, "full_width_half_max": 0.038 } ] }, "B05": { "title": "Band 5", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 20, "eo:bands": [ { "name": "B05", "center_wavelength": 0.7039, "full_width_half_max": 0.019 } ] }, "B06": { "title": "Band 6", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 20, "eo:bands": [ { "name": "B06", "center_wavelength": 0.7402, "full_width_half_max": 0.018 } ] }, "B07": { "title": "Band 7", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 20, "eo:bands": [ { "name": "B07", "center_wavelength": 0.7825, "full_width_half_max": 0.028 } ] }, "B08": { "title": "Band 8 (nir)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 10, "eo:bands": [ { "name": "B08", "common_name": "nir", "center_wavelength": 0.8351, "full_width_half_max": 0.145 } ] }, "B8A": { "title": "Band 8A", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 20, "eo:bands": [ { "name": "B8A", "center_wavelength": 0.8648, "full_width_half_max": 0.033 } ] }, "B09": { "title": "Band 9", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 60, "eo:bands": [ { "name": "B09", "center_wavelength": 0.945, "full_width_half_max": 0.026 } ] }, "B11": { "title": "Band 11 (swir16)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 20, "eo:bands": [ { "name": "B11", "common_name": "swir16", "center_wavelength": 1.6137, "full_width_half_max": 0.143 } ] }, "B12": { "title": "Band 12 (swir22)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "gsd": 20, "eo:bands": [ { "name": "B12", "common_name": "swir22", "center_wavelength": 2.22024, "full_width_half_max": 0.242 } ] }, "AOT": { "title": "Aerosol Optical Thickness (AOT)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"] }, "WVP": { "title": "Water Vapour (WVP)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"] }, "SCL": { "title": "Scene Classification Map (SCL)", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"] } }, "extent": { "spatial": { "bbox": [[-180, -90, 180, 90]] }, "temporal": { "interval": [["2015-06-27T10:25:31.456000Z", null]] } }, "license": "proprietary" } ], "links": [] } ================================================ FILE: intake/readers/tests/cats/test_sql.py ================================================ import os import pandas as pd import pytest from intake.readers import catalogs, datatypes, readers pytest.importorskip("pytest_postgresql") pytest.importorskip("psycopg") @pytest.fixture # uses pytest-postgresql def postgres_with_data(postgresql): """Check main postgresql fixture.""" cur = postgresql.cursor() cur.execute( "create table t_random as select s, md5(random()::text) from generate_Series(1,50) s;" ) postgresql.commit() cur.close() return int(cur._conn.pgconn.port) # this is the one I found to be dynamic @pytest.mark.skipif(os.name == "nt", reason="`postgresql` does not work on Windows") def test_pg_pandas(postgres_with_data): pytest.importorskip("psycopg2") pytest.importorskip("sqlalchemy") data = datatypes.SQLQuery( conn=f"postgresql://postgres@127.0.0.1:{postgres_with_data}/tests", query="t_random", ) reader = readers.PandasSQLAlchemy(data) out = reader.read() assert len(out) == 50 out = reader.discover() assert len(out) == 10 @pytest.mark.skipif(os.name == "nt", reason="`postgresql` does not work on Windows") def test_pg_duck_with_pandas_input(postgres_with_data): data = datatypes.SQLQuery( conn=f"postgresql://postgres@127.0.0.1:{postgres_with_data}/tests", query="t_random", ) reader = readers.DuckSQL(data) out = reader.read() assert len(out) == 50 out = reader.discover() assert len(out) == 10 @pytest.fixture def sqlite_with_data(tmpdir): """Check main postgresql fixture.""" pytest.importorskip("sqlalchemy", minversion="2") import sqlite3 fn = f"{tmpdir}/test.db" cnx = sqlite3.connect(fn) df = pd.DataFrame({"a": ["hi", "ho"] * 1000}) df.to_sql(name="oi", con=cnx) return fn def test_sqlite_pandas(sqlite_with_data): pytest.importorskip("pandas", minversion="2", reason="Not working on earlier version of pandas") data = datatypes.SQLQuery(conn=f"sqlite:///{sqlite_with_data}", query="oi") reader = readers.PandasSQLAlchemy(data) out = reader.read() assert len(out) == 2000 out = reader.discover() assert len(out) == 10 def test_sqlite_duck_with_pandas_input(sqlite_with_data): data = datatypes.SQLQuery(conn=f"sqlite:///{sqlite_with_data}", query="oi") reader = readers.DuckSQL(data) out = reader.read() assert len(out) == 2000 out = reader.discover() assert len(out) == 10 def test_pandas_duck_pandas(sqlite_with_data): pytest.importorskip("pandas", minversion="2", reason="Not working on earlier version of pandas") data = datatypes.SQLQuery(conn=f"sqlite:///{sqlite_with_data}", query="oi") reader = readers.PandasSQLAlchemy(data) comment_dict = {"args": [1, "2"], "kwargs": {"true": False}} reader2 = reader.PandasToDuck("out", comment=str(comment_dict)) reader3 = reader2.DuckToPandas() out = reader3.read() assert out[:2].to_dict() == {"a": {0: "hi", 1: "ho"}, "index": {0: 0, 1: 1}} reader = datatypes.SQLQuery({}, "SELECT comment FROM duckdb_tables();").to_reader( reader="DuckSQL" ) assert reader.read().fetchall() == [(str(comment_dict),)] def test_cat(sqlite_with_data): pytest.importorskip("pandas", minversion="2", reason="Not working on earlier version of pandas") data = datatypes.Service(url=f"sqlite:///{sqlite_with_data}") reader = catalogs.SQLAlchemyCatalog(data) cat = reader.read() assert list(cat.data) == ["oi"] out = cat["oi"].to_reader(outtype="pandas").read() assert len(out) == 2000 ================================================ FILE: intake/readers/tests/cats/test_stac.py ================================================ import os import pytest import intake.readers.datatypes here = os.path.dirname(os.path.abspath(__file__)) cat_url = os.path.join(here, "stac_data", "1.0.0", "catalog", "catalog.json") simple_item_url = os.path.join(here, "stac_data", "1.0.0", "collection", "simple-item.json") pytest.importorskip("pystac") def test_1(): data = intake.readers.datatypes.STACJSON(cat_url) cat = data.to_reader(reader="StacCatalog").read() assert "test" in cat cat2 = cat.test.read() assert isinstance(cat2, intake.entry.Catalog) assert cat2.metadata["description"] == "child catalog" def test_bands(): data = intake.readers.datatypes.STACJSON(simple_item_url) list_of_bands = ["B02", "B03"] reader = data.to_reader_cls(reader="Bands")(data, list_of_bands).read() assert isinstance(reader.kwargs["data"], intake.readers.datatypes.TIFF) assert len(reader.kwargs["data"].url) == 2 ================================================ FILE: intake/readers/tests/cats/test_thredds.py ================================================ import pytest import intake.readers pytest.importorskip("siphon") pytest.importorskip("xarray") pytest.importorskip("h5netcdf") def test_1(): req = pytest.importorskip("requests") u = "https://psl.noaa.gov/thredds/catalog.xml" try: req.head(u) assert req.ok except: pytest.xfail("server down") data = intake.readers.datatypes.THREDDSCatalog(url=u) ds = ( data.to_reader() .THREDDSCatToMergedDataset( path="Datasets/ncep.reanalysis.dailyavgs/surface/air.sig995.194*.nc" ) .read() ) assert "1948-01-01" in str(ds.time.min()) assert "1949-12-31" in str(ds.time.max()) ================================================ FILE: intake/readers/tests/cats/test_tiled.py ================================================ import shlex import subprocess import time import pytest import intake.readers.datatypes tiled = pytest.importorskip("tiled") @pytest.fixture() def tiled_server(): t0 = time.time() cmd = "tiled serve demo --port 8901" fail = False P = subprocess.Popen( shlex.split(cmd), stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, bufsize=1, text=True, ) while True: line = P.stderr.readline() if "api_key=" in line: url = line.lstrip().rstrip() # fails if another tiled server is already running yield url break time.sleep(0.01) if time.time() - t0 > 5: fail = True break P.kill() P.wait() if fail: raise RuntimeError("tiled fixture did not start") def test_catalog_workflow(tiled_server): from tiled.queries import FullText data = intake.readers.datatypes.TiledService(tiled_server) node = data.to_reader("tiled") cat = intake.entry.Catalog() cat["cat"] = ( node.TiledSearch(query=FullText("dog")) .TiledSearch(query=FullText("red")) .TiledNodeToCatalog ) cat.to_yaml_file("memory://cat.yaml") cat2 = intake.entry.Catalog.from_yaml_file("memory://cat.yaml") with intake.conf.set(allow_pickle=True): # tiled query instances are pickled scat = cat2["cat"].read() assert "short_table" in scat ================================================ FILE: intake/readers/tests/test_basic.py ================================================ import os from intake.readers import datatypes, readers, entry here = os.path.dirname(__file__) testdir = os.path.abspath(os.path.join(here, "..", "..", "catalog/tests")) def test1(): data = datatypes.CSV(url=f"{testdir}/entry1_1.csv") reader = readers.PandasCSV(data) assert reader.doc() out = reader.read() assert list(out.columns) == ["name", "score", "rank"] def test_recommend_filetype(): assert datatypes.Parquet in datatypes.recommend(url="myfile.parq") assert datatypes.Parquet in datatypes.recommend(head=b"PAR1") assert all( p in datatypes.recommend(mime="text/yaml", head=False) for p in {datatypes.YAMLFile, datatypes.CatalogFile, datatypes.Text} ) def test_recommend_reader(): pp = datatypes.Parquet("") rec = readers.recommend(pp) assert all(p not in rec["importable"] for p in rec["not_importable"]) assert all(p not in rec["not_importable"] for p in rec["importable"]) assert all( p in rec["importable"] + rec["not_importable"] for p in {readers.PandasParquet, readers.AwkwardParquet, readers.DaskParquet} ) pp = datatypes.CSV("") assert readers.PandasCSV in readers.recommend(pp)["importable"] def test_data_metadata(): cat = entry.Catalog() cat["d"] = datatypes.BaseData(metadata={"oi": "io"}) cat.get_entity("d").metadata.update(blag=0) out = cat["d"] assert out.metadata == {"oi": "io", "blag": 0} ================================================ FILE: intake/readers/tests/test_consistency.py ================================================ import pytest import intake from intake.readers.utils import subclasses from intake.readers.readers import FileReader from intake import BaseConverter from intake.readers.convert import SameType @pytest.mark.parametrize("cls", subclasses(intake.BaseReader)) def test_readers(cls): assert isinstance(cls.imports, set) assert all(isinstance(_, str) for _ in cls.imports) assert isinstance(cls.implements, set) assert all(issubclass(_, intake.BaseData) for _ in cls.implements) assert isinstance(cls.func, str) assert cls.func.count(":") == 1 assert cls.func_doc is None or cls.func_doc.count(":") == 1 assert isinstance(cls.output_instance, str) or cls.output_instance is None if cls.other_funcs: assert isinstance(cls.other_funcs, set) and all(isinstance(c, str) for c in cls.other_funcs) @pytest.mark.parametrize("cls", subclasses(intake.BaseData)) def test_data(cls): assert isinstance(cls.filepattern, str) assert isinstance(cls.mimetypes, str) assert isinstance(cls.structure, set) and all(isinstance(s, str) for s in cls.structure) assert isinstance(cls.magic, set) and all(isinstance(m, (bytes, tuple)) for m in cls.magic) assert isinstance(cls.contains, set) and all(isinstance(c, str) for c in cls.contains) @pytest.mark.parametrize("cls", subclasses(FileReader)) def test_filereaders(cls): assert isinstance(cls.url_arg, str) @pytest.mark.parametrize("cls", subclasses(BaseConverter)) def test_converters(cls): assert all(isinstance(s, str) and (s.count(":") == 1 or s == ".*") for s in cls.instances) assert all( s is SameType or isinstance(s, str) and (s.count(":") == 1 or s == ".*") for s in cls.instances.values() ) ================================================ FILE: intake/readers/tests/test_dict.py ================================================ import intake.readers from intake.readers import entry def test_yaml_roundtrip(): cat = entry.Catalog() cat["one"] = intake.readers.BaseReader(intake.BaseData(), output_instance="blah") cat.to_yaml_file("memory://cat.yaml") cat2 = entry.Catalog.from_yaml_file("memory://cat.yaml") assert cat2.user_parameters.pop("CATALOG_DIR") assert cat2.user_parameters.pop("STORAGE_OPTIONS") == {} assert cat.data == cat2.data assert list(cat.entries) == list(cat2.entries) assert cat2["one"].output_instance == "blah" ================================================ FILE: intake/readers/tests/test_errors.py ================================================ import pytest import intake def test_func_ser(): class A: def get(self): def inner(): return self.x return inner func = A().get() reader = intake.readers.convert.GenericFunc(data=func) cat = intake.Catalog() with pytest.raises(RuntimeError): cat.add_entry(reader) ================================================ FILE: intake/readers/tests/test_reader.py ================================================ import tempfile import pytest import intake def test_reader_from_call(): import pandas as pd df = pd.DataFrame( { "col1": ["a", "b"], "col2": [1.0, 3.0], }, columns=["col1", "col2"], ) with tempfile.NamedTemporaryFile(delete=False) as fp: df.to_csv(fp.name) fp.close() reader = intake.reader_from_call("df = pd.read_csv(fp.name)") read_df = reader.read() assert all(read_df.col1 == df.col1) assert all(read_df.col2 == df.col2) @pytest.fixture() def xarray_dataset(): xr = pytest.importorskip("xarray") import numpy as np import pandas as pd temperature = 15 + 8 * np.random.randn(2, 3, 4) lon = [-99.83, -99.32] lat = [42.25, 42.21] instruments = ["manufac1", "manufac2", "manufac3"] time = pd.date_range("2014-09-06", periods=4) return xr.Dataset( data_vars=dict( temperature=(["loc", "instrument", "time"], temperature), ), coords=dict( lon=("loc", lon), lat=("loc", lat), instrument=instruments, time=time, ), ) def test_xarray_pattern(tmpdir, xarray_dataset): import numpy as np from intake.readers.readers import XArrayPatternReader if np.__version__.split(".") > ["2"]: pytest.skip("HDF does not yet support numpy 2") pytest.importorskip("h5netcdf") path1 = f"{tmpdir}/1.nc" path2 = f"{tmpdir}/2.nc" xarray_dataset.to_netcdf(path1) xarray_dataset.to_netcdf(path2) data = intake.datatypes.HDF5("%s/{part}.nc" % tmpdir) reader = XArrayPatternReader(data) ds = reader.read() assert ds.part.values.tolist() == ["1", "2"] assert ds.temperature.shape == (2, 2, 3, 4) data = intake.datatypes.HDF5("%s/{part:d}.nc" % tmpdir) reader = XArrayPatternReader(data) ds = reader.read() assert ds.part.values.tolist() == [1, 2] def test_xarray_dataset_remote_url_glob_str(tmpdir, xarray_dataset): """Test opening HDF5 data with Xarray via remote URL with glob. We're using tar archive to create a remote URL, since this was the case that raised Issue #879. """ from pathlib import Path import shutil from intake.readers.readers import XArrayDatasetReader pytest.importorskip("h5netcdf") root_dir = Path(tmpdir) path = root_dir / "test.nc" xarray_dataset.to_netcdf(path) # make archive.tar.gz in tmpdir tarname = f"{tmpdir}/archive" tarpath = shutil.make_archive(tarname, "gztar", root_dir=root_dir, base_dir="test.nc") data_url = "tar://*.nc::" + tarpath data = intake.datatypes.HDF5(data_url) reader = XArrayDatasetReader(data) ds = reader.read() # check that result is not empty assert ds.sizes.get("time", 0) > 0 @pytest.fixture def icechunk_xr_repo(tmpdir): xr = pytest.importorskip("xarray") icechunk = pytest.importorskip("icechunk") pd = pytest.importorskip("pandas") import numpy as np np.random.seed(0) temperature = 15 + 8 * np.random.randn(2, 3, 4) precipitation = 10 * np.random.rand(2, 3, 4) lon = [-99.83, -99.32] lat = [42.25, 42.21] instruments = ["manufac1", "manufac2", "manufac3"] time = pd.date_range("2014-09-06", periods=4) reference_time = pd.Timestamp("2014-09-05") ds = xr.Dataset( data_vars=dict( temperature=(["loc", "instrument", "time"], temperature), precipitation=(["loc", "instrument", "time"], precipitation), ), coords=dict( lon=("loc", lon), lat=("loc", lat), instrument=instruments, time=time, reference_time=reference_time, ), attrs=dict(description="Weather related data."), ) storage = icechunk.local_filesystem_storage(tmpdir.strpath) repo = icechunk.Repository.create(storage) session = repo.writable_session("main") store = session.store ds.to_zarr(store, consolidated=False) session.commit("Initial commit") return tmpdir.strpath def test_icechunk(icechunk_xr_repo): data = intake.readers.datatypes.IcechunkRepo( "local_filesystem", storage_options={"path": icechunk_xr_repo}, ref="main" ) reader = intake.readers.XArrayDatasetReader(data) ds = reader.read() assert (~ds.temperature.isnull()).all() ================================================ FILE: intake/readers/tests/test_search.py ================================================ from intake.readers.entry import Catalog, ReaderDescription from intake.readers.readers import BaseReader from intake.readers.search import Importable, Text class NotImportable(BaseReader): # TODO: this will show up in the class hierarchy throughout testing - make a temporary mock? # see https://stackoverflow.com/a/52428851/3821154 imports = {"unknown_package"} def test_1(): cat = Catalog() cat["en1"] = ReaderDescription("intake.readers.readers:BaseReader", kwargs={"allow_me": True}) cat["en2"] = ReaderDescription("intake.readers.readers:BaseReader", kwargs={"nope": True}) cat["en3"] = ReaderDescription( "intake.readers.tests.test_search:NotImportable", kwargs={"allow_me": True} ) # simple text cat2 = cat.search("allow") assert "en1" in cat2 assert "en2" not in cat2 assert "en3" in cat2 assert cat2["en3"] == cat["en3"] # single term cat2 = cat.search(Importable()) assert "en1" in cat2 assert "en2" in cat2 assert "en3" not in cat2 # expression cat2 = cat.search(Importable() & Text("allow")) assert "en1" in cat2 assert "en2" not in cat2 assert "en3" not in cat2 ================================================ FILE: intake/readers/tests/test_up.py ================================================ import pytest def test_basic(): from intake.readers import user_parameters as up p = up.SimpleUserParameter(default=1, dtype=int) pars = {"k": ["{p}", 1]} out = up.set_values({"p": p}, pars) assert out == {"k": [1, 1]} pars = {"k": ["{p}", 1], "p": 2} out = up.set_values({"p": p}, pars) assert out == {"k": [2, 1]} # extra space here results in list member being string formatted pars = {"k": [" {p}", 1], "p": 2} out = up.set_values({"p": p}, pars) assert out == {"k": [" 2", 1]} with pytest.raises(TypeError): # supplied None as a value to int parameter pars = {"k": ["{p}", 1], "p": None} up.set_values({"p": p}, pars) def test_named_options(): from intake.readers import user_parameters as up p = up.NamedOptionsUserParameter({"a": "athing", "b": "bthing"}, default="b") pars = {"k": ["{p}", 1]} out = up.set_values({"p": p}, pars) assert out == {"k": ["bthing", 1]} pars = {"k": ["{p}", 1], "p": "a"} out = up.set_values({"p": p}, pars) assert out == {"k": ["athing", 1]} ================================================ FILE: intake/readers/tests/test_utils.py ================================================ import pytest from intake.readers.utils import LazyDict, PartlyLazyDict class OnlyOkeKey(LazyDict): def __getitem__(self, item): if item == 5: return 5 raise KeyError def __iter__(self): return iter(range(10)) def test_lazy_dict(): ld = OnlyOkeKey() assert list(ld) == list(range(10)) assert 5 in ld assert ld[5] == 5 with pytest.raises(KeyError): ld[4] pld = PartlyLazyDict({12: 2}, ld) assert set(pld) == {12} | set(range(10)) assert 12 in pld assert 5 in pld assert 0 in pld assert pld[12] == 2 assert pld[5] == 5 with pytest.raises(KeyError): pld[0] ================================================ FILE: intake/readers/tests/test_workflows.py ================================================ import os import fsspec import pytest import intake.readers from intake.readers import convert, readers, utils, entry pd = pytest.importorskip("pandas") bindata = b"apple,beet,carrot\n" + b"a,1,0.1\nb,2,0.2\nc,3,0.3\n" * 100 @pytest.fixture() def dataframe_file(): m = fsspec.filesystem("memory") m.pipe("/data", bindata) return "memory://data" @pytest.fixture() def df(dataframe_file): return pd.read_csv(dataframe_file) def test_pipelines_in_catalogs(dataframe_file, df): data = intake.readers.datatypes.CSV(url=dataframe_file) reader = intake.readers.PandasCSV(data) reader2 = reader[["apple", "beet"]].set_index(keys="beet") cat = intake.entry.Catalog() cat["mydata"] = reader2 assert cat.mydata.read().equals(df[["apple", "beet"]].set_index(keys="beet")) assert cat.mydata.discover().equals(df[["apple", "beet"]].set_index("beet")[:10]) cat["eq"] = reader.equals(other=reader) assert reader.equals(other=reader).read() is True assert cat.eq.read() is True with pytest.raises(NotImplementedError): cat.delete("eq", recursive=True) cat.delete("eq") assert "eq" not in cat def test_pipeline_steps(dataframe_file, df): data = intake.readers.datatypes.CSV(url=dataframe_file) reader = intake.readers.PandasCSV(data) with pytest.raises(AttributeError): # cannot auto on private methods; could still apply out = reader._nonexistent() breakpoint() reader2 = reader[["apple", "beet"]].set_index(keys="beet") stepper = reader2.read_stepwise() assert isinstance(stepper, convert.PipelineExecution) assert stepper.data is None assert stepper.next[0] == 0 assert isinstance(stepper.step(), convert.PipelineExecution) assert stepper.next[0] == 1 assert stepper.data is not None assert isinstance(stepper.step(), convert.PipelineExecution) out = stepper.step() assert out.equals(df[["apple", "beet"]].set_index(keys="beet")) stepper = reader2.read_stepwise(breakpoint=1) assert stepper.next[0] == 1 assert stepper.data is not None out = stepper.cont() assert out.equals(df[["apple", "beet"]].set_index(keys="beet")) def test_parameters(dataframe_file, monkeypatch): data = intake.readers.datatypes.CSV(url=dataframe_file) reader = readers.PandasCSV(data) reader2 = reader[["apple", "beet"]].set_index(keys="beet") ent = reader2.to_entry() ent.extract_parameter(name="index_key", value="beet") assert ent.user_parameters["index_key"].default == "beet" assert str(ent.to_dict()).count("{index_key}") == 2 # once in select, once in set_index assert utils.descend_to_path("steps.2.2.keys", ent.kwargs) == "{index_key}" assert ent.to_reader() == reader2 cat = entry.Catalog() cat.add_entry(reader2) datadesc = list(cat.data.values())[0] datadesc.extract_parameter(name="protocol", value="memory:") assert datadesc.kwargs["url"] == dataframe_file.replace("memory:", "{protocol}") datadesc.user_parameters["protocol"].set_default("env(TEMP_TEST)") monkeypatch.setenv("TEMP_TEST", "memory:") out = datadesc.to_data() assert out == data def test_namespace(dataframe_file): data = intake.readers.datatypes.CSV(url=dataframe_file) reader = readers.PandasCSV(data) assert "np" in reader._namespaces assert reader.apply(getattr, "beet").np.max().read() == 3 calls = 0 def fails(x): global calls if calls < 2: calls += 1 raise RuntimeError return x def test_retry(dataframe_file): from intake.readers.readers import Retry data = intake.readers.datatypes.CSV(url=dataframe_file) reader = readers.PandasCSV(data) pipe = Retry(reader.apply(fails), allowed_exceptions=(ValueError,)) cat = entry.Catalog() cat["ret1"] = pipe pipe = Retry(reader.apply(fails), allowed_exceptions=(RuntimeError,)) cat["ret2"] = pipe with pytest.raises(RuntimeError): cat["ret1"].read() assert calls == 1 assert cat["ret2"].read() is not None assert calls > 1 def dir_non_empty(d): import os return os.path.exists(d) and os.path.isdir(d) and bool(os.listdir(d)) def test_custom_cache(dataframe_file, tmpdir, df): from intake.readers.readers import Condition, PandasCSV, PandasParquet, FileExistsReader fn = f"{tmpdir}/file.parquet" data = intake.readers.datatypes.CSV(url=dataframe_file) part = PandasCSV(data) output = part.PandasToParquet(url=fn).transform(PandasParquet) data2 = intake.readers.datatypes.Parquet(url=fn) cached = PandasParquet(data=data2) reader2 = Condition(cached, if_false=output, condition=FileExistsReader(data2)) assert os.listdir(tmpdir) == [] out = reader2.read() assert os.listdir(tmpdir) assert df.equals(out) m = fsspec.filesystem("memory") m.rm(dataframe_file) with pytest.raises(IOError): # file has gone part.read() # but condition still picks read from cache out = reader2.read() assert df.equals(out) def test_cat_mapper(dataframe_file): # setup cat = intake.Catalog() data = intake.readers.datatypes.CSV(url=dataframe_file) cat["one"] = readers.PandasCSV(data) cat["two"] = readers.PandasCSV(data, usecols=[0]) cat.to_yaml_file("memory://cat.yaml") reader = intake.readers.YAMLCatalogReader("memory://cat.yaml") reader2 = reader.transform.CatalogMapper(getattr, "shape", transform=False) cat2 = reader2.read() assert "one" in cat2 and "two" in cat2 result1 = cat2["one"].read() assert result1 == (300, 3) result2 = cat2["two"].read() assert result2 == (300, 1) ================================================ FILE: intake/readers/transform.py ================================================ """Manipulate data: functions that change the data but not the container type """ from __future__ import annotations import intake from intake.readers.convert import BaseConverter, SameType from intake.readers.utils import one_to_one class DataFrameColumns(BaseConverter): instances = one_to_one({"pandas:DataFrame", "dask.dataframe:DataFrame"}) func = "pandas:DataFrame.loc" def run(self, x, columns, **_): return x[columns] class XarraySel(BaseConverter): instances = one_to_one({"xarray:Dataset", "xarray:DataArray"}) func = "xarray:Dataset.sel" def run(self, x, indexers, **_): return x.sel(indexers) class THREDDSCatToMergedDataset(BaseConverter): instances = {"intake.readers.catalogs:THREDDSCatalog": "xarray:Dataset"} def run(self, cat, path, driver="h5netcdf", xarray_kwargs=None, concat_kwargs=None, **_): """Merges multiple datasets into a single datasets. Recreates the merged-dataset functionality of intake-thredds This source takes a THREDDS URL and a path to descend down, and calls the combine function on all of the datasets found. Parameters ---------- url : str Location of server path : str, list of str Subcats to follow; include glob characters (*, ?) in here for matching. driver : str Select driver to access data. Choose from 'netcdf' and 'opendap'. xarray_kwargs: dict kwargs to be passed to xr.open_dataset concat_kwargs: dict kwargs to be passed to xr.concat() filled by files opened by xr.open_dataset previously """ import fnmatch import xarray as xr path = path.split("/") if isinstance(path, str) else path if driver not in ["pydap", "h5netcdf"]: raise ValueError xarray_kwargs = xarray_kwargs or {} xarray_kwargs["engine"] = driver for i, part in enumerate(path): if "*" not in part and "?" not in part: cat = cat[part] else: break path = "/".join(path) cat = cat.read() data = [] suffix = {"pydap": "_DAP", "h5netcdf": "_CDF"}[driver] for name in list(cat): if fnmatch.fnmatch(name[:-4], path) and name[-4:] == suffix: data.append(cat[name].read(**xarray_kwargs)) if concat_kwargs: return xr.concat(data, **concat_kwargs) else: return xr.combine_by_coords(data, combine_attrs="override") class PysparkColumns(BaseConverter): instances = {"pyspark.sql:DataFrame": "pyspark.sql:DataFrame"} def run(self, x, columns, **_): return x.select(columns) class Method(BaseConverter): """Call named method on object Assumes output type is the same as input. """ instances = {".*": SameType} def run(self, x, *args, method_name: str = "", **kw): method = getattr(x, method_name) if callable(method): return method(*args, **kw) else: return method class GetItem(BaseConverter): """Equivalent of x[item] Assumes output type is the same as input. """ instances = {".*": SameType} func = "operator:getitem" def _read(self, item, data=None): return data[item] def identity(x): return x class CatalogMapper(BaseConverter): instances = {"intake.readers.entry:Catalog": "intake.readers.entry:Catalog"} def run( self, in_cat: intake.Catalog, func, *args, transform=True, name_arg=None, read=False, **kwargs, ): """ Parameters ---------- func: function to apply to each entry (as callable or string equivalent) transform: do we expect this to be a named transform that intake already knows about? name_arg: if given, pass the entry name to the action to be performed using this as the kwarg name read: if True, execute the pipeline produced. This might be used where the pipeline output is itself another reader. """ out = intake.Catalog() for name in in_cat.entries: if name_arg: kwargs[name_arg] = name if transform: pipe = in_cat[name].__getattr__(func)(*args, **kwargs) else: if isinstance(func, str): func = intake.import_name(func) pipe = in_cat[name].apply(func, *args, **kwargs) if read: out[name] = pipe.read() else: out[name] = pipe return out ================================================ FILE: intake/readers/user_parameters.py ================================================ """ Parametrization of data/reader entries, as they appear in Catalogs Parameters can be used to template values across readers, wither to indicate choices that a user wishes to make at run time, or to require the user to fill in details such as credentials. Providing options in this way is a simpler experience for a user than replicating a reader/pipeline with different options. In a catalog, user parameters can exist at the global scope, to be used anywhere, as a part of the data description, to be inherited by all readers of that data, or as part of the reader-specific descriptions. The value can be set in-place, or provided during ``read()``. """ from __future__ import annotations import builtins import os import re from typing import Any, Iterable from intake import import_name, conf from intake.readers.utils import FormatWithPassthrough, SecurityError, Tokenizable class BaseUserParameter(Tokenizable): """The base class allows for any default without checking/coercing""" def __init__(self, default, description=""): self.default = default #: the value to use without user input self.description = description #: what is the function of this parameter def __repr__(self): dic = {k: v for k, v in self.__dict__.items() if not k.startswith("_")} return f"{type(self).__name__}, {self.description}\n{dic}" def set_default(self, value): """Change the default, if it validates""" value = self.coerce(value) if self.validate(value): self.default = value else: raise ValueError("Could not validate %s with %s", value, self) def with_default(self, value): """A new instance with different default, if it validates (original object is left unchanged) """ import copy up = copy.copy(self) up.set_default(value) return up def coerce(self, value): """Change given type to one that matches this parameter's intent""" return value def _validate(self, value): return True def validate(self, value) -> bool: """Is the given value allowed by this parameter? Exceptions are treated as False """ try: return self._validate(value) except (TypeError, ValueError): return False def to_dict(self): dic = super().to_dict() dic["cls"] = self.qname() return dic class SimpleUserParameter(BaseUserParameter): """This class is enough for simple type coercion.""" def __init__(self, dtype: type = object, **kw): self.dtype = dtype.__name__ if self.dtype not in dir(builtins) or not isinstance(dtype, type): raise ValueError("Only supports classes from the builtins module") super().__init__(**kw) @property def _dtype(self): return getattr(builtins, self.dtype) def coerce(self, value): if not isinstance(value, self._dtype): return self._dtype(value) # works for dtype like str, int, list return value def _validate(self, value): return isinstance(value, self._dtype) class OptionsUserParameter(SimpleUserParameter): """One choice out of a given allow list""" def __init__(self, options, dtype=object, **kw): super().__init__(dtype=dtype, **kw) self.options = {self.coerce(o) for o in options} def _validate(self, value): return self.coerce(value) in self.options class NamedOptionsUserParameter(SimpleUserParameter): """One choice out of a given allow dictionary accessed by its keys In this case, `dtype` is the types of the dictionary values, and we have a separate "keytype" for the keys, str by default """ def __init__(self, options, default, dtype=object, keytype=str, **kw): super().__init__(default=options[default], dtype=dtype, **kw) self.keytype = keytype self.options = options def coerce(self, value): return self.options.get(value, None) def _validate(self, value): if not isinstance(value, self.keytype): value = self.keytype(value) return value in self.options.values() class MultiOptionUserParameter(OptionsUserParameter): """Multiple choices out of a given allow list/tuple In this case, dtype is the type of each member of the list """ def __init__(self, options: list | tuple, dtype=object, **kw): super().__init__(options=options, dtype=dtype, **kw) def coerce_one(self, value): return super().coerce(value) def coerce(self, value): return [self.coerce_one(v) for v in value] def _validate(self, value): return isinstance(value, (list, tuple)) and all(v in self.options for v in value) class BoundedNumberUserParameter(SimpleUserParameter): """A number within a range bound""" def __init__(self, dtype=float, max_value=None, min_value=None, **kw): super().__init__(dtype=dtype, **kw) self.max = max_value self.min = min_value def _validate(self, value): out = True if self.max: out = out and self.max > value if self.min: out = out and self.min < value return out # TODO: Date type and generic functions of user_parameters like date(value).day templates = {} class NoMatch(ValueError): ... def register_template(name): def wrapper(func): regex = re.compile(f"{name}[(]([^)]+)[)]") templates[regex] = func def go(text): m = regex.match(text) if m: return func(m) raise NoMatch return go return wrapper template_env = re.compile(r"env[(]([^)]+)[)]") template_data = re.compile(r"data[(]([^)]+)[)]") template_func = re.compile(r"func[(]([^)]+)[)]") @register_template(r"env") def env(match, up): """Value from an environment variable""" return os.getenv(match.groups()[0]) @register_template(r"data") def data(match, up): """The value from reading a dataset Used in pipelines to point to the outputs of upstream readers """ # TODO: this might never be called, since Catalog._rehydrate does this job from intake.readers.convert import Pipeline from intake.readers.entry import ReaderDescription var = match.groups()[0] if "," in var: var, part = var.split(",") else: part = None thing = up[var.strip()] if isinstance(thing, ReaderDescription): thing = thing.to_reader(user_parameters=up) if part and isinstance(thing, Pipeline): thing = thing.first_n_stages(int(part)) return thing @register_template(r"import") @register_template(r"func") def imp(match, up): """The result of importing the string, an arbitrary python object Format of the input string is like "{import(package.module:object)}" """ if not conf["allow_import"]: from intake.readers.utils import SecurityError raise SecurityError("Arbitrary imports are not allowed by the Intake config") return import_name(match.groups()[0]) @register_template(r"pickle64") def unpickle(match, up): if not conf["allow_pickle"]: raise SecurityError("Unpickling is disallowed by the Intake config") import base64 import pickle return pickle.loads(base64.b64decode(match.groups()[0].encode())) def _set_values(up, arguments): if isinstance(arguments, dict): return {k: _set_values(up, v) for k, v in arguments.copy().items()} elif isinstance(arguments, str) and arguments.startswith("{") and arguments.endswith("}"): arg = arguments[1:-1] if arg in up: return up[arguments[1:-1]] else: for k, v in templates.items(): m = re.match(k, arg) if m: return v(m, up) if isinstance(arguments, str): # missing env keys become empty strings envdict = {f"env({k})": os.getenv(k) for k in template_env.findall(arguments)} data = FormatWithPassthrough(**up) data.update(envdict) try: out = arguments.format_map(data) # missing keys remain unformatted, but don't raise except ValueError: # in case this is a string with genuine "{"s out = arguments return out elif isinstance(arguments, Iterable): return type(arguments)([_set_values(up, v) for v in arguments]) return arguments def set_values(user_parameters: dict[str, BaseUserParameter], arguments: dict[str, Any]): """Walk kwargs and set the value and types of any parameters found If one of arguments matches the name of a user_parameter, it will set the value of that parameter before proceeding. """ up = user_parameters.copy() for k, v in arguments.copy().items(): if k in user_parameters: u = up[k] if isinstance(u, BaseUserParameter): up[k] = up[k].with_default(v) else: up[k] = v arguments.pop(k) for k, v in up.copy().items(): # v can be a literal DataDescription (from a reader) rather than a UP if isinstance(getattr(v, "default", None), str): for templ, func in templates.items(): m = re.match(templ, v.default) if m: up[k] = up[k].with_default(func(m, up)) # m = template_env.match(v.default) # if m: # up[k] = up[k].with_default(os.getenv(m.groups()[0])) # m = template_func.match(v.default) # if m: # var = m.groups()[0] # up[k] = up[k].with_default(import_name(var)) return _set_values( {k: (u.default if isinstance(u, BaseUserParameter) else u) for k, u in up.items()}, arguments, ) ================================================ FILE: intake/readers/utils.py ================================================ from __future__ import annotations import importlib.metadata import numbers import re import typing from functools import lru_cache as cache from hashlib import md5 from itertools import zip_longest from typing import Any, Iterable, Mapping from intake import import_name class SecurityError(RuntimeError): """The given operation is disabled in the Intake config""" def subclasses(cls: type) -> set: """Find all direct and indirect subclasses Most recently created and most specialised classes come first """ out = set() for cl in reversed(cls.__subclasses__()): # TODO: if cls.check_imports exists and returns False, do we descend? out |= subclasses(cl) out.add(cl) return out def merge_dicts(*dicts: dict) -> dict: """Deep-merge dictionary values, latest value wins Examples -------- >>> merge_dicts({"a": {"a": 0, "b": 1}}, {"a": {"a": 1}, "b": 1}) {"a": {"a": 1, "b": 1}}, "b": 1) >>> merge_dicts({"a": [None, True]}, {"a": [False, None]}) {"a": [False, True} """ if isinstance(dicts[0], dict): out = {} for dic in dicts: for k, v in dic.items(): if k in out and isinstance(v, Iterable) and not isinstance(v, (bytes, str)): # deep-merge nested dicts out[k] = merge_dicts(out[k], v) else: out[k] = v elif isinstance(dicts[0], Iterable) and not isinstance(dicts[0], (bytes, str)): stuff = [] for values in zip_longest(*(d for d in dicts if d is not None)): stuff.append(merge_dicts(*values)) out = type(dicts[0])(stuff) else: out = next((d for d in dicts if d is not None), None) return out def nested_keys_to_dict(kw: dict[str, Any]) -> dict: """Nest keys of the form "field.subfield.item" into dicts Examples -------- >>> nested_keys_to_dict({"field": 0, "deeper.field": 1, "deeper.other": 2, "deep.est.field": 3}) {'field': 0, 'deeper': {'field': 1, 'other': 2}, 'deep': {'est': {'field': 3}}} >>> nested_keys_to_dict({"deeper.1.field": 1, "list.1.1.1": True, "list.1.0": False}) {'deeper': [None, {"field": 1}], "list": [None, [False, [None, True]]]} """ out = {} for k, v in kw.items(): bits = k.split(".") o = out for bit, bit2 in zip(bits[:-1], bits[1:]): if bit2.isnumeric(): bit2 = int(bit2) newpart = [None] * (int(bit2) + 1) else: newpart = {} if isinstance(o, dict): o = o.setdefault(bit, newpart) else: if o[int(bit)] is None: o[int(bit)] = newpart o = o[int(bit)] bit = bits[-1] if bit.isnumeric(): bit = int(bit) o[bit] = v return out func_or_method = re.compile(r"<(function|method) ([^ ]+) at 0x[0-9a-f]+>") def find_funcs(val, tokens={}): """Walk nested dict/iterables, replacing functions with string package.mod:func form""" import base64 import pickle from intake.readers import BaseData, BaseReader if isinstance(val, dict): return {k: find_funcs(v, tokens=tokens) for k, v in val.items()} elif isinstance(val, (str, bytes)): return val elif isinstance(val, Iterable): # list, tuple, set-like return type(val)([find_funcs(v, tokens=tokens) for v in val]) if isinstance(val, (BaseReader, BaseData)): ent = val.to_entry() find_funcs(ent, tokens) tok = ent.token tokens[tok] = val return "{data(%s)}" % tok if isinstance(val, Tokenizable): return val.to_dict() elif callable(val): name = "{func(%s)}" % f"{val.__module__}:{val.__name__}" if "" in name or "__main__" in name or getattr(val, "__closure__", False): raise RuntimeError("Cannot store dynamically defined function: %s", val) return name elif val is None or isinstance(val, (numbers.Number, BaseData, BaseReader)): return val else: return "{pickle64(%s)}" % base64.b64encode(pickle.dumps(val)).decode() class LazyDict(Mapping): """Subclass this to make lazy dictionaries, where getting values can be expensive""" _keys = None def __getitem__(self, item): raise NotImplementedError def __len__(self): return len(self.keys()) def keys(self): if self._keys is None: self._keys = set(self) return self._keys def __contains__(self, item): return item in self.keys() def __iter__(self): raise NotImplementedError class PartlyLazyDict(LazyDict): """A dictionary-like, where some components may be lazy""" def __init__(self, *mappings): self.dic = {} self.lazy = [] for mapping in mappings: if isinstance(mapping, LazyDict): self.lazy.append(mapping) else: self.dic.update(mapping) def keys(self): out = set(self.dic) for mapping in self.lazy: out.update(mapping) return out def __len__(self): return len(self.keys()) def __iter__(self): return iter(self.keys()) def __getitem__(self, item): if item in self.dic: return self.dic[item] for mapping in self.lazy: if item in mapping: return mapping[item] raise KeyError(item) def __setitem__(self, key, value): self.dic[key] = value def update(self, data): if isinstance(data, LazyDict): self.lazy.append(data) else: self.dic.update(data) def copy(self): return PartlyLazyDict(self.dic.copy(), *self.lazy) class FormatWithPassthrough(dict): """When calling .format(), use this to replace only those keys that are found""" def __getitem__(self, item): try: return super().__getitem__(item) except KeyError: return "{%s}" % item @cache def check_imports(*imports: Iterable[str]) -> bool: """See if required packages are importable, but don't import them""" import sys try: for package in imports: if package: package in sys.modules or importlib.metadata.distribution(package) return True except (ImportError, ModuleNotFoundError, NameError): return False class Completable: """Helper mixin for classes with dynamic tab completion""" @classmethod @cache def check_imports(cls): return check_imports(*getattr(cls, "imports")) @staticmethod def tab_completion_fixer(item): # just make this a function? if item in { "_ipython_key_completions_", "_ipython_display_", "__wrapped__", "_ipython_canary_method_should_not_exist_", "_render_traceback_", }: raise AttributeError if item.startswith("_repr_"): raise AttributeError class Tokenizable(Completable): """Provides reliable hash/eq support to classes that hold dict attributes The convention is, that attributes starting with _ are not included in the hash and so can be mutated. Changing a non-_ attribute should only be done when making a new instance. """ _tok = None _avoid = {"metadata"} def _dic_for_comp(self): # TODO: we don't consider metadata part of the token. Any others? # Do we just want to exclude others? return { k: find_funcs(v) for k, v in self.__dict__.items() if not k.startswith("_") and k not in self._avoid } def _token(self): dic = self._dic_for_comp() dictxt = func_or_method.sub(r"\2", str(dic)) return md5(f"{self.qname()}|{dictxt}".encode()).hexdigest()[:16] @property def token(self): """Token is computed from all non-_ attributes and then cached. Even if those attributes are mutated, the token will not change, but the resultant might or might not beequal to the original. """ if self._tok is None: self._tok = self._token() return self._tok def __hash__(self): """Hash depends on class name and all non-_* attributes""" return int(self.token, 16) def __eq__(self, other): return type(self) == type(other) and ( self.token == other.token or self.to_dict() == other.to_dict() ) @classmethod def qname(cls): """package.module:class name of this class, makes str for import_name""" return f"{cls.__module__}:{cls.__name__}" def to_dict(self): """Dictionary representation of the instances contents""" return to_dict(self) def pprint(self): """Produce nice text formatting of the instance's contents""" from pprint import pp pp(self.to_dict()) @classmethod def from_dict(cls, data): """Recreate instance from the results of to_dict()""" data = data.copy() if "cls" in data: cls = import_name(data.pop("cls")) obj = object.__new__(cls) obj.__dict__.update(data) # walk data return obj def to_dict(thing): """Serialise deep structure into JSON-like hierarchy, invoking to_dict() on intake instances""" if isinstance(thing, dict): return {k: to_dict(v) for k, v in thing.items()} elif isinstance(thing, (tuple, list)): return [to_dict(v) for v in thing] elif isinstance(thing, Tokenizable): return {k: to_dict(v) for k, v in thing.__dict__.items() if not k.startswith("_")} else: return thing def make_cls(cls: str | type, kwargs: dict): """Recreate class instance from class/kwargs pair""" if isinstance(cls, str): cls = import_name(cls) return cls(**kwargs) def descend_to_path(path: str | list, kwargs: dict | list | tuple, name: str = ""): """Find the value at the location `path` in the deeply nested dict `kwargs` If a name is given, replace that value by "{name}" - used by parameter extraction. """ if isinstance(path, str): path = path.split(".") part = path.pop(0) if part.isnumeric(): part = int(part) if path: return descend_to_path(path, kwargs[part], name) out = kwargs[part] if name: kwargs[part] = "{%s}" % name return out def extract_by_path(path: str, cls: type, name: str, kwargs: dict, **kw) -> tuple: """Walk kwargs, replacing dotted keys in path by UserParameter template""" value = descend_to_path(path, kwargs, name) up = cls(default=value, **kw) return merge_dicts(kwargs, nested_keys_to_dict({path: "{%s}" % name})), up def _by_value(val, up, name): if isinstance(val, dict): return {k: _by_value(v, up, name) for k, v in val.items()} elif isinstance(val, Iterable) and not isinstance(val, (str, bytes)): return type(val)([_by_value(v, up, name) for v in val]) elif isinstance(val, str) and isinstance(up.default, str) and up.default in val: return val.replace(up.default, "{%s}" % name) else: try: if isinstance(val, (str, bytes)) and up.default in val: return val.replace(up.default, "{%s}" % name) if val == up.default: return "{%s}" % name except (TypeError, ValueError): pass return val def extract_by_value(value: Any, cls: type, name: str, kwargs: dict, **kw) -> tuple: """Walk kwargs, replacing given value with UserParameter placeholder""" up = cls(default=value, **kw) kw = _by_value(kwargs, up, name) return kw, up def replace_values(val, needle, replace): """Find `needle` in the given values and replace with `replace` Useful for removing sensitive values from kwargs and replacing with functions like "{env(...)}". """ if isinstance(val, dict): return {k: replace_values(v, needle, replace) for k, v in val.items()} elif isinstance(val, Iterable) and not isinstance(val, (str, bytes)): return type(val)([replace_values(v, needle, replace) for v in val]) try: if val == needle: return replace elif isinstance(val, (str, bytes)): return val.replace(needle, replace) except (ValueError, TypeError): pass return val def one_to_one(it: Iterable) -> dict: return {_: _ for _ in it} def all_to_one(it: Iterable, one: Any) -> dict: return {_: one for _ in it} s1 = re.compile("(.)([A-Z][a-z]+)") s2 = re.compile("([a-z0-9])([A-Z])") @cache def camel_to_snake(name: str) -> str: # https://stackoverflow.com/a/1176023/3821154 name = s1.sub(r"\1_\2", name) return s2.sub(r"\1_\2", name).lower() @cache def snake_to_camel(name: str) -> str: # https://stackoverflow.com/a/1176023/3821154 return "".join(word.title() for word in name.split("_")) def pattern_to_glob(pattern: str) -> str: """ Convert a path-as-pattern into a glob style path Uses the pattern's indicated number of '?' instead of '*' where an int was specified. Parameters ---------- pattern : str Path as pattern optionally containing format_strings Returns ------- glob_path : str Path with int format strings replaced with the proper number of '?' and '*' otherwise. Examples -------- >>> pattern_to_glob('{year}/{month}/{day}.csv') '*/*/*.csv' >>> pattern_to_glob('{year:4}/{month:2}/{day:2}.csv') '????/??/??.csv' >>> pattern_to_glob('data/{year:4}{month:02}{day:02}.csv') 'data/????????.csv' >>> pattern_to_glob('data/*.csv') 'data/*.csv' """ # https://github.com/intake/intake/issues/776#issuecomment-1917737732 by @JessicaS11 from string import Formatter fmt = Formatter() glob_path = "" for literal_text, field_name, format_specs, _ in fmt.parse(format_string=pattern): glob_path += literal_text if field_name and (glob_path != "*"): try: glob_path += "?" * int(format_specs) except ValueError: glob_path += "*" return glob_path def safe_dict(x): """Make a dict or list-like int a form you can JSON serialize""" if isinstance(x, str): return x if isinstance(x, typing.Mapping): return {k: safe_dict(v) for k, v in x.items()} if isinstance(x, typing.Iterable): return [safe_dict(v) for v in x] return str(x) def port_in_use(host, port=None): import socket if isinstance(host, tuple): host, port = host elif isinstance(host, str) and host.startswith("http"): from urllib.parse import urlparse parsed = urlparse(host) host = parsed.hostname if parsed.port is None: port = port if (port is None) or (host is None): raise ValueError(f"host={host} and port={port} is not valid.") s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((host, int(port))) s.shutdown(2) return True except: # noqa: E722 return False def find_free_port(): import socket from contextlib import closing with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(("", 0)) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) return s.getsockname()[1] def _is_tok(s: str) -> bool: """Check if a string is a valid token""" if len(s) == 16 and not s.isupper(): try: int(s, 16) return True except ValueError: return False return False ================================================ FILE: intake/source/__init__.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import logging from collections.abc import MappingView from intake.source.base import DataSource from intake.source.discovery import drivers logger = logging.getLogger("intake") class DriverRegistry(MappingView): """Dict of driver: DataSource class If the value object is a EntryPoint or str, will load it when accesses, which does the import. """ def __init__(self, drivers_source=drivers): self.drivers = drivers_source def __getitem__(self, item): it = self.drivers.enabled_plugins()[item] if hasattr(it, "load"): return it.load() if isinstance(it, str): return import_name(it) elif issubclass(it, DataSource): return it raise ValueError def __iter__(self): return iter(self.drivers.enabled_plugins()) def keys(self): return list(self) def __len__(self): return len(self.drivers.enabled_plugins()) def __repr__(self): return """""" def __contains__(self, item): return item in self.keys() registry = DriverRegistry() register_driver = drivers.register_driver unregister_driver = drivers.unregister_driver # A set of fully-qualified package.module.Class mappings classes = {} def import_name(name): import importlib if ":" in name: if name.count(":") > 1: raise ValueError("Cannot decipher name to import: %s", name) mod, rest = name.split(":") bit = importlib.import_module(mod) for part in rest.split("."): bit = getattr(bit, part) return bit else: mod, cls = name.rsplit(".", 1) module = importlib.import_module(mod) return getattr(module, cls) def get_plugin_class(name): if name in registry: return registry[name] if "." not in name: logger.debug('Plugin name "%s" not known' % name) return None if name not in classes: try: classes[name] = import_name(name) except (KeyError, NameError, ImportError): logger.debug('Failed to import "%s"' % name) return classes.get(name, None) ================================================ FILE: intake/source/base.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- """ Base classes for Data Loader interface """ from yaml import dump from ..utils import DictSerialiseMixin, pretty_describe class Schema(dict): def __getattr__(self, item): return self[item] class NoEntry(AttributeError): pass class DataSourceBase(DictSerialiseMixin): """An object which can produce data This is the base class for all Intake plugins, including catalogs and remote (server) data objects. To produce a new plugin commonly involves subclassing this definition and overriding some or all of the methods. This class is not useful in itself, most methods raise NotImplemented. """ name = None version = None container = None partition_access = False description = None dtype = None shape = None npartitions = 0 _schema = None on_server = False cat = None # the cat from which this source was made _entry = None metadata = {} def __init__(self, storage_options=None, metadata=None): # default data self.metadata = metadata or {} if isinstance(self.metadata, dict) and storage_options is None: storage_options = self._captured_init_kwargs.get("storage_options", {}) self.storage_options = storage_options def _get_schema(self): """Subclasses should return an instance of base.Schema""" raise NotImplementedError def _get_partition(self, i): """Subclasses should return a container object for this partition This function will never be called with an out-of-range value for i. """ raise NotImplementedError def __eq__(self, other): return ( type(self) is type(other) and self._captured_init_args == other._captured_init_args and self._captured_init_kwargs == other._captured_init_kwargs ) def __hash__(self): return hash((type(self), str(self._captured_init_args), str(self._captured_init_kwargs))) def _close(self): """Subclasses should close all open resources""" raise NotImplementedError def _load_metadata(self): """load metadata only if needed""" if self._schema is None: self._schema = self._get_schema() self.dtype = self._schema.dtype self.shape = self._schema.shape self.npartitions = self._schema.npartitions self.metadata.update(self._schema.extra_metadata) def _yaml(self): import inspect kwargs = self._captured_init_kwargs.copy() meta = kwargs.pop("metadata", self.metadata) or {} kwargs.update( dict( zip( inspect.signature(self.__init__).parameters, self._captured_init_args, ) ) ) data = { "sources": { self.name: { "driver": self.classname, "description": self.description or "", "metadata": meta, "args": kwargs, } } } return data def yaml(self): """Return YAML representation of this data-source The output may be roughly appropriate for inclusion in a YAML catalog. This is a best-effort implementation """ data = self._yaml() return dump(data, default_flow_style=False) def _ipython_display_(self): """Display the entry as a rich object in an IPython session.""" from IPython.display import display data = self._yaml()["sources"] contents = dump(data, default_flow_style=False) display( {"application/yaml": contents, "text/plain": pretty_describe(contents)}, metadata={"application/json": {"root": self.name}}, raw=True, ) def __repr__(self): return self.yaml() @property def is_persisted(self): """The base class does not interact with persistence""" return False @property def has_been_persisted(self): """The base class does not interact with persistence""" return False def _get_cache(self, urlpath): """The base class does not interact with caches""" return [urlpath] def discover(self): """Open resource and populate the source attributes.""" self._load_metadata() return dict( dtype=self.dtype, shape=self.shape, npartitions=self.npartitions, metadata=self.metadata, ) def read(self): """Load entire dataset into a container and return it""" if not self.partition_access or self.npartitions == 1: return self._get_partition(0) else: raise NotImplementedError def read_chunked(self): """Return iterator over container fragments of data source""" self._load_metadata() for i in range(self.npartitions): yield self._get_partition(i) def read_partition(self, i): """Return a part of the data corresponding to i-th partition. By default, assumes i should be an integer between zero and npartitions; override for more complex indexing schemes. """ self._load_metadata() if i < 0 or i >= self.npartitions: raise IndexError("%d is out of range" % i) return self._get_partition(i) def to_dask(self): """Return a dask container for this data source""" raise NotImplementedError def to_spark(self): """Provide an equivalent data object in Apache Spark The mapping of python-oriented data containers to Spark ones will be imperfect, and only a small number of drivers are expected to be able to produce Spark objects. The standard arguments may b translated, unsupported or ignored, depending on the specific driver. This method requires the package intake-spark """ raise NotImplementedError @property def entry(self): if self._entry is None: raise NoEntry("Source was not made from a catalog entry") return self._entry def configure_new(self, **kwargs): """Create a new instance of this source with altered arguments Enables the picking of options and re-evaluating templates from any user-parameters associated with this source, or overriding any of the init arguments. Returns a new data source instance. The instance will be recreated from the original entry definition in a catalog **if** this source was originally created from a catalog. """ if self._entry is not None: kw = {k: v for k, v in self._captured_init_kwargs.items() if k in self._passed_kwargs} kw.update(kwargs) obj = self._entry(**kw) obj._entry = self._entry return obj else: kw = self._captured_init_kwargs.copy() kw.update(kwargs) return type(self)(*self._captured_init_args, **kw) __call__ = get = configure_new # compatibility aliases def describe(self): """Description from the entry spec""" return self.entry.describe() def close(self): """Close open resources corresponding to this data source.""" self._close() # Boilerplate to make this object also act like a context manager def __enter__(self): self._load_metadata() return self def __exit__(self, exc_type, exc_value, traceback): self.close() class DataSource(DataSourceBase): """A Data Source will all optional functionality When subclassed, child classes will have the base data source functionality, plus caching, plotting and persistence abilities. """ pass class PatternMixin: ... ================================================ FILE: intake/source/csv.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from __future__ import annotations from intake.source.base import DataSource from intake.readers.datatypes import CSV from intake.readers.readers import DaskCSV, PandasCSV, SparkDataFrame class CSVSource(DataSource): """Read CSV files into dataframes Backward compatibility for V1 catalogs. """ name = "csv" container = "dataframe" def __init__(self, urlpath, storage_options=None, metadata=None, **kwargs): super().__init__(metadata=metadata) self.data = CSV(url=urlpath, storage_options=storage_options, metadata=metadata) self.kwargs = kwargs def discover(self): return PandasCSV(self.data).discover(**self.kwargs) def to_dask(self): return DaskCSV(self.data).read(**self.kwargs) def read(self): return PandasCSV(self.data).read(**self.kwargs) def to_spark(self): return SparkDataFrame(self.data).read(**self.kwargs) ================================================ FILE: intake/source/derived.py ================================================ from copy import deepcopy from functools import lru_cache, partial from textwrap import dedent from .. import open_catalog from ..catalog.exceptions import CatalogException from . import import_name from .base import DataSource, Schema def _kwargs_string(kwargs_dict): return ", ".join([f"{k}={v}" for k, v in kwargs_dict.items()]) class PipelineStepError(CatalogException): pass class MissingTargetError(CatalogException): def __init__(self, source, step_index, method, target): self.message = f"{source} step {step_index} {method}: {target} is not listed in the targets key of this pipeline." super().__init__(self.message) cached_cats = lru_cache(10)(open_catalog) def get_source(target, cat, kwargs, cat_kwargs): if ":" in target: caturl, target = target.rsplit(":", 1) cat = cached_cats(caturl, **cat_kwargs) if cat: return cat[target].configure_new(**kwargs) # for testing only return target # pragma: no cover class AliasSource(DataSource): """Refer to another named source, unmodified The purpose of an Alias is to be able to refer to other source(s) in the same catalog or an external catalog, perhaps leaving the choice of which target to load up to the user. This source makes no sense outside of a catalog. The "target" for an aliased data source will normally be a string. In the simple case, it is the name of a data source in the same catalog. However, we use the syntax "catalog:source" to refer to sources in other catalogs, where the part before ":" will be passed to intake.open_catalog, together with any keyword arguments from cat_kwargs. In this case, the output of the target source is not modified, but this class acts as a prototype 'derived' source for processing the output of some standard driver. After initial discovery, the source's container and other details will be updated from the target; initially, the AliasSource container is not any standard. """ container = "other" version = 2 name = "alias" def __init__(self, target, mapping=None, metadata=None, kwargs=None, cat_kwargs=None): """ Parameters ---------- target: str Name of the source to load, must be a key in the same catalog mapping: dict or None If given, use this to map the string passed as ``target`` to entries in the catalog metadata: dict or None Extra metadata to associate kwargs: passed on to the target cat_kwargs: passed on to the target catalog if target is in another catalog """ super(AliasSource, self).__init__(metadata) self.target = target self.mapping = mapping or {target: target} self.kwargs = kwargs or {} self.cat_kwargs = cat_kwargs or {} self.metadata = metadata self.source = None def _get_source(self): if self.cat is None: raise ValueError("AliasSource cannot be used outside a catalog") if self.source is None: target = self.mapping[self.target] self.source = get_source(target, self.cat, self.kwargs, self.cat_kwargs) self.metadata = self.source.metadata.copy() self.container = self.source.container self.partition_access = self.source.partition_access self.description = self.source.description def discover(self): self._get_source() return self.source.discover() def read(self): self._get_source() return self.source.read() def read_partition(self, i): self._get_source() return self.source.read_partition(i) def read_chunked(self): self._get_source() return self.source.read_chunked() def to_dask(self): self._get_source() return self.source.to_dask() def first(targets, cat, kwargs, cat_kwargs): """A target chooser that simply picks the first from the given list This is the default, particularly for the case of only one element in the list """ targ = targets[0] return get_source(targ, cat, kwargs.get(targ, {}), cat_kwargs) def first_discoverable(targets, cat, kwargs, cat_kwargs): """A target chooser: the first target for which discover() succeeds This may be useful where some drivers are not importable, or some sources can be available only sometimes. """ for t in targets: try: s = get_source(t, cat, kwargs.get(t, {}), cat_kwargs) s.discover() return s except Exception: pass raise RuntimeError("No targets succeeded at discover()") class DerivedSource(DataSource): """Base source deriving from another source in the same catalog Target picking and parameter validation are performed here, but you probably want to subclass from one of the more specific classes like ``DataFrameTransform``. """ input_container = "other" # no constraint container = "other" # to be filled in per instance at access time required_params = [] # list of kwargs that must be present optional_params = {} # optional kwargs with defaults def __init__( self, targets, target_chooser=first, target_kwargs=None, cat_kwargs=None, container=None, metadata=None, **kwargs, ): """ Parameters ---------- targets: list of string or DataSources If string(s), refer to entries of the same catalog as this Source target_chooser: function to choose between targets function(targets, cat) -> source, or a fully-qualified dotted string pointing to it target_kwargs: dict of dict with keys matching items of targets cat_kwargs: to pass to intake.open_catalog, if the target is in another catalog container: str (optional) Assumed output container, if known/different from input [Note: the exact form of target_kwargs and cat_kwargs may be subject to change] """ self.targets = targets self._chooser = target_chooser if callable(target_chooser) else import_name(target_chooser) self._kwargs = target_kwargs or {} self._source = None self._params = kwargs self._cat_kwargs = cat_kwargs or {} if container: self.container = container self._validate_params() super().__init__(metadata=metadata) def _validate_params(self): """That all required params are present and that optional types match""" assert set(self.required_params) - set(self._params) == set() for par, val in self.optional_params.items(): if par not in self._params: self._params[par] = val def _pick(self): """Pick the source from the given targets""" self._source = self._chooser(self.targets, self.cat, self._kwargs, self._cat_kwargs) if self.input_container != "other": assert self._source.container == self.input_container self.metadata["target"] = self._source.metadata if self.container is None: self.container = self._source.container class GenericTransform(DerivedSource): required_params = ["transform", "transform_kwargs"] optional_params = {"allow_dask": True} """ Perform an arbitrary function to transform an input transform: function to perform transform function(container_object) -> output, or a fully-qualified dotted string pointing to it transform_params: dict The keys are names of kwargs to pass to the transform function. Values are either concrete values to pass; or param objects which can be made into widgets (but must have a default value) - or a spec to be able to make these objects. allow_dask: bool (optional, default True) Whether to_dask() is expected to work, which will in turn call the target's to_dask() """ def _validate_params(self): super()._validate_params() transform = self._params["transform"] self._transform = transform if callable(transform) else import_name(transform) def _get_schema(self): """We do not know the schema of a generic transform""" self._pick() return Schema() def to_dask(self): self._get_schema() if not self._params["allow_dask"]: raise ValueError( "This transform is not compatible with Dask" "because it has use_dask=False" ) return self._transform(self._source.to_dask(), **self._params["transform_kwargs"]) def read(self): self._get_schema() return self._transform(self._source.read(), **self._params["transform_kwargs"]) class DataFrameTransform(GenericTransform): """Transform where the input and output are both Dask-compatible dataframes This derives from GenericTransform, and you must supply ``transform`` and any ``transform_kwargs``. """ input_container = "dataframe" container = "dataframe" optional_params = {} _df = None def to_dask(self): if self._df is None: self._pick() self._df = self._transform(self._source.to_dask(), **self._params["transform_kwargs"]) return self._df def _get_schema(self): """load metadata only if needed""" self.to_dask() return Schema( dtype=self._df.dtypes, shape=(None, len(self._df.columns)), npartitions=self._df.npartitions, metadata=self.metadata, ) def read(self): return self.to_dask().compute() class Columns(DataFrameTransform): """Simple dataframe transform to pick columns Given as an example of how to make a specific dataframe transform. Note that you could use DataFrameTransform directly, by writing a function to choose the columns instead of a method as here. """ input_container = "dataframe" container = "dataframe" required_params = ["columns"] def __init__(self, columns, **kwargs): """ columns: list of labels (usually str) or slice Columns to choose from the target dataframe """ # this class wants requires "columns", but DataFrameTransform # uses "transform_kwargs", which we don't need since we use a method for the # transform kwargs.update(transform=self.pick_columns, columns=columns, transform_kwargs={}) super().__init__(**kwargs) def pick_columns(self, df): return df[self._params["columns"]] class DataFramePipeline(DataFrameTransform): """Apply a sequence of transformations over a DataFrame The sequence of steps is defined as a list-of-dicts under the key "steps". Each step requires a "method" key and can optionally take "kwargs". loc/iloc are not supported, but query is recommended for filtering. A special method called 'cols' enables column selection and takes the kwarg 'columns', which can be a single value or a list of column names. merge, join, and assign will look for a source defined in the targets key of the pipeline. A special method called 'concat' will perform dd.concat operations where the list of dataframes to concat is provided in the kwarg 'dfs'. If a method cannot be found as an attribute of the output of the previous step, the pipeline will attempt to import the method and apply it. Here's an example of performing a groupby, selecting two columns and computing the mean. by_origin: driver: intake.source.derived.DataFramePipeline args: targets: - auto steps: - method: groupby kwargs: by: origin - method: cols kwargs: columns: - hp - mpg - method: mean """ input_container = "dataframe" container = "dataframe" required_params = ["steps"] def __init__(self, steps, **kwargs): kwargs.update(transform=self.pipeline, steps=steps, transform_kwargs={}) super().__init__(**kwargs) self._sources = {} def _get_sources(self): for target in self.targets: s = get_source(target, self.cat, self._kwargs, self._cat_kwargs).to_dask() self._sources[target] = s def pipeline(self, df): """Apply pipeline steps""" if not self._sources: self._get_sources() for idx, step in enumerate(self._params["steps"]): method = step["method"] kwargs = deepcopy(step.get("kwargs", {})) if callable(method): func = partial(method, df) elif method in ("loc", "iloc"): msg = dedent( """\ iloc and loc are not supported. To select one or more columns use - method: cols kwargs: columns: To filter use the the query method - method: query kwargs: arg: A > 2 """ ) raise ValueError(msg) # this will catch accessor methods like .dt. and .str. elif method.count(".") == 1 and hasattr(df, method.split(".")[0]): sel, f = method.split(".") mod = getattr(df, sel) func = getattr(mod, f) elif method == "cols": columns = kwargs.pop("columns") func = partial(df.__getitem__, columns) elif method == "assign": to_assign = {} for col, value in kwargs.items(): if value in self.targets: to_assign[col] = self._sources[value] else: to_assign[col] = value kwargs = to_assign func = df.assign elif method == "join": other = kwargs["other"] if not isinstance(other, list): other = [other] kwargs["other"] = [] for s in other: if s in self.targets: kwargs["other"].append(self._sources[s]) else: raise MissingTargetError(self.name, idx + 1, method, s) func = df.join elif method == "merge": right = kwargs["right"] source = self._sources.get(right) if source is None: raise MissingTargetError(self.name, idx + 1, method, right) else: kwargs["right"] = source func = df.merge elif method in ("apply", "transform"): kwargs_func = kwargs.pop("func") f = kwargs_func if callable(kwargs_func) else import_name(kwargs_func) func = partial(getattr(df, method), f) elif method == "concat": objs = kwargs["dfs"] kwargs["dfs"] = [self._sources[s] for s in objs] func = import_name("dask.dataframe.concat") else: try: func = getattr(df, method) except AttributeError: func = partial(import_name(method), df) try: df = func(**kwargs) except Exception as e: original_kwargs = step.get("kwargs", {}) s = _kwargs_string(original_kwargs) msg = dedent( f"""\ DataFramePipeline source {self.name} step {idx+1} failed {method}({s}) {repr(e)} """ ) raise PipelineStepError(msg) from e return df ================================================ FILE: intake/source/discovery.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from importlib.metadata import entry_points import logging import re import warnings from ..config import conf logger = logging.getLogger("intake") class DriverSouces: """ Handles the various ways in which drivers can be known to Intake """ def __init__(self, config=None, do_scan=None): """ Parameters ---------- config: intake.config.Config oinstance, optional If not given, will use ``intake.config.conf`` singleton do_scan: bool or None Whether to scan packages with names ``intake_*`` for valid drivers. This is for backward compatibility only. If not given, value comes from the config key "package_scan", default False. """ self.conf = config or conf self.do_scan = do_scan self._scanned = None self._entrypoints = None self._registered = None self._disabled = None @property def package_scan(self): return self.conf.get("package_scan", False) if self.do_scan is None else self.do_scan @package_scan.setter def package_scan(self, val): self.conf["package_scan"] = val def from_entrypoints(self): if self._entrypoints is None: eps = entry_points() if hasattr(eps, "select"): # Python 3.10+ / importlib_metadata >= 3.9.0 specs = eps.select(group="intake.drivers") else: specs = eps.get("intake.drivers", []) # must cache this, since lookup if fairly slow and we do this a lot self._entrypoints = list(specs) return self._entrypoints def from_conf(self): return [] def __setitem__(self, key, value): super(DriverSouces, self).__setitem__(key, value) self.save() def __delitem__(self, key): super(DriverSouces, self).__delitem__(key) self.save() @property def scanned(self): return {} def disabled(self): if self._disabled is None: self._disabled = {k for k, v in self.conf.get("drivers", {}).items() if v is False} return self._disabled def registered(self): # priority order (decreasing): runtime, config, entrypoints, package scan if self._registered is None: out = {} if self.package_scan: out.update(self.scanned) for ep in self.from_entrypoints() + self.from_conf(): out[ep.name] = ep self._registered = out return self._registered def enabled_plugins(self): return {k: v for k, v in self.registered().items() if k not in self.disabled()} def register_driver(self, name, value, clobber=False, do_enable=False): """Add runtime driver definition to list of registered drivers (drivers in global scope with corresponding ``intake.open_*`` function) Parameters ---------- name: str Name of the driver value: str, entrypoint or class Pointer to the implementation clobber: bool If True, perform the operation even if the driver exists do_enable: bool If True, unset the disabled flag for this driver """ name = _normalize(name) if name in self.registered() and not clobber: raise ValueError(f"Driver {name} already enabled") if name in self.disabled(): if do_enable: self.enable(name, value) else: logger.warning(f"Adding driver {name}, but it is disabled") self.registered()[name] = value def unregister_driver(self, name): """Remove runtime registered driver""" name = _normalize(name) self.registered().pop(name) def enable(self, name, driver=None): """ Explicitly assign a driver to a name, or remove ban Updates the associated config, which will be persisted Parameters ---------- name : string As in ``'zarr'`` driver : string Dotted object name, as in ``'intake_xarray.xzarr.ZarrSource'``. If None, simply remove driver disable flag, if it is found """ config = self.conf if "drivers" not in config: config["drivers"] = {} if driver: config["drivers"][name] = driver elif config["drivers"].get(name) is False: del config["drivers"][name] if name in self.disabled(): self.disabled().remove(name) config.save() def disable(self, name): """Disable a driver by name. Updates the associated config, which will be persisted Parameters ---------- name : string As in ``'zarr'`` """ name = _normalize(name) config = self.conf if "drivers" not in config: config["drivers"] = {} config["drivers"][name] = False config.save() self.disabled().add(name) drivers = DriverSouces() def _load_entrypoint(entrypoint): """ Call entrypoint.load() and, if it fails, raise context-specific errors. """ try: return entrypoint.load() except ImportError as err: raise ConfigurationError( f"Failed to load {entrypoint.name} driver because module " f"{entrypoint.module_name} could not be imported." ) from err except AttributeError as err: raise ConfigurationError( f"Failed to load {entrypoint.name} driver because no object " f"named {entrypoint.object_name} could be found in the module " f"{entrypoint.module_name}." ) from err def _normalize(name): if not name.isidentifier(): # primitive name normalization name = re.sub("[-=~^&|@+]", "_", name) if not name.isidentifier(): warnings.warn('Invalid Intake plugin name "%s" found.', name, stacklevel=2) return name class ConfigurationError(Exception): pass ================================================ FILE: intake/source/jsonfiles.py ================================================ import contextlib import json from itertools import islice from intake.source.base import DataSource class JSONFileSource(DataSource): """ Read JSON files as a single dictionary or list The files can be local or remote. Extra parameters for encoding, etc., go into ``storage_options``. """ name = "json" version = "0.0.1" container = "python" def __init__( self, urlpath: str, text_mode: bool = True, text_encoding: str = "utf8", compression: str = None, read: bool = True, metadata: dict = None, storage_options: dict = None, ): """ Parameters ---------- urlpath : str Target file. Can include protocol specified (e.g., "s3://"). text_mode : bool Whether to open the file in text mode, recoding binary characters on the fly text_encoding : str If text_mode is True, apply this encoding. UTF* is by far the most common compression : str or None If given, decompress the file with the given codec on load. Can be something like "zip", "gzip", "bz2", or to try to guess from the filename, 'infer' storage_options: dict Options to pass to the file reader backend, including text-specific encoding arguments, and parameters specific to the remote file-system driver, if using. """ from fsspec.utils import compressions VALID_COMPRESSIONS = list(compressions.values()) + ["infer"] self._urlpath = urlpath self._storage_options = storage_options or {} self._dataframe = None self._file = None self.compression = compression if compression is not None: if compression not in VALID_COMPRESSIONS: raise ValueError( f"Compression value {compression} must be one of {VALID_COMPRESSIONS}" ) self.mode = "rt" if text_mode else "rb" self.encoding = text_encoding self._read = read super(JSONFileSource, self).__init__(metadata=metadata) def read(self): import fsspec urlpath = self._get_cache(self._urlpath)[0] with fsspec.open( urlpath, mode=self.mode, encoding=self.encoding, compression=self.compression, **self._storage_options, ) as f: return json.load(f) def _load_metadata(self): pass def _get_schema(self): pass class JSONLinesFileSource(DataSource): """ Read a JSONL (https://jsonlines.org/) file and return a list of objects, each being valid json object (e.g. a dictionary or list) """ name = "jsonl" version = "0.0.1" container = "python" def __init__( self, urlpath: str, text_mode: bool = True, text_encoding: str = "utf8", compression: str = None, read: bool = True, metadata: dict = None, storage_options: dict = None, ): """ Parameters ---------- urlpath : str Target file. Can include protocol specified (e.g., "s3://"). text_mode : bool Whether to open the file in text mode, recoding binary characters on the fly text_encoding : str If text_mode is True, apply this encoding. UTF* is by far the most common compression : str or None If given, decompress the file with the given codec on load. Can be something like "zip", "gzip", "bz2", or to try to guess from the filename, 'infer'. storage_options: dict Options to pass to the file reader backend, including text-specific encoding arguments, and parameters specific to the remote file-system driver, if using. """ from fsspec.utils import compressions VALID_COMPRESSIONS = list(compressions.values()) + ["infer"] self._urlpath = urlpath self._storage_options = storage_options or {} self._dataframe = None self._file = None self.compression = compression if compression is not None: if compression not in VALID_COMPRESSIONS: raise ValueError( f"Compression value {compression} must be one of {VALID_COMPRESSIONS}" ) self.mode = "rt" if text_mode else "rb" self.encoding = text_encoding self._read = read super().__init__(metadata=metadata) @contextlib.contextmanager def _open(self): """ Yields an fsspec.OpenFile object """ import fsspec urlpath = self._get_cache(self._urlpath)[0] with fsspec.open( urlpath, mode=self.mode, encoding=self.encoding, compression=self.compression, **self._storage_options, ) as f: yield f def read(self): with self._open() as f: return list(map(json.loads, f)) def head(self, nrows: int = 100): """ return the first `nrows` lines from the file """ with self._open() as f: return list(map(json.loads, islice(f, nrows))) def _load_metadata(self): pass def _get_schema(self): pass ================================================ FILE: intake/source/npy.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from .base import DataSource from intake.readers.datatypes import NumpyFile from intake.readers.readers import DaskNPYStack, NumpyReader class NPySource(DataSource): """Read numpy binary files into an array Prototype source showing example of working with arrays Each file becomes one or more partitions, but partitioning within a file is only along the largest dimension, to ensure contiguous data. """ container = "ndarray" name = "numpy" version = "0.0.1" partition_access = True def __init__(self, path, storage_options=None, metadata=None): """ The parameters dtype and shape will be determined from the first file, if not given. Parameters ---------- path: str of list of str Location of data file(s), possibly including glob and protocol information storage_options: dict Passed to file-system backend. """ self.data = NumpyFile(url=path, storage_options=storage_options, metadata=metadata) self.metadata = metadata def to_dask(self): return DaskNPYStack(self.data).read() def read(self): return NumpyReader(self.data).read() ================================================ FILE: intake/source/tests/__init__.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- ================================================ FILE: intake/source/tests/alias.yaml ================================================ sources: csvs: driver: textfiles args: urlpath: '{{ CATALOG_DIR }}/*.csv' yamls: driver: textfiles args: urlpath: '{{ CATALOG_DIR }}/*.yaml' alias1: driver: intake.source.derived.AliasSource args: mapping: first: csvs second: yamls target: first alias2: driver: intake.source.derived.AliasSource args: target: csvs ================================================ FILE: intake/source/tests/cached.yaml ================================================ sources: calvert: driver: csv args: urlpath: '{{ CATALOG_DIR }}/calvert_uk.zip' cache: - type: compressed argkey: urlpath calvert_infer: driver: csv args: urlpath: '{{ CATALOG_DIR }}/calvert_uk.zip' cache: - type: compressed argkey: urlpath decomp: infer calvert_badkey: driver: csv args: urlpath: '{{ CATALOG_DIR }}/calvert_uk.zip' cache: - type: compressed argkey: urlpath decomp: unknown calvert_filter: driver: csv args: urlpath: '{{ CATALOG_DIR }}/calvert_uk_filter.tar.gz' cache: - type: compressed argkey: urlpath regex_filter: '.*calvert_uk_research2017_nodes.csv' dirs: driver: textfiles args: urlpath: '{{ CATALOG_DIR }}/main' cache: - type: dir argkey: urlpath depth: 2 dat_data: driver: textfiles args: urlpath: 'dat://66ef52101a2543e1721c901e84d2dd7a758c94283b8501d34a691abefe3fb3d6/*.json' decoder: json.loads cache: - type: dat ================================================ FILE: intake/source/tests/data.zarr/.zarray ================================================ { "chunks": [ 10 ], "compressor": { "blocksize": 0, "clevel": 5, "cname": "lz4", "id": "blosc", "shuffle": 1 }, "dtype": " out.index(b"foo") subprocess.check_output( shlex.split("intake drivers disable foo"), stderr=subprocess.STDOUT, env=env ) out = subprocess.check_output( shlex.split("intake drivers list"), stderr=subprocess.STDOUT, env=env ) assert b"foo" in out assert out.index(b"Disabled") < out.index(b"foo") def test_discover(extra_pythonpath, tmp_config_path): drivers = intake.source.discovery.DriverSouces(do_scan=True) with pytest.warns(PendingDeprecationWarning): assert "foo" in drivers.scanned registry = intake.source.DriverRegistry(drivers) # Check that package scan (name-based) discovery worked. assert "foo" in registry registry["foo"]() # Check that entrypoints-based discovery worked. assert "some_test_driver" in registry registry["some_test_driver"]() # Now again, turning off the package scan. drivers = intake.source.discovery.DriverSouces() registry = intake.source.DriverRegistry(drivers) # Check that package scan (name-based) discovery did *not* happen. assert "foo" not in registry # Check that entrypoints-based discovery worked. assert "some_test_driver" in registry registry["some_test_driver"]() def test_enable_and_disable(extra_pythonpath, tmp_config_path): # Disable and then enable a package scan result. try: drivers = intake.source.discovery.DriverSouces(do_scan=True) registry = intake.source.DriverRegistry(drivers) assert "foo" in registry drivers.disable("foo") with pytest.warns(PendingDeprecationWarning): assert "foo" in discovery.drivers.scanned assert "foo" not in registry drivers.enable("foo", "intake_foo.FooPlugin") assert "foo" in registry finally: drivers.enable("foo", "intake_foo.FooPlugin") # Disable and then enable an entrypoint result. try: drivers.disable("some_test_driver") assert "some_test_driver" not in registry drivers.enable("some_test_driver", "driver_with_entrypoints.SomeTestDriver") assert "some_test_driver" in registry finally: drivers.enable("some_test_driver", "driver_with_entrypoints.SomeTestDriver") def test_register_and_unregister(extra_pythonpath, tmp_config_path): registry = intake.source.registry assert "bar" not in registry with pytest.raises(ImportError): from intake import open_bar intake.register_driver("bar", "intake_foo.FooPlugin") assert "bar" in registry from intake import open_bar # noqa intake.unregister_driver("bar") assert "bar" not in registry with pytest.raises(ImportError): from intake import open_bar # noqa def test_discover_collision(extra_pythonpath, tmp_config_path): with pytest.warns(UserWarning): discovery._package_scan(plugin_prefix="collision_") ================================================ FILE: intake/source/tests/test_json.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2021, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import json import os import pytest from fsspec import open_files from fsspec.utils import compressions from intake.source.jsonfiles import JSONFileSource, JSONLinesFileSource here = os.path.abspath(os.path.dirname(__file__)) EXTENSIONS = {compression: f".{extension}" for extension, compression in compressions.items()} @pytest.fixture(params=[None, "gzip", "bz2"]) def json_file(request, tmp_path) -> str: data = {"hello": "world"} file_path = str(tmp_path / "1.json") file_path += EXTENSIONS.get(request.param, "") with open_files([file_path], mode="wt", compression=request.param)[0] as f: f.write(json.dumps(data)) return file_path @pytest.fixture(params=[None, "gzip", "bz2"]) def jsonl_file(request, tmp_path) -> str: data = [{"hello": "world"}, [1, 2, 3]] file_path = str(tmp_path / "1.jsonl") file_path += EXTENSIONS.get(request.param, "") with open_files([file_path], mode="wt", compression=request.param)[0] as f: f.write("\n".join(json.dumps(row) for row in data)) return file_path def test_jsonfile(json_file: str): j = JSONFileSource(json_file, text_mode=True, compression="infer") out = j.read() assert isinstance(out, dict) assert out["hello"] == "world" def test_jsonfile_none(json_file: str): try: j = JSONFileSource(json_file, text_mode=True, compression=None) out = j.read() # compression=None should not raise exception for uncompressed files assert json_file.endswith(".json") except UnicodeDecodeError: # compression=None should raise exception for compressed files assert not json_file.endswith(".json") return assert isinstance(out, dict) assert out["hello"] == "world" def test_jsonfile_discover(json_file: str): j = JSONFileSource(json_file, text_mode=True, compression=None) schema = j.discover() assert schema == {"dtype": None, "shape": None, "npartitions": 0, "metadata": {}} def test_jsonlfile(jsonl_file: str): j = JSONLinesFileSource(jsonl_file, compression="infer") out = j.read() assert isinstance(out, list) assert isinstance(out[0], dict) assert out[0]["hello"] == "world" assert isinstance(out[1], list) assert out[1] == [1, 2, 3] def test_jsonfilel_none(jsonl_file: str): try: j = JSONLinesFileSource(jsonl_file, compression=None) out = j.read() # compression=None should not raise exception for uncompressed files assert jsonl_file.endswith(".jsonl") except UnicodeDecodeError: # compression=None should raise exception for compressed files assert not jsonl_file.endswith(".jsonl") return assert isinstance(out, list) assert isinstance(out[0], dict) assert out[0]["hello"] == "world" assert isinstance(out[1], list) assert out[1] == [1, 2, 3] def test_jsonfilel_discover(json_file: str): j = JSONLinesFileSource(jsonl_file, compression=None) schema = j.discover() assert schema == {"dtype": None, "shape": None, "npartitions": 0, "metadata": {}} def test_jsonl_head(jsonl_file: str): j = JSONLinesFileSource(jsonl_file, compression="infer") out = j.head(1) assert isinstance(out, list) assert len(out) == 1 assert out[0]["hello"] == "world" ================================================ FILE: intake/source/tests/test_npy.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import os import posixpath import numpy as np import pytest import intake from ..npy import NPySource here = os.path.abspath(os.path.dirname(__file__)) @pytest.mark.parametrize("shape", [(1,), (1, 1), (10,), (5, 2), (3, 3, 3)]) def test_one_file(tempdir, shape): size = 1 for s in shape: size *= s data = np.random.randint(1, 100, size=size).reshape(shape) fn = os.path.join(tempdir, "out.npy") np.save(fn, data) s = NPySource(fn) out = s.read() assert (out == data).all() s = NPySource(fn, chunks=1) out = s.read() assert (out == data).all() s = NPySource(fn, shape=shape, dtype="int", chunks=1) out = s.read() assert (out == data).all() @pytest.mark.parametrize("shape", [(1,), (1, 1), (10,), (5, 2), (3, 3, 3)]) def test_multi_file(tempdir, shape): size = 1 for s in shape: size *= s data0 = np.random.randint(1, 100, size=size).reshape(shape) fn0 = os.path.join(tempdir, "out0.npy") np.save(fn0, data0) data1 = np.random.randint(1, 100, size=size).reshape(shape) fn1 = os.path.join(tempdir, "out1.npy") np.save(fn1, data1) data = np.stack([data0, data1]) fn = [fn0, fn1] s = NPySource(fn) out = s.read() assert (out == data).all() assert (s.to_dask().compute() == data).all() s = NPySource(fn, chunks=1) out = s.read() assert (out == data).all() assert (s.to_dask().compute() == data).all() s = NPySource(fn, shape=shape, dtype="int", chunks=1) out = s.read() assert (out == data).all() da = s.to_dask() assert (da.compute() == data).all() schema = s.discover() assert schema["shape"] == data.shape assert schema["npartitions"] == da.npartitions s = NPySource(os.path.join(tempdir, "out*.npy")) out = s.read() assert (out == data).all() assert (s.to_dask().compute() == data).all() def test_zarr_minimal(): pytest.importorskip("zarr") cat = intake.open_catalog(posixpath.join(here, "sources.yaml")) s = cat.zarr1() assert s.container == "ndarray" assert s.read().tolist() == [73, 98, 46, 38, 20, 12, 31, 8, 89, 72] assert s.npartitions == 1 assert s.dtype.kind == "i" assert s.shape == (10,) assert (s.read_partition((0,)) == s.read()).all() def test_zarr_parts(): zarr = pytest.importorskip("zarr") out = {} g = zarr.open(out, mode="w") z = g.create_dataset("data", dtype="i4", shape=(10, 10), chunks=(5, 5), compression=None) z[:5, :5] = 1 z[:5, 5:] = 2 z[5:, :5] = 3 z[5:, 5:] = 4 source = intake.open_ndzarr(out, component="data") assert (source.read_partition((0, 0)) == 1).all() assert (source.read_partition((1, 1)) == 4).all() da = source.to_dask() assert da.npartitions == 4 assert (da.compute() == z[:]).all() g = zarr.open(out, mode="w") gg = g.create_group("inner") z = gg.create_dataset("data", dtype="i4", shape=(10, 10), chunks=(5, 5), compression=None) z[:5, :5] = 1 z[:5, 5:] = 2 z[5:, :5] = 3 z[5:, 5:] = 4 source = intake.open_ndzarr(out, component="inner/data") assert (source.read_partition((0, 0)) == 1).all() assert (source.read_partition((1, 1)) == 4).all() da = source.to_dask() assert da.npartitions == 4 assert (da.compute() == z[:]).all() ================================================ FILE: intake/source/tests/test_text.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import os import pytest from fsspec import open_files import intake from intake.source import import_name from intake.source.textfiles import TextFilesSource here = os.path.abspath(os.path.dirname(__file__)) def test_textfiles(tempdir): open(os.path.join(tempdir, "1.txt"), "wt").write("hello\nworld") open(os.path.join(tempdir, "2.txt"), "wt").write("hello\nworld") path = os.path.join(tempdir, "*.txt") t = TextFilesSource(path) t.discover() assert t.npartitions == 2 assert t._get_partition(0) == t.to_dask().to_delayed()[0].compute() out = t.read() assert isinstance(out, list) assert out[0] == "hello\n" @pytest.mark.parametrize("comp", [None, "gzip", "bz2"]) def test_complex_text(tempdir, comp): dump, load, read = "json.dumps", "json.loads", True dump = import_name(dump) data = [{"something": "simple", "and": 0}] * 2 for f in ["1.out", "2.out"]: fn = os.path.join(tempdir, f) with open_files([fn], mode="wt", compression=comp)[0] as fo: if read: fo.write(dump(data)) else: dump(data, fo) # that was all setup path = os.path.join(tempdir, "*.out") t = TextFilesSource(path, text_mode=True, compression=comp, decoder=load) t.discover() assert t.npartitions == 2 assert t._get_partition(0) == t.to_dask().to_delayed()[0].compute() out = t.read() assert isinstance(out, list) assert out[0] == data[0] @pytest.mark.parametrize("comp", [None, "gzip", "bz2"]) @pytest.mark.parametrize( "pars", [ ["msgpack.pack", "msgpack.unpack", False], ["msgpack.packb", "msgpack.unpackb", True], ["pickle.dump", "pickle.load", False], ["pickle.dumps", "pickle.loads", True], ], ) def test_complex_bytes(tempdir, comp, pars): dump, load, read = pars dump = import_name(dump) # using bytestrings means not needing extra en/decode argument to msgpack data = [{b"something": b"simple", b"and": 0}] * 2 for f in ["1.out", "2.out"]: fn = os.path.join(tempdir, f) with open_files([fn], mode="wb", compression=comp)[0] as fo: if read: fo.write(dump(data)) else: dump(data, fo) # that was all setup path = os.path.join(tempdir, "*.out") t = TextFilesSource(path, text_mode=False, compression=comp, decoder=load, read=read) t.discover() assert t.npartitions == 2 assert t._get_partition(0) == t.to_dask().to_delayed()[0].compute() out = t.read() assert isinstance(out, list) assert out[0] == data[0] def test_text_persist(temp_cache): cat = intake.open_catalog(os.path.join(here, "sources.yaml")) s = cat.sometext() s2 = s.persist() assert s.read() == s2.read() def test_text_export(temp_cache): import tempfile outdir = tempfile.mkdtemp() cat = intake.open_catalog(os.path.join(here, "sources.yaml")) s = cat.sometext() out = s.export(outdir) fn = os.path.join(outdir, "cat.yaml") with open(fn, "w") as f: f.write(out.yaml()) cat = intake.open_catalog(fn) s2 = cat[s.name]() assert s.read() == s2.read() ================================================ FILE: intake/source/tests/test_tiled.py ================================================ import shlex import subprocess import time import pytest import intake pytest.importorskip("tiled") import httpx # required by tiled, so will be here @pytest.fixture() def server(): cmd = shlex.split("tiled serve pyobject --public tiled.examples.generated:tree") P = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) url = "http://localhost:8000" timeout = 20 while True: try: r = httpx.get(url) if r.status_code == 200: break except: pass timeout -= 0.1 if timeout < 0: P.terminate() out = P.communicate() raise RuntimeError("timeout waiting for Tiled server\n%s", out) time.sleep(0.1) yield url + "/api" P.terminate() P.wait() def test_simple(server): cat = intake.open_tiled_cat(server) out = cat.tiny_image.read() assert out.shape assert out.all() ================================================ FILE: intake/source/tests/test_utils.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- ================================================ FILE: intake/source/tests/util.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- def verify_plugin_interface(plugin): assert isinstance(plugin.version, str) assert isinstance(plugin.container, str) assert isinstance(plugin.partition_access, bool) def verify_datasource_interface(source): for attr in [ "container", "description", "dtype", "shape", "npartitions", "metadata", ]: assert hasattr(source, attr) for method in [ "discover", "read", "read_chunked", "read_partition", "to_dask", "close", ]: assert hasattr(source, method) def zscore(s): return (s - s.mean()) / s.std(ddof=0) def reverse(s): return s[::-1] ================================================ FILE: intake/source/textfiles.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from . import base from intake.readers.datatypes import Text from intake.readers.readers import SparkText, FileByteReader class TextFilesSource(base.DataSource): """Read textfiles as sequence of lines Prototype of sources reading sequential data. Takes a set of files, and returns an iterator over the text in each of them. The files can be local or remote. Extra parameters for encoding, etc., go into ``storage_options``. """ name = "textfiles" version = "0.0.1" container = "python" partition_access = True def __init__( self, urlpath, text_mode=True, text_encoding="utf8", compression=None, decoder=None, metadata=None, storage_options=None, ): """ Parameters ---------- urlpath : str or list(str) Target files. Can be a glob-path (with "*") and include protocol specified (e.g., "s3://"). Can also be a list of absolute paths. text_mode : bool Whether to open the file in text mode, recoding binary characters on the fly text_encoding : str If text_mode is True, apply this encoding. UTF* is by far the most common compression : str or None If given, decompress the file with the given codec on load. Can be something like "gzip", "bz2", or to try to guess from the filename, 'infer' decoder : function, str or None Use this to decode the contents of files. If None, you will get a list of lines of text/bytes. If a function, it must operate on an open file-like object or a bytes/str instance, and return a list storage_options: dict Options to pass to the file reader backend, including text-specific encoding arguments, and parameters specific to the remote file-system driver, if using. """ if compression: storage_options["compression"] = compression self.data = Text(url=urlpath, storage_options=storage_options, metadata=metadata) self.metadata = metadata self.kwargs = dict(text_mode=text_mode, text_encoding=text_encoding, decoder=decoder) def read(self): reader = FileByteReader(self.data) if self.kwargs["text_mode"]: if self.kwargs["decoder"]: reader = reader.apply(self.kwargs["decoder"]) else: reader = reader.apply(bytes.decode, encoding=self.kwargs["text_encoding"]) return reader.read() def to_spark(self): return SparkText(self.data).read() ================================================ FILE: intake/source/tiled.py ================================================ from intake.catalog import Catalog from intake.source import DataSource class TiledCatalog(Catalog): """View Tiled server as a catalog See the documentation for setting up such a server at https://blueskyproject.io/tiled/ A tiled server may contain sources of dataframe, array or xarray type. This driver exposes the full tree as exposed by the server, but you can also specify the sub-path of that tree. """ name = "tiled_cat" def __init__(self, server, path=None): """ Parameters ---------- server: str or tiled.client.node.Node Location of tiles server. Usually of the form "http[s]://address:port/" May include a path. If the protocol is "tiled", we assume HTTP connection. Alternatively, can be a Node instance, already connected to a server. path: str (optional) If given, restrict the catalog to this part of the server's catalog tree. Equivalent to extending the server URL. """ from tiled.client import from_uri self.path = path if isinstance(server, str): if server.startswith("tiled"): uri = server.replace("tiled", "http", 1) else: uri = server client = from_uri(uri, "dask") else: client = server uri = server.uri self.uri = uri if path is not None: client = client[path] super().__init__(entries=client, name="tiled:" + uri.split(":", 1)[1]) def search(self, query, type="text"): """Full text search Queries other than full text will be added later """ if type == "text": from tiled.queries import FullText q = FullText(query) else: raise NotImplementedError return TiledCatalog.from_dict(self._entries.search(q), uri=self.uri, path=self.path) def __getitem__(self, item): from tiled.client.node import Node node = self._entries[item] if isinstance(node, Node): return TiledCatalog(node) else: return TiledSource(uri=self.uri, path=item, instance=node) types = { "DaskArrayClient": "ndarray", "DaskDataArrayClient": "xarray", "DaskDatasetClient": "xarray", "DaskVariableClient": "xarray", "DaskDataFrameClient": "dataframe", } class TiledSource(DataSource): """A source on a Tiled server The container type of this source is determined at runtime. The attribute ``.instance`` gives access to the underlying Tiled API, but most users will only call ``.to_dask()``. """ name = "tiled" def __init__(self, uri="", path="", instance=None, metadata=None): """ Parameters ---------- uri: str (optional) Location of the server. If ``instance`` is given, this is only used for the repr pathL str (optional) Path of the data source within the server tree. If ``instance`` is given, this is only used for the repr instance: tiled.client.node.None (optional) The tiled object pointing to the data source; normally created by a ``TiledCatalog`` metadata: dict Extra metadata for this source; metadata will also be provided by the server. """ from tiled.client import from_uri if instance is None: instance = from_uri(uri, "dask")[path].read() self.instance = instance md = dict(instance.metadata) if metadata: md.update(metadata) super().__init__(metadata=md) self.name = path self.container = types[type(self.instance).__name__] def discover(self): x = self.to_dask() dt = getattr(x, "dtype", None) or getattr(x, "dtypes", None) parts = getattr(x, "npartitions", None) or x.data.npartitions return dict( dtype=dt, shape=getattr(self.instance.structure().macro, "shape", x.shape), npartitions=parts, metadata=self.metadata, ) def to_dask(self): # cache this? return self.instance.read() def read(self): return self.instance.read().compute() def _yaml(self): y = super()._yaml() v = list(y["sources"].values())[0] v["args"].pop("instance") return y ================================================ FILE: intake/source/utils.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from hashlib import md5 def tokenize(*args, **kwargs): """Deterministic token copied from dask """ hasher = md5(str(tuple(args)).encode()) if kwargs: hasher.update(str(args).encode()) return hasher.hexdigest() def _validate_format_spec(format_spec): if format_spec[-1].isalpha(): format_spec = format_spec[:-1] if not format_spec.isdigit(): raise ValueError("Format specifier must have a set width") return int(format_spec) def _get_parts_of_format_string(resolved_string, literal_texts, format_specs): """ Inner function of reverse_format, returns the resolved value for each field in pattern. """ _text = resolved_string bits = [] if literal_texts[-1] != "" and _text.endswith(literal_texts[-1]): _text = _text[: -len(literal_texts[-1])] literal_texts = literal_texts[:-1] format_specs = format_specs[:-1] for i, literal_text in enumerate(literal_texts): if literal_text != "": if literal_text not in _text: raise ValueError( ("Resolved string must match pattern. " "'{}' not found.".format(literal_text)) ) bit, _text = _text.split(literal_text, 1) if bit: bits.append(bit) elif i == 0: continue else: try: format_spec = _validate_format_spec(format_specs[i - 1]) bits.append(_text[0:format_spec]) _text = _text[format_spec:] except Exception: if i == len(format_specs) - 1: format_spec = _validate_format_spec(format_specs[i]) bits.append(_text[:-format_spec]) bits.append(_text[-format_spec:]) _text = [] else: _validate_format_spec(format_specs[i - 1]) if _text: bits.append(_text) if len(bits) > len([fs for fs in format_specs if fs is not None]): bits = bits[1:] return bits def reverse_format(format_string, resolved_string): """ Reverse the string method format. Given format_string and resolved_string, find arguments that would give ``format_string.format(**arguments) == resolved_string`` Parameters ---------- format_string : str Format template string as used with str.format method resolved_string : str String with same pattern as format_string but with fields filled out. Returns ------- args : dict Dict of the form {field_name: value} such that ``format_string.(**args) == resolved_string`` Examples -------- >>> reverse_format('data_{year}_{month}_{day}.csv', 'data_2014_01_03.csv') {'year': '2014', 'month': '01', 'day': '03'} >>> reverse_format('data_{year:d}_{month:d}_{day:d}.csv', 'data_2014_01_03.csv') {'year': 2014, 'month': 1, 'day': 3} >>> reverse_format('data_{date:%Y_%m_%d}.csv', 'data_2016_10_01.csv') {'date': datetime.datetime(2016, 10, 1, 0, 0)} >>> reverse_format('{state:2}{zip:5}', 'PA19104') {'state': 'PA', 'zip': '19104'} See also -------- str.format : method that this reverses reverse_formats : method for reversing a list of strings using one pattern """ from datetime import datetime from string import Formatter from fsspec.implementations.local import make_path_posix fmt = Formatter() args = {} # ensure that format_string is in posix format format_string = make_path_posix(format_string) # split the string into bits literal_texts, field_names, format_specs, conversions = zip(*fmt.parse(format_string)) if not any(field_names): return {} for i, conversion in enumerate(conversions): if conversion: raise ValueError(("Conversion not allowed. Found on {}.".format(field_names[i]))) # ensure that resolved string is in posix format resolved_string = make_path_posix(resolved_string) # get a list of the parts that matter bits = _get_parts_of_format_string(resolved_string, literal_texts, format_specs) for i, (field_name, format_spec) in enumerate(zip(field_names, format_specs)): if field_name: try: if format_spec.startswith("%"): args[field_name] = datetime.strptime(bits[i], format_spec) elif format_spec[-1] in list("bcdoxX"): args[field_name] = int(bits[i]) elif format_spec[-1] in list("eEfFgGn"): args[field_name] = float(bits[i]) elif format_spec[-1] == "%": args[field_name] = float(bits[i][:-1]) / 100 else: args[field_name] = fmt.format_field(bits[i], format_spec) except Exception: args[field_name] = bits[i] return args def reverse_formats(format_string, resolved_strings): """ Reverse the string method format for a list of strings. Given format_string and resolved_strings, for each resolved string find arguments that would give ``format_string.format(**arguments) == resolved_string``. Each item in the output corresponds to a new column with the key setting the name and the values representing a mapping from list of resolved_strings to the related value. Parameters ---------- format_string : str Format template string as used with str.format method resolved_strings : list List of strings with same pattern as format_string but with fields filled out. Returns ------- args : dict Dict of the form ``{field: [value_0, ..., value_n], ...}`` where values are in the same order as resolved_strings, so: ``format_sting.format(**{f: v[0] for f, v in args.items()}) == resolved_strings[0]`` Examples -------- >>> paths = ['data_2014_01_03.csv', 'data_2014_02_03.csv', 'data_2015_12_03.csv'] >>> reverse_formats('data_{year}_{month}_{day}.csv', paths) {'year': ['2014', '2014', '2015'], 'month': ['01', '02', '12'], 'day': ['03', '03', '03']} >>> reverse_formats('data_{year:d}_{month:d}_{day:d}.csv', paths) {'year': [2014, 2014, 2015], 'month': [1, 2, 12], 'day': [3, 3, 3]} >>> reverse_formats('data_{date:%Y_%m_%d}.csv', paths) {'date': [datetime.datetime(2014, 1, 3, 0, 0), datetime.datetime(2014, 2, 3, 0, 0), datetime.datetime(2015, 12, 3, 0, 0)]} >>> reverse_formats('{state:2}{zip:5}', ['PA19104', 'PA19143', 'MA02534']) {'state': ['PA', 'PA', 'MA'], 'zip': ['19104', '19143', '02534']} See also -------- str.format : method that this reverses reverse_format : method for reversing just one string using a pattern """ from string import Formatter fmt = Formatter() # get the fields from the format_string field_names = [i[1] for i in fmt.parse(format_string) if i[1]] # itialize the args dict with an empty dict for each field args = {field_name: [] for field_name in field_names} for resolved_string in resolved_strings: for field, value in reverse_format(format_string, resolved_string).items(): args[field].append(value) return args ================================================ FILE: intake/source/zarr.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- from .base import DataSource from intake.readers.datatypes import Zarr from intake.readers.readers import DaskZarr, NumpyZarr class ZarrArraySource(DataSource): """Read Zarr format files into an array Zarr is an numerical array storage format which works particularly well with remote and parallel access. For specifics of the format, see https://zarr.readthedocs.io/en/stable/ """ container = "ndarray" name = "ndzarr" version = "0.0.1" partition_access = True def __init__(self, urlpath, storage_options=None, component=None, metadata=None): """ Parameters ---------- urlpath : str Location of data file(s), possibly including protocol information storage_options : dict Passed on to storage backend for remote files component : str or None If None, assume the URL points to an array. If given, assume the URL points to a group, and descend the group to find the array at this location in the hierarchy; components are separated by the "/" character. """ self.data = Zarr( url=urlpath, storage_options=storage_options, root=component, metadata=metadata, ) self.metadata = metadata def to_dask(self): return DaskZarr(self.data).read() def read(self): return NumpyZarr(self.data).read() ================================================ FILE: intake/tests/__init__.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- ================================================ FILE: intake/tests/catalog1.yml ================================================ sources: ex1: description: this source doesn't work driver: csv args: {} ex2: description: this source doesn't work metadata: foo: 'bar' bar: [1, 2, 3] driver: csv args: {} ================================================ FILE: intake/tests/catalog2.yml ================================================ sources: ex3: description: this source doesn't work driver: csv args: {} ex4: description: this source doesn't work metadata: foo: 'bar' bar: [1, 2, 3] driver: csv args: {} ================================================ FILE: intake/tests/catalog_inherit_params.yml ================================================ --- metadata: version: 1 parameters: bucket: type: str description: description default: test_bucket sources: param: driver: parquet description: description args: urlpath: s3://{{bucket}}/file.parquet local_param_overwrites: driver: parquet description: description parameters: bucket: type: str description: description default: local_param args: urlpath: s3://{{bucket}}/file.parquet local_and_global_params: driver: parquet description: description parameters: filename: type: str description: description default: local_filename.parquet args: urlpath: s3://{{bucket}}/{{filename}} subcat: driver: yaml_file_cat args: path: "{{CATALOG_DIR}}/catalog_nested_sub.yml" user_parameters: inner: type: str description: description default: test_name ================================================ FILE: intake/tests/catalog_nested.yml ================================================ sources: nested: description: References catalog_nested_sub.yml driver: yaml_file_cat args: path: "{{ CATALOG_DIR }}/__unit_test_catalog_nested_sub.yml" ================================================ FILE: intake/tests/catalog_nested_sub.yml ================================================ sources: ex1: description: this is a sub-resource driver: csv args: urlpath: "" ex2: description: this is a sub-resource driver: csv args: urlpath: "{{bucket}}/{{inner}}" ================================================ FILE: intake/tests/test_config.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import os import pytest from intake import config from intake.config import Config, defaults from intake.util_tests import temp_conf @pytest.mark.parametrize("conf", [{}, {"port": 5000}, {"other": True}]) def test_load_conf(conf): # This test will only work if your config is set to default inconf = defaults.copy() expected = inconf.copy() with temp_conf(conf) as fn: config = Config(fn) config.load(fn) expected.update(conf) assert dict(config) == expected config.reset() assert dict(config) == inconf @pytest.mark.skipif(os.name == "nt", reason="Paths are different on win") def test_pathdirs(): assert config.intake_path_dirs([]) == [] assert config.intake_path_dirs(["paths"]) == ["paths"] assert config.intake_path_dirs("") == [""] assert config.intake_path_dirs("path1:path2") == ["path1", "path2"] assert config.intake_path_dirs("memory://path1:memory://path2") == [ "memory://path1", "memory://path2", ] @pytest.mark.parametrize( "conf", [ {"environment_conf_parse": "error"}, {"environment_conf_parse": "warn"}, {"environment_conf_parse": "ignore"}, ], ) def test_load_env(conf): # test the parsing of environment variables as strings os.environ["INTAKE_CACHE"] = "./tmp" # this causes a SyntaxError os.environ["INTAKE_CACHE2"] = "tmp" # this causes a ValueError with temp_conf(conf) as fn: if conf["environment_conf_parse"] == "error": # When raise_on_error is True, ensure the exception is raised with pytest.raises(ValueError, SyntaxError): Config(fn) elif conf["environment_conf_parse"] == "warn": # When raise_on_error is False, ensure the variable is parsed as a string with pytest.warns(UserWarning, match="environment variable"): config = Config(fn) assert config["cache"] == "./tmp" assert config["cache2"] == "tmp" else: with pytest.warns(None) as warnings: config = Config(fn) assert not warnings assert config["cache"] == "./tmp" assert config["cache2"] == "tmp" ================================================ FILE: intake/tests/test_top_level.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import os import posixpath import sys import tempfile import time import platformdirs import pytest import intake import intake.catalog.local from .test_utils import copy_test_file @pytest.fixture def user_catalog(): target_catalog = copy_test_file( "catalog1.yml", platformdirs.user_data_dir(appname="intake", appauthor="intake") ) yield target_catalog # Remove the file, but not the directory (because there might be other # files already there) os.remove(target_catalog) @pytest.fixture def tmp_path_catalog(): tmp_path = posixpath.join(tempfile.gettempdir(), "intake") try: os.makedirs(tmp_path) except: pass target_catalog = copy_test_file("catalog1.yml", tmp_path) yield target_catalog # Remove the file, but not the directory (because there might be other # files already there) os.remove(target_catalog) def test_autoregister_open(): assert hasattr(intake, "open_csv") def test_default_catalogs(): # No assumptions about contents of these catalogs. # Just make sure they exist and don't raise exceptions list(intake.cat) def test_user_catalog(user_catalog): cat = intake.load_combo_catalog() assert set(cat) >= set(["ex1", "ex2"]) def test_open_styles(tmp_path_catalog): cat = intake.catalog.local.YAMLFileCatalog(tmp_path_catalog) cat2 = intake.open_catalog(tmp_path_catalog) assert list(cat) == list(cat2) cat2 = intake.open_catalog([tmp_path_catalog]) assert list(cat) == list(cat2) cat2 = intake.open_catalog(os.path.join(os.path.dirname(tmp_path_catalog), "*")) assert list(cat) == list(cat2) assert type(cat2).name == "yaml_files_cat" cat2 = intake.open_catalog(os.path.dirname(tmp_path_catalog)) assert list(cat) == list(cat2) assert type(cat2).name == "yaml_files_cat" cat2 = intake.open_yaml_file_cat(tmp_path_catalog) assert list(cat) == list(cat2) cat2 = intake.open_yaml_files_cat([tmp_path_catalog]) assert list(cat) == list(cat2) cat2 = intake.open_yaml_files_cat(os.path.join(os.path.dirname(tmp_path_catalog), "*")) assert list(cat) == list(cat2) def test_path_catalog(tmp_path_catalog): intake.config.conf["catalog_path"] = [posixpath.join(tempfile.gettempdir(), "intake")] cat = intake.load_combo_catalog() time.sleep(2) # wait 2 seconds for catalog to refresh assert set(cat) >= set(["ex1", "ex2"]) del intake.config.conf["catalog_path"] def test_bad_open(): with pytest.raises(ValueError): # unknown driver intake.open_catalog("", driver="unknown") with pytest.raises(ValueError): # bad URI type (NB falsish values become empty catalogs) intake.open_catalog(True) # default empty catalog assert intake.open_catalog() == intake.open_catalog(None) def test_bad_open_helptext(): with pytest.raises(ValueError) as val_err: # unknown driver intake.open_catalog("", driver="unknown") assert "plugin directory" in str(val_err.value).lower() with pytest.raises(AttributeError) as attr_err: intake.open_not_a_real_plugin() assert "plugin directory" in str(attr_err.value).lower() def test_output_notebook(): pytest.importorskip("hvplot") intake.output_notebook() def test_old_usage(): assert isinstance(intake.Catalog(), intake.Catalog) assert intake.Catalog is intake.catalog.base.Catalog def test_no_imports(): mods = [mod for mod in sys.modules if mod.startswith("intake")] [sys.modules.pop(mod) for mod in mods] import intake # noqa assert "intake" in sys.modules for mod in [ "intake.tests", "intake.interface", "intake.source.csv", "intake.cli", "intake.auth", ]: assert mod not in sys.modules @pytest.fixture def tmp_path_catalog_nested(): with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = posixpath.join(tmp_dir, "intake") target_catalog = copy_test_file("catalog_nested.yml", tmp_path) copy_test_file("catalog_nested_sub.yml", tmp_path) yield target_catalog def test_nested_catalog_access(tmp_path_catalog_nested): cat = intake.open_catalog(tmp_path_catalog_nested) entry1 = cat.nested.ex1 entry2 = cat["nested.ex1"] entry3 = cat[["nested", "ex1"]] entry4 = cat["nested", "ex1"] assert entry1 == entry2 == entry3 == entry4 ================================================ FILE: intake/tests/test_utils.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import os import posixpath import shutil import pytest import yaml from intake.utils import make_path_posix, no_duplicate_yaml def test_windows_file_path(): path = "C:\\Users\\user\\fake.file" actual = make_path_posix(path) expected = "C:/Users/user/fake.file" assert actual == expected def test_make_path_posix_removes_double_sep(): path = "user//fake.file" actual = make_path_posix(path) expected = "user/fake.file" assert actual == expected @pytest.mark.parametrize( "path", [ "~/fake.file", "https://example.com", ], ) def test_noops(path): """For non windows style paths, make_path_posix should be a noop""" assert make_path_posix(path) == path def test_roundtrip_file_path(): path = os.path.dirname(__file__) actual = make_path_posix(path) assert "\\" not in actual assert os.path.samefile(actual, path) def test_yaml_tuples(): data = (1, 2) text = yaml.dump(data) with no_duplicate_yaml(): assert yaml.safe_load(text) == data def copy_test_file(filename, target_dir): if not os.path.exists(target_dir): os.makedirs(target_dir) # can't use exist_ok in Python 2.7 target_dir = make_path_posix(target_dir) # Put a catalog file in the user catalog directory test_dir = make_path_posix(os.path.dirname(__file__)) test_catalog = posixpath.join(test_dir, filename) target_catalog = posixpath.join(target_dir, "__unit_test_" + filename) shutil.copyfile(test_catalog, target_catalog) return target_catalog ================================================ FILE: intake/util_tests.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import os import shutil import tempfile from contextlib import contextmanager import yaml @contextmanager def tempdir(): d = tempfile.mkdtemp() try: yield d finally: if os.path.exists(d): shutil.rmtree(d) @contextmanager def temp_conf(conf): with tempdir() as d: fn = os.path.join(d, "conf.yaml") with open(fn, "w") as f: yaml.dump(conf, f) yield fn ================================================ FILE: intake/utils.py ================================================ # ----------------------------------------------------------------------------- # Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors # All rights reserved. # # The full license is in the LICENSE file, distributed with this software. # ----------------------------------------------------------------------------- import collections import collections.abc import datetime import importlib import logging import os import sys import warnings from collections import OrderedDict from contextlib import contextmanager import yaml logger = logging.getLogger("intake") def import_name(name): modname = name.split(":", 1)[0] logger.debug("Importing: '%s'" % modname) mod = importlib.import_module(modname) if ":" in name: end = name.split(":")[1] for bit in end.split("."): mod = getattr(mod, bit) return mod def make_path_posix(path): """Make path generic""" # TODO: migrate to fsspec' implementation if "://" in path: return path return path.replace("\\", "/").replace("//", "/") def no_duplicates_constructor(loader, node, deep=False): """Check for duplicate keys while loading YAML https://gist.github.com/pypt/94d747fe5180851196eb """ mapping = {} for key_node, value_node in node.value: key = loader.construct_object(key_node, deep=deep) value = loader.construct_object(value_node, deep=deep) if key in mapping: from intake.catalog.exceptions import DuplicateKeyError raise DuplicateKeyError( "while constructing a mapping", node.start_mark, "found duplicate key (%s)" % key, key_node.start_mark, ) mapping[key] = value return loader.construct_mapping(node, deep) def tuple_constructor(loader, node, deep=False): return tuple(loader.construct_object(node, deep=deep) for node in node.value) def represent_dictionary_order(self, dict_data): return self.represent_mapping("tag:yaml.org,2002:map", dict_data.items()) yaml.add_representer(OrderedDict, represent_dictionary_order) @contextmanager def no_duplicate_yaml(): yaml.SafeLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, no_duplicates_constructor ) yaml.SafeLoader.add_constructor("tag:yaml.org,2002:python/tuple", tuple_constructor) try: yield finally: yaml.SafeLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, yaml.constructor.SafeConstructor.construct_yaml_map, ) def yaml_load(stream): """Parse YAML in a context where duplicate keys raise exception""" with no_duplicate_yaml(): return yaml.safe_load(stream) def classname(ob): """Get the object's class's name as package.module.Class""" import inspect if inspect.isclass(ob): return ".".join([ob.__module__, ob.__name__]) else: return ".".join([ob.__class__.__module__, ob.__class__.__name__]) class DictSerialiseMixin(object): __tok_cache = None def __new__(cls, *args, **kwargs): """Capture creation args when instantiating""" o = object.__new__(cls) o._captured_init_args = args o._captured_init_kwargs = kwargs return o @property def classname(self): return classname(self) def __dask_tokenize__(self): if self.__tok_cache is None: from intake.source.utils import tokenize self.__tok_cache = tokenize(self) return self.__tok_cache def __getstate__(self): args = [ arg.__getstate__() if isinstance(arg, DictSerialiseMixin) else arg for arg in self._captured_init_args ] # We employ OrderedDict in several places. The motivation # is to speed up dask tokenization. When dask tokenizes a plain dict, # it sorts the keys, and it turns out that this sort operation # dominates the call time, even for very small dicts. Using an # OrderedDict steers dask toward a different and faster tokenization. kwargs = collections.OrderedDict( { k: arg.__getstate__() if isinstance(arg, DictSerialiseMixin) else arg for k, arg in self._captured_init_kwargs.items() } ) return collections.OrderedDict(cls=self.classname, args=args, kwargs=kwargs) def __setstate__(self, state): # reconstitute instances here self._captured_init_kwargs = state["kwargs"] self._captured_init_args = state["args"] state.pop("cls", None) self.__init__(*state["args"], **state["kwargs"]) def __hash__(self): from intake.source.utils import tokenize return int(tokenize(self), 16) def __eq__(self, other): return hash(self) == hash(other) def remake_instance(data): import importlib if isinstance(data, str): data = {"cls": data} else: data = data.copy() mod, klass = data.pop("cls").rsplit(".", 1) module = importlib.import_module(mod) cl = getattr(module, klass) return cl(*data.get("args", ()), **data.get("kwargs", {})) def pretty_describe(object, nestedness=0, indent=2): """Maintain dict ordering - but make string version prettier""" if not isinstance(object, dict): return str(object) sep = f'\n{" " * nestedness * indent}' out = sep.join(f"{k}: {pretty_describe(v, nestedness + 1)}" for k, v in object.items()) if nestedness > 0 and out: return f"{sep}{out}" return out def decode_datetime(obj): import numpy if not isinstance(obj, numpy.ndarray) and "__datetime__" in obj: try: obj = datetime.datetime.strptime( obj["as_str"], "%Y%m%dT%H:%M:%S.%f%z", ) except ValueError: # Perhaps lacking tz info obj = datetime.datetime.strptime( obj["as_str"], "%Y%m%dT%H:%M:%S.%f", ) return obj def encode_datetime(obj): if isinstance(obj, datetime.datetime): return {"__datetime__": True, "as_str": obj.strftime("%Y%m%dT%H:%M:%S.%f%z")} return obj class RegistryView(collections.abc.Mapping): """ Wrap registry dict in a read-only dict view. Subclasses define attributes filled into warning and error messages: - self._registry_name - self._register_func_name - self._unregister_func_name """ def __init__(self, registry): self._registry = registry def __repr__(self): return f"{self.__class__.__name__}({self._registry!r})" def __getitem__(self, key): return self._registry[key] def __iter__(self): yield from self._registry def __len__(self): return len(self._registry) # Support the common mutation methods for now, but warn. def update(self, *args, **kwargs): warnings.warn( f"In a future release of intake, the {self._registry_name} will " f"not be directly mutable. Use {self._register_func_name}.", DeprecationWarning, ) self._registry.update(*args, **kwargs) # raise TypeError( # f"The registry cannot be edited directly. " # f"Instead, use the {self._register_func_name{") def __setitem__(self, key, value): warnings.warn( f"In a future release of intake, the {self._registry_name} will " f"not be directly mutable. Use {self._register_func_name}.", DeprecationWarning, ) self._registry[key] = value # raise TypeError( # f"The registry cannot be edited directly. " # f"Instead, use the {self._register_func_name{") def __delitem__(self, key): warnings.warn( f"In a future release of intake, the {self._registry_name} will " f"not be directly mutable. Use {self._unregister_func_name}.", DeprecationWarning, ) del self._registry[key] # raise TypeError( # f"The registry cannot be edited directly. " # f"Instead, use the {self._unregister_func_name{") class DriverRegistryView(RegistryView): # This attributes are used by the base class # to fill in warning and error messages. _registry_name = "intake.registry" _register_func_name = "intake.register_driver" _unregister_func_name = "intake.unregister_driver" class ContainerRegistryView(RegistryView): # This attributes are used by the base class # to fill in warning and error messages. _registry_name = "intake.container_map" _register_func_name = "intake.register_container" _unregister_func_name = "intake.unregister_container" class ModuleImporter: def __init__(self, destination): self.destination = destination self.module = None def __getattribute__(self, item): d = object.__getattribute__(self, "__dict__") if item in d: return d[item] if self.module is None: print("Importing module: ", self.destination) self.module = __import__(self.destination) else: print("Referencing module: ", self.destination) sys.modules[self.destination] = self.module return getattr(self.module, item) def is_notebook() -> bool: """Check if code is running in a notebook Copied from tqdm.autonotebook Returns ------- bool True if inside a notebook. False if not inside a notebook. """ try: get_ipython = sys.modules["IPython"].get_ipython if "IPKernelApp" not in get_ipython().config: # pragma: no cover raise ImportError("console") if "VSCODE_PID" in os.environ: # pragma: no cover raise ImportError("vscode") return True except Exception: return False def is_fsspec_url(s: str) -> bool: """Simple test to see if given string is likely an fsspec URL""" return "://" in s or "::" in s ================================================ FILE: pyproject.toml ================================================ [tool.black] line-length = 100 [tool.ruff] line-length = 130 ignore = ["E402"] # top of file imports [tool.ruff.per-file-ignores] "*/tests/*" = ["E722"] # bare `except` "intake/util_tests.py" = ["E722"] # bare `except` "__init__.py" = ["F401"] # imported but unused [tool.isort] profile = "black" [build-system] requires = ["setuptools>=64", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" [project] name = "intake" dynamic = ["version"] description = "Data catalog, search and load" readme = "README.md" requires-python = ">=3.10" license = {text = "MIT"} authors = [ {name = "Martin Durant", email = "martin.durant@alumni.utoronto.ca"}, ] classifiers = [ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", "Development Status :: 4 - Beta", ] dependencies = [ "fsspec >=2023.0.0", "pyyaml", "platformdirs", "networkx" ] [tool.setuptools_scm] version_file = "intake/_version.py" [tool.setuptools] include-package-data = false zip-safe = false [tool.setuptools.packages.find] exclude = ["*tests*", "dist", "*examples*", "*scripts*", "*docs*"] namespaces = false [flake8] exclude = "__init__.py" ignore = "E,W" select = "F,E101,E111,E501" max-line-length = 100 [project.entry-points."intake.drivers"] csv = "intake.source.csv:CSVSource" jsonfiles = "intake.source.jsonfiles:JSONFileSource" numpy = "intake.source.npy:NPySource" tiled_cat = "intake.source.tiled:TiledCatalog" textfiles = "intake.source.textfiles:TextFilesSource" ndzarr = "intake.source.zarr:ZarrArraySource" ================================================ FILE: readthedocs.yml ================================================ version: 2 formats: all build: os: ubuntu-22.04 tools: python: mambaforge-4.10 conda: environment: docs/environment.yml python: install: - method: pip path: . sphinx: configuration: docs/source/conf.py ================================================ FILE: scripts/ci/environment-pip.yml ================================================ name: test_env channels: - defaults dependencies: - python=3.10 - pip - pip: - rangehttpserver - aiohttp - flask - appdirs - dask - jinja2 - numpy - pyyaml - requests - msgpack-numpy - pytest-cov - coveralls - pytest - fsspec - intake-parquet - zarr - notebook - panel - hvplot - bokeh - dask - h5netcdf - h5py < 3.15 # due to HDF error, see netcdf4 Issue #1438 - intake - netcdf4 - pip - pydap - pytest - rasterio - s3fs - scikit-image - xarray - zarr - moto - toolz - pre-commit - siphon ================================================ FILE: scripts/ci/environment-py310.yml ================================================ name: test_env channels: - conda-forge dependencies: - python=3.10 - aiohttp - flask - appdirs - dask - jinja2 - numpy - pyyaml - requests - msgpack-numpy - pytest - fsspec - intake-parquet - zarr - notebook - panel - hvplot - bokeh - dask - intake - pip - pytest - rasterio - s3fs - h5netcdf - xarray - zarr - moto - httpx - typer - pydantic >=1.8 - sqlalchemy - click - rangehttpserver - pre-commit - sqlalchemy - python-duckdb - pytest-postgresql - psycopg - siphon - postgresql ================================================ FILE: scripts/ci/environment-py311.yml ================================================ name: test_env channels: - conda-forge dependencies: - python=3.11 - aiohttp - flask - appdirs - dask - jinja2 - numpy - pyyaml - requests - msgpack-numpy - pytest - fsspec - intake-parquet - zarr - notebook - panel - hvplot - bokeh - dask - intake - pip - pytest - rasterio - s3fs - xarray - h5netcdf - zarr - moto - httpx - typer - pydantic >=1.8 - sqlalchemy - click - rangehttpserver - pre-commit - pytest-postgresql - psycopg - python-duckdb - siphon - postgresql ================================================ FILE: scripts/ci/environment-py312.yml ================================================ name: test_env channels: - conda-forge dependencies: - python=3.12 - aiohttp - flask - appdirs - dask - jinja2 - numpy - pyyaml - requests - msgpack-numpy - pytest - fsspec - intake-parquet - zarr - notebook - panel - hvplot - bokeh - dask - intake - pip - pytest - rasterio - s3fs - xarray - h5netcdf - zarr - moto - httpx - typer - pydantic >=1.8 - sqlalchemy - click - rangehttpserver - pre-commit - pytest-postgresql - psycopg - python-duckdb - siphon - postgresql ================================================ FILE: scripts/ci/environment-py313.yml ================================================ name: test_env channels: - conda-forge dependencies: - python=3.13 - aiohttp - flask - appdirs - dask - jinja2 - numpy - pyyaml - requests - msgpack-numpy - pytest-cov - coveralls - pytest - fsspec - intake-parquet - zarr - notebook - panel - hvplot - bokeh - dask - h5netcdf - intake - netcdf4 - pip - pydap - pytest - rasterio - s3fs - scikit-image - xarray - zarr - moto - rangehttpserver - pre-commit - sqlalchemy - python-duckdb - pytest-postgresql <7 - psycopg - siphon - postgresql ================================================ FILE: scripts/ci/environment-py314.yml ================================================ name: test_env channels: - conda-forge dependencies: - python=3.14 - aiohttp - flask - appdirs - dask - jinja2 - numpy - pyyaml - requests - msgpack-numpy - pytest - fsspec - intake-parquet - zarr - notebook - panel - hvplot - bokeh - intake - pip - pytest - rasterio - s3fs - xarray - zarr - moto - httpx - typer - pydantic >=1.8 - sqlalchemy - click - rangehttpserver - pre-commit - sqlalchemy - python-duckdb - pytest-postgresql - psycopg - siphon - postgresql