Repository: ngr-t/SublimeHermes Branch: master Commit: 0feec81caa12 Files: 114 Total size: 624.0 KB Directory structure: gitextract_9beunihp/ ├── .build ├── .codecov.yml ├── .flake8 ├── .github/ │ └── workflows/ │ └── ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .python-version ├── CONTRIBUTORS.md ├── Default.sublime-commands ├── Default.sublime-keymap ├── Helium.sublime-settings ├── Helium.sublime-syntax ├── LICENSE.TXT ├── Main.sublime-menu ├── README.md ├── dependencies.json ├── helium.py ├── lib/ │ ├── __init__.py │ ├── client/ │ │ ├── __init__.py │ │ ├── decorator.py │ │ ├── decorator_version │ │ ├── ipython_genutils/ │ │ │ ├── __init__.py │ │ │ ├── _version.py │ │ │ ├── encoding.py │ │ │ ├── importstring.py │ │ │ ├── ipstruct.py │ │ │ ├── path.py │ │ │ ├── py3compat.py │ │ │ ├── tempdir.py │ │ │ └── text.py │ │ ├── jupyter_client/ │ │ │ ├── __init__.py │ │ │ ├── _version.py │ │ │ ├── adapter.py │ │ │ ├── blocking/ │ │ │ │ ├── __init__.py │ │ │ │ ├── channels.py │ │ │ │ └── client.py │ │ │ ├── channels.py │ │ │ ├── channelsabc.py │ │ │ ├── client.py │ │ │ ├── clientabc.py │ │ │ ├── connect.py │ │ │ ├── consoleapp.py │ │ │ ├── ioloop/ │ │ │ │ ├── __init__.py │ │ │ │ ├── manager.py │ │ │ │ └── restarter.py │ │ │ ├── jsonutil.py │ │ │ ├── kernelapp.py │ │ │ ├── kernelspec.py │ │ │ ├── kernelspecapp.py │ │ │ ├── launcher.py │ │ │ ├── localinterfaces.py │ │ │ ├── manager.py │ │ │ ├── managerabc.py │ │ │ ├── multikernelmanager.py │ │ │ ├── restarter.py │ │ │ ├── runapp.py │ │ │ ├── session.py │ │ │ ├── threaded.py │ │ │ └── win_interrupt.py │ │ ├── jupyter_core/ │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ ├── application.py │ │ │ ├── command.py │ │ │ ├── migrate.py │ │ │ ├── paths.py │ │ │ ├── troubleshoot.py │ │ │ ├── utils/ │ │ │ │ ├── __init__.py │ │ │ │ └── shutil_which.py │ │ │ └── version.py │ │ ├── traitlets/ │ │ │ ├── __init__.py │ │ │ ├── _version.py │ │ │ ├── config/ │ │ │ │ ├── __init__.py │ │ │ │ ├── application.py │ │ │ │ ├── configurable.py │ │ │ │ ├── loader.py │ │ │ │ └── manager.py │ │ │ ├── log.py │ │ │ ├── traitlets.py │ │ │ └── utils/ │ │ │ ├── __init__.py │ │ │ ├── bunch.py │ │ │ ├── getargspec.py │ │ │ ├── importstring.py │ │ │ ├── sentinel.py │ │ │ └── tests/ │ │ │ ├── __init__.py │ │ │ ├── test_bunch.py │ │ │ └── test_importstring.py │ │ ├── update_decorator.sh │ │ ├── update_ipython_genutils.sh │ │ ├── update_jupyter_client.sh │ │ ├── update_jupyter_core.sh │ │ └── update_traitlets.sh │ ├── kernel.py │ └── utils.py ├── messages/ │ ├── 0.3.1.txt │ ├── 0.3.2.txt │ ├── 0.3.3.txt │ ├── 0.3.4.txt │ ├── 0.3.5.txt │ ├── 0.3.6.txt │ ├── 0.4.0.txt │ ├── 0.4.1.txt │ ├── 0.4.2.txt │ ├── 0.4.3.txt │ ├── 0.5.0.txt │ ├── 0.5.1.txt │ ├── 0.6.0.txt │ ├── 0.6.1.txt │ ├── 0.6.2.txt │ └── 0.6.3.txt ├── messages.json ├── pyproject.toml └── tests/ ├── _helpers.py ├── test_cell_handling.py └── test_tests_running.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .build ================================================ { "automatic_order": false, "iterations": 1, "mods_load_order": [ "tests/test_tests_running.py", "lib/utils.py", "lib/kernel.py", "helium.py", "lib/__init__.py", "lib/client/__init__.py", "lib/client/decorator.py", "lib/client/jupyter_client/channelsabc.py", "lib/client/jupyter_client/threaded.py", "lib/client/jupyter_client/restarter.py", "lib/client/jupyter_client/__init__.py", "lib/client/jupyter_client/client.py", "lib/client/jupyter_client/managerabc.py", "lib/client/jupyter_client/adapter.py", "lib/client/jupyter_client/localinterfaces.py", "lib/client/jupyter_client/win_interrupt.py", "lib/client/jupyter_client/_version.py", "lib/client/jupyter_client/connect.py", "lib/client/jupyter_client/kernelspec.py", "lib/client/jupyter_client/launcher.py", "lib/client/jupyter_client/session.py", "lib/client/jupyter_client/multikernelmanager.py", "lib/client/jupyter_client/jsonutil.py", "lib/client/jupyter_client/manager.py", "lib/client/jupyter_client/kernelspecapp.py", "lib/client/jupyter_client/runapp.py", "lib/client/jupyter_client/consoleapp.py", "lib/client/jupyter_client/channels.py", "lib/client/jupyter_client/clientabc.py", "lib/client/jupyter_client/blocking/__init__.py", "lib/client/jupyter_client/blocking/client.py", "lib/client/jupyter_client/blocking/channels.py", "lib/client/jupyter_client/ioloop/restarter.py", "lib/client/jupyter_client/ioloop/__init__.py", "lib/client/jupyter_client/ioloop/manager.py", "lib/client/jupyter_client/tests/__init__.py", "lib/client/jupyter_client/tests/signalkernel.py", "lib/client/jupyter_client/tests/test_multikernelmanager.py", "lib/client/jupyter_client/tests/test_client.py", "lib/client/jupyter_client/tests/test_connect.py", "lib/client/jupyter_client/tests/test_session.py", "lib/client/jupyter_client/tests/test_adapter.py", "lib/client/jupyter_client/tests/test_kernelspec.py", "lib/client/jupyter_client/tests/test_public_api.py", "lib/client/jupyter_client/tests/test_localinterfaces.py", "lib/client/jupyter_client/tests/test_jsonutil.py", "lib/client/jupyter_client/tests/test_kernelmanager.py", "lib/client/jupyter_client/tests/utils.py", "lib/client/jupyter_client/tests/test_kernelapp.py", "lib/client/jupyter_core/__init__.py", "lib/client/jupyter_core/__main__.py", "lib/client/jupyter_core/migrate.py", "lib/client/jupyter_core/troubleshoot.py", "lib/client/jupyter_core/application.py", "lib/client/jupyter_core/paths.py", "lib/client/jupyter_core/version.py", "lib/client/jupyter_core/command.py", "lib/client/jupyter_core/utils/__init__.py", "lib/client/jupyter_core/utils/shutil_which.py", "lib/client/jupyter_core/tests/__init__.py", "lib/client/jupyter_core/tests/test_paths.py", "lib/client/jupyter_core/tests/test_application.py", "lib/client/jupyter_core/tests/test_migrate.py", "lib/client/jupyter_core/tests/mocking.py", "lib/client/jupyter_core/tests/test_command.py", "lib/client/jupyter_core/tests/dotipython_empty/profile_default/ipython_notebook_config.py", "lib/client/jupyter_core/tests/dotipython_empty/profile_default/ipython_kernel_config.py", "lib/client/jupyter_core/tests/dotipython_empty/profile_default/ipython_console_config.py", "lib/client/jupyter_core/tests/dotipython_empty/profile_default/ipython_config.py", "lib/client/jupyter_core/tests/dotipython_empty/profile_default/ipython_nbconvert_config.py", "lib/client/jupyter_core/tests/dotipython/profile_default/ipython_notebook_config.py", "lib/client/jupyter_core/tests/dotipython/profile_default/ipython_kernel_config.py", "lib/client/jupyter_core/tests/dotipython/profile_default/ipython_console_config.py", "lib/client/jupyter_core/tests/dotipython/profile_default/ipython_config.py", "lib/client/jupyter_core/tests/dotipython/profile_default/ipython_nbconvert_config.py", "lib/client/traitlets/log.py", "lib/client/traitlets/traitlets.py", "lib/client/traitlets/__init__.py", "lib/client/traitlets/_version.py", "lib/client/traitlets/utils/getargspec.py", "lib/client/traitlets/utils/__init__.py", "lib/client/traitlets/utils/importstring.py", "lib/client/traitlets/utils/bunch.py", "lib/client/traitlets/utils/sentinel.py", "lib/client/traitlets/utils/tests/__init__.py", "lib/client/traitlets/utils/tests/test_bunch.py", "lib/client/traitlets/utils/tests/test_importstring.py", "lib/client/traitlets/config/__init__.py", "lib/client/traitlets/config/configurable.py", "lib/client/traitlets/config/application.py", "lib/client/traitlets/config/loader.py", "lib/client/traitlets/config/manager.py", "lib/client/traitlets/config/tests/test_configurable.py", "lib/client/traitlets/config/tests/__init__.py", "lib/client/traitlets/config/tests/test_loader.py", "lib/client/traitlets/config/tests/test_application.py", "lib/client/traitlets/tests/_warnings.py", "lib/client/traitlets/tests/__init__.py", "lib/client/traitlets/tests/test_traitlets_enum.py", "lib/client/traitlets/tests/test_traitlets.py", "lib/client/traitlets/tests/utils.py", "lib/client/ipython_genutils/__init__.py", "lib/client/ipython_genutils/importstring.py", "lib/client/ipython_genutils/ipstruct.py", "lib/client/ipython_genutils/path.py", "lib/client/ipython_genutils/_version.py", "lib/client/ipython_genutils/py3compat.py", "lib/client/ipython_genutils/encoding.py", "lib/client/ipython_genutils/tempdir.py", "lib/client/ipython_genutils/text.py", "lib/client/ipython_genutils/tests/__init__.py", "lib/client/ipython_genutils/tests/test_text.py", "lib/client/ipython_genutils/tests/test_tempdir.py", "lib/client/ipython_genutils/tests/test_path.py", "lib/client/ipython_genutils/tests/test_importstring.py", "lib/client/ipython_genutils/testing/__init__.py", "lib/client/ipython_genutils/testing/decorators.py", ".venv/lib/python3.8/site-packages/mypy_extensions.py", ".venv/lib/python3.8/site-packages/flake8_docstrings.py", ".venv/lib/python3.8/site-packages/filelock.py", ".venv/lib/python3.8/site-packages/flake8_eradicate.py", ".venv/lib/python3.8/site-packages/typing_extensions.py", ".venv/lib/python3.8/site-packages/easy_install.py", ".venv/lib/python3.8/site-packages/cfgv.py", ".venv/lib/python3.8/site-packages/toml.py", ".venv/lib/python3.8/site-packages/black.py", ".venv/lib/python3.8/site-packages/nodeenv.py", ".venv/lib/python3.8/site-packages/pycodestyle.py", ".venv/lib/python3.8/site-packages/flake8_blind_except.py", ".venv/lib/python3.8/site-packages/appdirs.py", ".venv/lib/python3.8/site-packages/eradicate.py", ".venv/lib/python3.8/site-packages/_black_version.py", ".venv/lib/python3.8/site-packages/blackd.py", ".venv/lib/python3.8/site-packages/flake8_comprehensions.py", ".venv/lib/python3.8/site-packages/mccabe.py", ".venv/lib/python3.8/site-packages/six.py", ".venv/lib/python3.8/site-packages/entrypoints.py", ".venv/lib/python3.8/site-packages/mypy/nodes.py", ".venv/lib/python3.8/site-packages/mypy/fastparse2.py", ".venv/lib/python3.8/site-packages/mypy/errorcodes.py", ".venv/lib/python3.8/site-packages/mypy/gclogger.py", ".venv/lib/python3.8/site-packages/mypy/semanal_typeddict.py", ".venv/lib/python3.8/site-packages/mypy/stubgenc.py", ".venv/lib/python3.8/site-packages/mypy/message_registry.py", ".venv/lib/python3.8/site-packages/mypy/__init__.py", ".venv/lib/python3.8/site-packages/mypy/literals.py", ".venv/lib/python3.8/site-packages/mypy/tvar_scope.py", ".venv/lib/python3.8/site-packages/mypy/semanal_shared.py", ".venv/lib/python3.8/site-packages/mypy/applytype.py", ".venv/lib/python3.8/site-packages/mypy/ipc.py", ".venv/lib/python3.8/site-packages/mypy/semanal_infer.py", ".venv/lib/python3.8/site-packages/mypy/__main__.py", ".venv/lib/python3.8/site-packages/mypy/stats.py", ".venv/lib/python3.8/site-packages/mypy/semanal_newtype.py", ".venv/lib/python3.8/site-packages/mypy/errors.py", ".venv/lib/python3.8/site-packages/mypy/expandtype.py", ".venv/lib/python3.8/site-packages/mypy/checker.py", ".venv/lib/python3.8/site-packages/mypy/meet.py", ".venv/lib/python3.8/site-packages/mypy/build.py", ".venv/lib/python3.8/site-packages/mypy/constraints.py", ".venv/lib/python3.8/site-packages/mypy/fscache.py", ".venv/lib/python3.8/site-packages/mypy/git.py", ".venv/lib/python3.8/site-packages/mypy/stubdoc.py", ".venv/lib/python3.8/site-packages/mypy/api.py", ".venv/lib/python3.8/site-packages/mypy/traverser.py", ".venv/lib/python3.8/site-packages/mypy/fswatcher.py", ".venv/lib/python3.8/site-packages/mypy/main.py", ".venv/lib/python3.8/site-packages/mypy/fastparse.py", ".venv/lib/python3.8/site-packages/mypy/modulefinder.py", ".venv/lib/python3.8/site-packages/mypy/find_sources.py", ".venv/lib/python3.8/site-packages/mypy/semanal_pass1.py", ".venv/lib/python3.8/site-packages/mypy/plugin.py", ".venv/lib/python3.8/site-packages/mypy/infer.py", ".venv/lib/python3.8/site-packages/mypy/scope.py", ".venv/lib/python3.8/site-packages/mypy/typestate.py", ".venv/lib/python3.8/site-packages/mypy/semanal_enum.py", ".venv/lib/python3.8/site-packages/mypy/mixedtraverser.py", ".venv/lib/python3.8/site-packages/mypy/bogus_type.py", ".venv/lib/python3.8/site-packages/mypy/semanal.py", ".venv/lib/python3.8/site-packages/mypy/argmap.py", ".venv/lib/python3.8/site-packages/mypy/options.py", ".venv/lib/python3.8/site-packages/mypy/visitor.py", ".venv/lib/python3.8/site-packages/mypy/stubutil.py", ".venv/lib/python3.8/site-packages/mypy/binder.py", ".venv/lib/python3.8/site-packages/mypy/typeanal.py", ".venv/lib/python3.8/site-packages/mypy/strconv.py", ".venv/lib/python3.8/site-packages/mypy/typeops.py", ".venv/lib/python3.8/site-packages/mypy/freetree.py", ".venv/lib/python3.8/site-packages/mypy/config_parser.py", ".venv/lib/python3.8/site-packages/mypy/parse.py", ".venv/lib/python3.8/site-packages/mypy/suggestions.py", ".venv/lib/python3.8/site-packages/mypy/sitepkgs.py", ".venv/lib/python3.8/site-packages/mypy/checkstrformat.py", ".venv/lib/python3.8/site-packages/mypy/dmypy_util.py", ".venv/lib/python3.8/site-packages/mypy/dmypy_server.py", ".venv/lib/python3.8/site-packages/mypy/stubgen.py", ".venv/lib/python3.8/site-packages/mypy/indirection.py", ".venv/lib/python3.8/site-packages/mypy/type_visitor.py", ".venv/lib/python3.8/site-packages/mypy/semanal_main.py", ".venv/lib/python3.8/site-packages/mypy/renaming.py", ".venv/lib/python3.8/site-packages/mypy/types.py", ".venv/lib/python3.8/site-packages/mypy/semanal_classprop.py", ".venv/lib/python3.8/site-packages/mypy/checkmember.py", ".venv/lib/python3.8/site-packages/mypy/erasetype.py", ".venv/lib/python3.8/site-packages/mypy/reachability.py", ".venv/lib/python3.8/site-packages/mypy/state.py", ".venv/lib/python3.8/site-packages/mypy/report.py", ".venv/lib/python3.8/site-packages/mypy/maptype.py", ".venv/lib/python3.8/site-packages/mypy/subtypes.py", ".venv/lib/python3.8/site-packages/mypy/metastore.py", ".venv/lib/python3.8/site-packages/mypy/fixup.py", ".venv/lib/python3.8/site-packages/mypy/semanal_typeargs.py", ".venv/lib/python3.8/site-packages/mypy/messages.py", ".venv/lib/python3.8/site-packages/mypy/checkexpr.py", ".venv/lib/python3.8/site-packages/mypy/sametypes.py", ".venv/lib/python3.8/site-packages/mypy/split_namespace.py", ".venv/lib/python3.8/site-packages/mypy/join.py", ".venv/lib/python3.8/site-packages/mypy/memprofile.py", ".venv/lib/python3.8/site-packages/mypy/exprtotype.py", ".venv/lib/python3.8/site-packages/mypy/util.py", ".venv/lib/python3.8/site-packages/mypy/lookup.py", ".venv/lib/python3.8/site-packages/mypy/semanal_namedtuple.py", ".venv/lib/python3.8/site-packages/mypy/solve.py", ".venv/lib/python3.8/site-packages/mypy/sharedparse.py", ".venv/lib/python3.8/site-packages/mypy/moduleinspect.py", ".venv/lib/python3.8/site-packages/mypy/moduleinfo.py", ".venv/lib/python3.8/site-packages/mypy/dmypy_os.py", ".venv/lib/python3.8/site-packages/mypy/typetraverser.py", ".venv/lib/python3.8/site-packages/mypy/treetransform.py", ".venv/lib/python3.8/site-packages/mypy/mro.py", ".venv/lib/python3.8/site-packages/mypy/version.py", ".venv/lib/python3.8/site-packages/mypy/typevars.py", ".venv/lib/python3.8/site-packages/mypy/defaults.py", ".venv/lib/python3.8/site-packages/mypy/typeshed/tests/mypy_selftest.py", ".venv/lib/python3.8/site-packages/mypy/typeshed/tests/mypy_test.py", ".venv/lib/python3.8/site-packages/mypy/typeshed/tests/check_consistent.py", ".venv/lib/python3.8/site-packages/mypy/typeshed/tests/pytype_test.py", ".venv/lib/python3.8/site-packages/mypy/test/testmoduleinfo.py", ".venv/lib/python3.8/site-packages/mypy/test/testgraph.py", ".venv/lib/python3.8/site-packages/mypy/test/testsamples.py", ".venv/lib/python3.8/site-packages/mypy/test/__init__.py", ".venv/lib/python3.8/site-packages/mypy/test/testfinegrained.py", ".venv/lib/python3.8/site-packages/mypy/test/testerrorstream.py", ".venv/lib/python3.8/site-packages/mypy/test/testdaemon.py", ".venv/lib/python3.8/site-packages/mypy/test/testinfer.py", ".venv/lib/python3.8/site-packages/mypy/test/testmerge.py", ".venv/lib/python3.8/site-packages/mypy/test/testpep561.py", ".venv/lib/python3.8/site-packages/mypy/test/collect.py", ".venv/lib/python3.8/site-packages/mypy/test/testmodulefinder.py", ".venv/lib/python3.8/site-packages/mypy/test/testdiff.py", ".venv/lib/python3.8/site-packages/mypy/test/visitors.py", ".venv/lib/python3.8/site-packages/mypy/test/testcmdline.py", ".venv/lib/python3.8/site-packages/mypy/test/testsubtypes.py", ".venv/lib/python3.8/site-packages/mypy/test/data.py", ".venv/lib/python3.8/site-packages/mypy/test/testcheck.py", ".venv/lib/python3.8/site-packages/mypy/test/testtransform.py", ".venv/lib/python3.8/site-packages/mypy/test/config.py", ".venv/lib/python3.8/site-packages/mypy/test/testformatter.py", ".venv/lib/python3.8/site-packages/mypy/test/testsemanal.py", ".venv/lib/python3.8/site-packages/mypy/test/testdeps.py", ".venv/lib/python3.8/site-packages/mypy/test/testpythoneval.py", ".venv/lib/python3.8/site-packages/mypy/test/testtypes.py", ".venv/lib/python3.8/site-packages/mypy/test/testreports.py", ".venv/lib/python3.8/site-packages/mypy/test/typefixture.py", ".venv/lib/python3.8/site-packages/mypy/test/update.py", ".venv/lib/python3.8/site-packages/mypy/test/testfinegrainedcache.py", ".venv/lib/python3.8/site-packages/mypy/test/helpers.py", ".venv/lib/python3.8/site-packages/mypy/test/testapi.py", ".venv/lib/python3.8/site-packages/mypy/test/teststubgen.py", ".venv/lib/python3.8/site-packages/mypy/test/testsolve.py", ".venv/lib/python3.8/site-packages/mypy/test/testparse.py", ".venv/lib/python3.8/site-packages/mypy/test/testargs.py", ".venv/lib/python3.8/site-packages/mypy/test/testipc.py", ".venv/lib/python3.8/site-packages/mypy/test/testtypegen.py", ".venv/lib/python3.8/site-packages/mypy/test/testmypyc.py", ".venv/lib/python3.8/site-packages/mypy/server/mergecheck.py", ".venv/lib/python3.8/site-packages/mypy/server/deps.py", ".venv/lib/python3.8/site-packages/mypy/server/__init__.py", ".venv/lib/python3.8/site-packages/mypy/server/target.py", ".venv/lib/python3.8/site-packages/mypy/server/astdiff.py", ".venv/lib/python3.8/site-packages/mypy/server/trigger.py", ".venv/lib/python3.8/site-packages/mypy/server/aststrip.py", ".venv/lib/python3.8/site-packages/mypy/server/update.py", ".venv/lib/python3.8/site-packages/mypy/server/objgraph.py", ".venv/lib/python3.8/site-packages/mypy/server/subexpr.py", ".venv/lib/python3.8/site-packages/mypy/server/astmerge.py", ".venv/lib/python3.8/site-packages/mypy/plugins/attrs.py", ".venv/lib/python3.8/site-packages/mypy/plugins/__init__.py", ".venv/lib/python3.8/site-packages/mypy/plugins/common.py", ".venv/lib/python3.8/site-packages/mypy/plugins/dataclasses.py", ".venv/lib/python3.8/site-packages/mypy/plugins/enums.py", ".venv/lib/python3.8/site-packages/mypy/plugins/ctypes.py", ".venv/lib/python3.8/site-packages/mypy/plugins/default.py", ".venv/lib/python3.8/site-packages/mypy/dmypy/__init__.py", ".venv/lib/python3.8/site-packages/mypy/dmypy/client.py", ".venv/lib/python3.8/site-packages/mypy/dmypy/__main__.py", ".venv/lib/python3.8/site-packages/pkg_resources/__init__.py", ".venv/lib/python3.8/site-packages/pkg_resources/py31compat.py", ".venv/lib/python3.8/site-packages/pkg_resources/extern/__init__.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/__init__.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/pyparsing.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/appdirs.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/six.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/__init__.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/_compat.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/__about__.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/_structures.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/requirements.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/utils.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/specifiers.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/version.py", ".venv/lib/python3.8/site-packages/pkg_resources/_vendor/packaging/markers.py", ".venv/lib/python3.8/site-packages/pre_commit/__init__.py", ".venv/lib/python3.8/site-packages/pre_commit/hook.py", ".venv/lib/python3.8/site-packages/pre_commit/staged_files_only.py", ".venv/lib/python3.8/site-packages/pre_commit/__main__.py", ".venv/lib/python3.8/site-packages/pre_commit/envcontext.py", ".venv/lib/python3.8/site-packages/pre_commit/output.py", ".venv/lib/python3.8/site-packages/pre_commit/git.py", ".venv/lib/python3.8/site-packages/pre_commit/error_handler.py", ".venv/lib/python3.8/site-packages/pre_commit/main.py", ".venv/lib/python3.8/site-packages/pre_commit/make_archives.py", ".venv/lib/python3.8/site-packages/pre_commit/xargs.py", ".venv/lib/python3.8/site-packages/pre_commit/color.py", ".venv/lib/python3.8/site-packages/pre_commit/five.py", ".venv/lib/python3.8/site-packages/pre_commit/logging_handler.py", ".venv/lib/python3.8/site-packages/pre_commit/prefix.py", ".venv/lib/python3.8/site-packages/pre_commit/parse_shebang.py", ".venv/lib/python3.8/site-packages/pre_commit/store.py", ".venv/lib/python3.8/site-packages/pre_commit/constants.py", ".venv/lib/python3.8/site-packages/pre_commit/util.py", ".venv/lib/python3.8/site-packages/pre_commit/repository.py", ".venv/lib/python3.8/site-packages/pre_commit/clientlib.py", ".venv/lib/python3.8/site-packages/pre_commit/color_windows.py", ".venv/lib/python3.8/site-packages/pre_commit/file_lock.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/__init__.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/docker.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/python.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/rust.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/docker_image.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/pcre.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/ruby.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/python_venv.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/script.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/conda.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/pygrep.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/helpers.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/node.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/golang.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/all.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/fail.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/system.py", ".venv/lib/python3.8/site-packages/pre_commit/languages/swift.py", ".venv/lib/python3.8/site-packages/pre_commit/resources/__init__.py", ".venv/lib/python3.8/site-packages/pre_commit/resources/empty_template_setup.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/hook_impl.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/run.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/__init__.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/clean.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/try_repo.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/sample_config.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/install_uninstall.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/autoupdate.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/gc.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/init_templatedir.py", ".venv/lib/python3.8/site-packages/pre_commit/commands/migrate_config.py", ".venv/lib/python3.8/site-packages/pre_commit/meta_hooks/identity.py", ".venv/lib/python3.8/site-packages/pre_commit/meta_hooks/__init__.py", ".venv/lib/python3.8/site-packages/pre_commit/meta_hooks/check_hooks_apply.py", ".venv/lib/python3.8/site-packages/pre_commit/meta_hooks/check_useless_excludes.py", ".venv/lib/python3.8/site-packages/virtualenv/error.py", ".venv/lib/python3.8/site-packages/virtualenv/info.py", ".venv/lib/python3.8/site-packages/virtualenv/pyenv_cfg.py", ".venv/lib/python3.8/site-packages/virtualenv/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/__main__.py", ".venv/lib/python3.8/site-packages/virtualenv/session.py", ".venv/lib/python3.8/site-packages/virtualenv/dirs.py", ".venv/lib/python3.8/site-packages/virtualenv/report.py", ".venv/lib/python3.8/site-packages/virtualenv/version.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/via_template.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/activator.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/batch/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/fish/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/powershell/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/cshell/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/python/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/python/activate_this.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/bash/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/activation/xonsh/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/util/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/util/zipapp.py", ".venv/lib/python3.8/site-packages/virtualenv/util/lock.py", ".venv/lib/python3.8/site-packages/virtualenv/util/six.py", ".venv/lib/python3.8/site-packages/virtualenv/util/subprocess/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/util/subprocess/_win_subprocess.py", ".venv/lib/python3.8/site-packages/virtualenv/util/path/_sync.py", ".venv/lib/python3.8/site-packages/virtualenv/util/path/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/util/path/_permission.py", ".venv/lib/python3.8/site-packages/virtualenv/util/path/_pathlib/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/util/path/_pathlib/via_os_path.py", ".venv/lib/python3.8/site-packages/virtualenv/run/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/run/plugin/discovery.py", ".venv/lib/python3.8/site-packages/virtualenv/run/plugin/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/run/plugin/seeders.py", ".venv/lib/python3.8/site-packages/virtualenv/run/plugin/activators.py", ".venv/lib/python3.8/site-packages/virtualenv/run/plugin/creators.py", ".venv/lib/python3.8/site-packages/virtualenv/run/plugin/base.py", ".venv/lib/python3.8/site-packages/virtualenv/discovery/discover.py", ".venv/lib/python3.8/site-packages/virtualenv/discovery/py_info.py", ".venv/lib/python3.8/site-packages/virtualenv/discovery/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/discovery/builtin.py", ".venv/lib/python3.8/site-packages/virtualenv/discovery/cached_py_info.py", ".venv/lib/python3.8/site-packages/virtualenv/discovery/py_spec.py", ".venv/lib/python3.8/site-packages/virtualenv/discovery/windows/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/discovery/windows/pep514.py", ".venv/lib/python3.8/site-packages/virtualenv/config/convert.py", ".venv/lib/python3.8/site-packages/virtualenv/config/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/config/ini.py", ".venv/lib/python3.8/site-packages/virtualenv/config/env_var.py", ".venv/lib/python3.8/site-packages/virtualenv/config/cli/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/config/cli/parser.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/seeder.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/via_app_data/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/via_app_data/via_app_data.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/via_app_data/pip_install/symlink.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/via_app_data/pip_install/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/via_app_data/pip_install/copy.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/via_app_data/pip_install/base.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/embed/base_embed.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/embed/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/embed/pip_invoke.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/embed/wheels/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/seed/embed/wheels/acquire.py", ".venv/lib/python3.8/site-packages/virtualenv/create/debug.py", ".venv/lib/python3.8/site-packages/virtualenv/create/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/create/describe.py", ".venv/lib/python3.8/site-packages/virtualenv/create/creator.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/venv.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/api.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/via_global_self_do.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/ref.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/builtin_way.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/cpython/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/cpython/common.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/pypy/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/pypy/pypy2.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/pypy/common.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/python2/__init__.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/python2/site.py", ".venv/lib/python3.8/site-packages/virtualenv/create/via_global_ref/builtin/python2/python2.py", ".venv/lib/python3.8/site-packages/yaml/nodes.py", ".venv/lib/python3.8/site-packages/yaml/error.py", ".venv/lib/python3.8/site-packages/yaml/representer.py", ".venv/lib/python3.8/site-packages/yaml/__init__.py", ".venv/lib/python3.8/site-packages/yaml/scanner.py", ".venv/lib/python3.8/site-packages/yaml/serializer.py", ".venv/lib/python3.8/site-packages/yaml/resolver.py", ".venv/lib/python3.8/site-packages/yaml/reader.py", ".venv/lib/python3.8/site-packages/yaml/events.py", ".venv/lib/python3.8/site-packages/yaml/emitter.py", ".venv/lib/python3.8/site-packages/yaml/composer.py", ".venv/lib/python3.8/site-packages/yaml/loader.py", ".venv/lib/python3.8/site-packages/yaml/parser.py", ".venv/lib/python3.8/site-packages/yaml/tokens.py", ".venv/lib/python3.8/site-packages/yaml/cyaml.py", ".venv/lib/python3.8/site-packages/yaml/constructor.py", ".venv/lib/python3.8/site-packages/yaml/dumper.py", ".venv/lib/python3.8/site-packages/pip/exceptions.py", ".venv/lib/python3.8/site-packages/pip/__init__.py", ".venv/lib/python3.8/site-packages/pip/status_codes.py", ".venv/lib/python3.8/site-packages/pip/cmdoptions.py", ".venv/lib/python3.8/site-packages/pip/__main__.py", ".venv/lib/python3.8/site-packages/pip/basecommand.py", ".venv/lib/python3.8/site-packages/pip/pep425tags.py", ".venv/lib/python3.8/site-packages/pip/wheel.py", ".venv/lib/python3.8/site-packages/pip/baseparser.py", ".venv/lib/python3.8/site-packages/pip/locations.py", ".venv/lib/python3.8/site-packages/pip/index.py", ".venv/lib/python3.8/site-packages/pip/download.py", ".venv/lib/python3.8/site-packages/pip/compat/__init__.py", ".venv/lib/python3.8/site-packages/pip/compat/dictconfig.py", ".venv/lib/python3.8/site-packages/pip/utils/__init__.py", ".venv/lib/python3.8/site-packages/pip/utils/setuptools_build.py", ".venv/lib/python3.8/site-packages/pip/utils/build.py", ".venv/lib/python3.8/site-packages/pip/utils/deprecation.py", ".venv/lib/python3.8/site-packages/pip/utils/ui.py", ".venv/lib/python3.8/site-packages/pip/utils/logging.py", ".venv/lib/python3.8/site-packages/pip/utils/outdated.py", ".venv/lib/python3.8/site-packages/pip/utils/glibc.py", ".venv/lib/python3.8/site-packages/pip/utils/encoding.py", ".venv/lib/python3.8/site-packages/pip/utils/appdirs.py", ".venv/lib/python3.8/site-packages/pip/utils/hashes.py", ".venv/lib/python3.8/site-packages/pip/utils/packaging.py", ".venv/lib/python3.8/site-packages/pip/utils/filesystem.py", ".venv/lib/python3.8/site-packages/pip/operations/__init__.py", ".venv/lib/python3.8/site-packages/pip/operations/freeze.py", ".venv/lib/python3.8/site-packages/pip/operations/check.py", ".venv/lib/python3.8/site-packages/pip/vcs/__init__.py", ".venv/lib/python3.8/site-packages/pip/vcs/git.py", ".venv/lib/python3.8/site-packages/pip/vcs/mercurial.py", ".venv/lib/python3.8/site-packages/pip/vcs/bazaar.py", ".venv/lib/python3.8/site-packages/pip/vcs/subversion.py", ".venv/lib/python3.8/site-packages/pip/commands/install.py", ".venv/lib/python3.8/site-packages/pip/commands/completion.py", ".venv/lib/python3.8/site-packages/pip/commands/__init__.py", ".venv/lib/python3.8/site-packages/pip/commands/help.py", ".venv/lib/python3.8/site-packages/pip/commands/hash.py", ".venv/lib/python3.8/site-packages/pip/commands/wheel.py", ".venv/lib/python3.8/site-packages/pip/commands/search.py", ".venv/lib/python3.8/site-packages/pip/commands/list.py", ".venv/lib/python3.8/site-packages/pip/commands/show.py", ".venv/lib/python3.8/site-packages/pip/commands/freeze.py", ".venv/lib/python3.8/site-packages/pip/commands/check.py", ".venv/lib/python3.8/site-packages/pip/commands/download.py", ".venv/lib/python3.8/site-packages/pip/commands/uninstall.py", ".venv/lib/python3.8/site-packages/pip/_vendor/__init__.py", ".venv/lib/python3.8/site-packages/pip/models/__init__.py", ".venv/lib/python3.8/site-packages/pip/models/index.py", ".venv/lib/python3.8/site-packages/pip/req/__init__.py", ".venv/lib/python3.8/site-packages/pip/req/req_set.py", ".venv/lib/python3.8/site-packages/pip/req/req_install.py", ".venv/lib/python3.8/site-packages/pip/req/req_file.py", ".venv/lib/python3.8/site-packages/pip/req/req_uninstall.py", ".venv/lib/python3.8/site-packages/mypyc/exceptions.py", ".venv/lib/python3.8/site-packages/mypyc/emitfunc.py", ".venv/lib/python3.8/site-packages/mypyc/prebuildvisitor.py", ".venv/lib/python3.8/site-packages/mypyc/ops_tuple.py", ".venv/lib/python3.8/site-packages/mypyc/ops.py", ".venv/lib/python3.8/site-packages/mypyc/__init__.py", ".venv/lib/python3.8/site-packages/mypyc/crash.py", ".venv/lib/python3.8/site-packages/mypyc/genops.py", ".venv/lib/python3.8/site-packages/mypyc/errors.py", ".venv/lib/python3.8/site-packages/mypyc/common.py", ".venv/lib/python3.8/site-packages/mypyc/sametype.py", ".venv/lib/python3.8/site-packages/mypyc/build.py", ".venv/lib/python3.8/site-packages/mypyc/emitwrapper.py", ".venv/lib/python3.8/site-packages/mypyc/subtype.py", ".venv/lib/python3.8/site-packages/mypyc/genops_for.py", ".venv/lib/python3.8/site-packages/mypyc/options.py", ".venv/lib/python3.8/site-packages/mypyc/ops_misc.py", ".venv/lib/python3.8/site-packages/mypyc/ops_dict.py", ".venv/lib/python3.8/site-packages/mypyc/ops_primitive.py", ".venv/lib/python3.8/site-packages/mypyc/emitclass.py", ".venv/lib/python3.8/site-packages/mypyc/ops_set.py", ".venv/lib/python3.8/site-packages/mypyc/analysis.py", ".venv/lib/python3.8/site-packages/mypyc/namegen.py", ".venv/lib/python3.8/site-packages/mypyc/ops_str.py", ".venv/lib/python3.8/site-packages/mypyc/emit.py", ".venv/lib/python3.8/site-packages/mypyc/refcount.py", ".venv/lib/python3.8/site-packages/mypyc/emitmodule.py", ".venv/lib/python3.8/site-packages/mypyc/uninit.py", ".venv/lib/python3.8/site-packages/mypyc/ops_exc.py", ".venv/lib/python3.8/site-packages/mypyc/rt_subtype.py", ".venv/lib/python3.8/site-packages/mypyc/ops_int.py", ".venv/lib/python3.8/site-packages/mypyc/ops_list.py", ".venv/lib/python3.8/site-packages/mypyc/cstring.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_external.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_refcount.py", ".venv/lib/python3.8/site-packages/mypyc/test/testutil.py", ".venv/lib/python3.8/site-packages/mypyc/test/__init__.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_namegen.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_emitfunc.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_genops.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_commandline.py", ".venv/lib/python3.8/site-packages/mypyc/test/config.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_exceptions.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_analysis.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_run.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_tuplename.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_emit.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_serialization.py", ".venv/lib/python3.8/site-packages/mypyc/test/test_emitwrapper.py", ".venv/lib/python3.8/site-packages/pyflakes/__init__.py", ".venv/lib/python3.8/site-packages/pyflakes/__main__.py", ".venv/lib/python3.8/site-packages/pyflakes/checker.py", ".venv/lib/python3.8/site-packages/pyflakes/api.py", ".venv/lib/python3.8/site-packages/pyflakes/reporter.py", ".venv/lib/python3.8/site-packages/pyflakes/messages.py", ".venv/lib/python3.8/site-packages/pyflakes/scripts/__init__.py", ".venv/lib/python3.8/site-packages/pyflakes/scripts/pyflakes.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_code_segment.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_builtin.py", ".venv/lib/python3.8/site-packages/pyflakes/test/__init__.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_api.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_return_with_arguments_inside_generator.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_checker.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_is_literal.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_doctests.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_other.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_imports.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_type_annotations.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_undefined_names.py", ".venv/lib/python3.8/site-packages/pyflakes/test/test_dict.py", ".venv/lib/python3.8/site-packages/pyflakes/test/harness.py", ".venv/lib/python3.8/site-packages/regex/__init__.py", ".venv/lib/python3.8/site-packages/regex/test_regex.py", ".venv/lib/python3.8/site-packages/regex/regex.py", ".venv/lib/python3.8/site-packages/regex/_regex_core.py", ".venv/lib/python3.8/site-packages/aspy/__init__.py", ".venv/lib/python3.8/site-packages/aspy/yaml/__init__.py", ".venv/lib/python3.8/site-packages/click/exceptions.py", ".venv/lib/python3.8/site-packages/click/__init__.py", ".venv/lib/python3.8/site-packages/click/testing.py", ".venv/lib/python3.8/site-packages/click/_termui_impl.py", ".venv/lib/python3.8/site-packages/click/formatting.py", ".venv/lib/python3.8/site-packages/click/_compat.py", ".venv/lib/python3.8/site-packages/click/core.py", ".venv/lib/python3.8/site-packages/click/termui.py", ".venv/lib/python3.8/site-packages/click/globals.py", ".venv/lib/python3.8/site-packages/click/parser.py", ".venv/lib/python3.8/site-packages/click/types.py", ".venv/lib/python3.8/site-packages/click/decorators.py", ".venv/lib/python3.8/site-packages/click/utils.py", ".venv/lib/python3.8/site-packages/click/_unicodefun.py", ".venv/lib/python3.8/site-packages/click/_winconsole.py", ".venv/lib/python3.8/site-packages/click/_bashcomplete.py", ".venv/lib/python3.8/site-packages/click/_textwrap.py", ".venv/lib/python3.8/site-packages/distlib/scripts.py", ".venv/lib/python3.8/site-packages/distlib/__init__.py", ".venv/lib/python3.8/site-packages/distlib/compat.py", ".venv/lib/python3.8/site-packages/distlib/metadata.py", ".venv/lib/python3.8/site-packages/distlib/wheel.py", ".venv/lib/python3.8/site-packages/distlib/index.py", ".venv/lib/python3.8/site-packages/distlib/resources.py", ".venv/lib/python3.8/site-packages/distlib/database.py", ".venv/lib/python3.8/site-packages/distlib/manifest.py", ".venv/lib/python3.8/site-packages/distlib/util.py", ".venv/lib/python3.8/site-packages/distlib/locators.py", ".venv/lib/python3.8/site-packages/distlib/version.py", ".venv/lib/python3.8/site-packages/distlib/markers.py", ".venv/lib/python3.8/site-packages/distlib/_backport/__init__.py", ".venv/lib/python3.8/site-packages/distlib/_backport/shutil.py", ".venv/lib/python3.8/site-packages/distlib/_backport/misc.py", ".venv/lib/python3.8/site-packages/distlib/_backport/sysconfig.py", ".venv/lib/python3.8/site-packages/distlib/_backport/tarfile.py", ".venv/lib/python3.8/site-packages/pathspec/__init__.py", ".venv/lib/python3.8/site-packages/pathspec/pattern.py", ".venv/lib/python3.8/site-packages/pathspec/compat.py", ".venv/lib/python3.8/site-packages/pathspec/pathspec.py", ".venv/lib/python3.8/site-packages/pathspec/util.py", ".venv/lib/python3.8/site-packages/pathspec/patterns/__init__.py", ".venv/lib/python3.8/site-packages/pathspec/patterns/gitwildmatch.py", ".venv/lib/python3.8/site-packages/pathspec/tests/__init__.py", ".venv/lib/python3.8/site-packages/pathspec/tests/test_util.py", ".venv/lib/python3.8/site-packages/pathspec/tests/test_pathspec.py", ".venv/lib/python3.8/site-packages/pathspec/tests/test_gitwildmatch.py", ".venv/lib/python3.8/site-packages/setuptools/__init__.py", ".venv/lib/python3.8/site-packages/setuptools/site-patch.py", ".venv/lib/python3.8/site-packages/setuptools/py33compat.py", ".venv/lib/python3.8/site-packages/setuptools/py31compat.py", ".venv/lib/python3.8/site-packages/setuptools/archive_util.py", ".venv/lib/python3.8/site-packages/setuptools/launch.py", ".venv/lib/python3.8/site-packages/setuptools/glob.py", ".venv/lib/python3.8/site-packages/setuptools/namespaces.py", ".venv/lib/python3.8/site-packages/setuptools/dep_util.py", ".venv/lib/python3.8/site-packages/setuptools/config.py", ".venv/lib/python3.8/site-packages/setuptools/msvc.py", ".venv/lib/python3.8/site-packages/setuptools/windows_support.py", ".venv/lib/python3.8/site-packages/setuptools/pep425tags.py", ".venv/lib/python3.8/site-packages/setuptools/wheel.py", ".venv/lib/python3.8/site-packages/setuptools/py36compat.py", ".venv/lib/python3.8/site-packages/setuptools/glibc.py", ".venv/lib/python3.8/site-packages/setuptools/extension.py", ".venv/lib/python3.8/site-packages/setuptools/depends.py", ".venv/lib/python3.8/site-packages/setuptools/ssl_support.py", ".venv/lib/python3.8/site-packages/setuptools/unicode_utils.py", ".venv/lib/python3.8/site-packages/setuptools/sandbox.py", ".venv/lib/python3.8/site-packages/setuptools/build_meta.py", ".venv/lib/python3.8/site-packages/setuptools/dist.py", ".venv/lib/python3.8/site-packages/setuptools/package_index.py", ".venv/lib/python3.8/site-packages/setuptools/py27compat.py", ".venv/lib/python3.8/site-packages/setuptools/monkey.py", ".venv/lib/python3.8/site-packages/setuptools/version.py", ".venv/lib/python3.8/site-packages/setuptools/lib2to3_ex.py", ".venv/lib/python3.8/site-packages/setuptools/command/install.py", ".venv/lib/python3.8/site-packages/setuptools/command/install_lib.py", ".venv/lib/python3.8/site-packages/setuptools/command/register.py", ".venv/lib/python3.8/site-packages/setuptools/command/install_egg_info.py", ".venv/lib/python3.8/site-packages/setuptools/command/upload.py", ".venv/lib/python3.8/site-packages/setuptools/command/__init__.py", ".venv/lib/python3.8/site-packages/setuptools/command/build_py.py", ".venv/lib/python3.8/site-packages/setuptools/command/bdist_wininst.py", ".venv/lib/python3.8/site-packages/setuptools/command/test.py", ".venv/lib/python3.8/site-packages/setuptools/command/alias.py", ".venv/lib/python3.8/site-packages/setuptools/command/easy_install.py", ".venv/lib/python3.8/site-packages/setuptools/command/upload_docs.py", ".venv/lib/python3.8/site-packages/setuptools/command/egg_info.py", ".venv/lib/python3.8/site-packages/setuptools/command/build_clib.py", ".venv/lib/python3.8/site-packages/setuptools/command/install_scripts.py", ".venv/lib/python3.8/site-packages/setuptools/command/rotate.py", ".venv/lib/python3.8/site-packages/setuptools/command/saveopts.py", ".venv/lib/python3.8/site-packages/setuptools/command/py36compat.py", ".venv/lib/python3.8/site-packages/setuptools/command/sdist.py", ".venv/lib/python3.8/site-packages/setuptools/command/bdist_egg.py", ".venv/lib/python3.8/site-packages/setuptools/command/dist_info.py", ".venv/lib/python3.8/site-packages/setuptools/command/setopt.py", ".venv/lib/python3.8/site-packages/setuptools/command/develop.py", ".venv/lib/python3.8/site-packages/setuptools/command/bdist_rpm.py", ".venv/lib/python3.8/site-packages/setuptools/command/build_ext.py", ".venv/lib/python3.8/site-packages/setuptools/extern/__init__.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/__init__.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/pyparsing.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/six.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/packaging/__init__.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/packaging/_compat.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/packaging/__about__.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/packaging/_structures.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/packaging/requirements.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/packaging/utils.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/packaging/specifiers.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/packaging/version.py", ".venv/lib/python3.8/site-packages/setuptools/_vendor/packaging/markers.py", ".venv/lib/python3.8/site-packages/flake8/exceptions.py", ".venv/lib/python3.8/site-packages/flake8/__init__.py", ".venv/lib/python3.8/site-packages/flake8/style_guide.py", ".venv/lib/python3.8/site-packages/flake8/__main__.py", ".venv/lib/python3.8/site-packages/flake8/checker.py", ".venv/lib/python3.8/site-packages/flake8/processor.py", ".venv/lib/python3.8/site-packages/flake8/statistics.py", ".venv/lib/python3.8/site-packages/flake8/utils.py", ".venv/lib/python3.8/site-packages/flake8/defaults.py", ".venv/lib/python3.8/site-packages/flake8/main/debug.py", ".venv/lib/python3.8/site-packages/flake8/main/setuptools_command.py", ".venv/lib/python3.8/site-packages/flake8/main/__init__.py", ".venv/lib/python3.8/site-packages/flake8/main/git.py", ".venv/lib/python3.8/site-packages/flake8/main/options.py", ".venv/lib/python3.8/site-packages/flake8/main/application.py", ".venv/lib/python3.8/site-packages/flake8/main/mercurial.py", ".venv/lib/python3.8/site-packages/flake8/main/cli.py", ".venv/lib/python3.8/site-packages/flake8/main/vcs.py", ".venv/lib/python3.8/site-packages/flake8/options/__init__.py", ".venv/lib/python3.8/site-packages/flake8/options/config.py", ".venv/lib/python3.8/site-packages/flake8/options/manager.py", ".venv/lib/python3.8/site-packages/flake8/options/aggregator.py", ".venv/lib/python3.8/site-packages/flake8/formatting/__init__.py", ".venv/lib/python3.8/site-packages/flake8/formatting/base.py", ".venv/lib/python3.8/site-packages/flake8/formatting/default.py", ".venv/lib/python3.8/site-packages/flake8/api/__init__.py", ".venv/lib/python3.8/site-packages/flake8/api/legacy.py", ".venv/lib/python3.8/site-packages/flake8/plugins/__init__.py", ".venv/lib/python3.8/site-packages/flake8/plugins/pyflakes.py", ".venv/lib/python3.8/site-packages/flake8/plugins/manager.py", ".venv/lib/python3.8/site-packages/pydocstyle/__init__.py", ".venv/lib/python3.8/site-packages/pydocstyle/wordlists.py", ".venv/lib/python3.8/site-packages/pydocstyle/__main__.py", ".venv/lib/python3.8/site-packages/pydocstyle/checker.py", ".venv/lib/python3.8/site-packages/pydocstyle/config.py", ".venv/lib/python3.8/site-packages/pydocstyle/parser.py", ".venv/lib/python3.8/site-packages/pydocstyle/violations.py", ".venv/lib/python3.8/site-packages/pydocstyle/cli.py", ".venv/lib/python3.8/site-packages/pydocstyle/utils.py", ".venv/lib/python3.8/site-packages/identify/__init__.py", ".venv/lib/python3.8/site-packages/identify/identify.py", ".venv/lib/python3.8/site-packages/identify/cli.py", ".venv/lib/python3.8/site-packages/identify/interpreters.py", ".venv/lib/python3.8/site-packages/identify/extensions.py", ".venv/lib/python3.8/site-packages/identify/vendor/__init__.py", ".venv/lib/python3.8/site-packages/identify/vendor/licenses.py", ".venv/lib/python3.8/site-packages/attr/exceptions.py", ".venv/lib/python3.8/site-packages/attr/validators.py", ".venv/lib/python3.8/site-packages/attr/__init__.py", ".venv/lib/python3.8/site-packages/attr/_version_info.py", ".venv/lib/python3.8/site-packages/attr/filters.py", ".venv/lib/python3.8/site-packages/attr/_compat.py", ".venv/lib/python3.8/site-packages/attr/converters.py", ".venv/lib/python3.8/site-packages/attr/_funcs.py", ".venv/lib/python3.8/site-packages/attr/_config.py", ".venv/lib/python3.8/site-packages/attr/_make.py", ".venv/lib/python3.8/site-packages/blib2to3/__init__.py", ".venv/lib/python3.8/site-packages/blib2to3/pytree.py", ".venv/lib/python3.8/site-packages/blib2to3/pygram.py", ".venv/lib/python3.8/site-packages/blib2to3/pgen2/__init__.py", ".venv/lib/python3.8/site-packages/blib2to3/pgen2/literals.py", ".venv/lib/python3.8/site-packages/blib2to3/pgen2/grammar.py", ".venv/lib/python3.8/site-packages/blib2to3/pgen2/conv.py", ".venv/lib/python3.8/site-packages/blib2to3/pgen2/pgen.py", ".venv/lib/python3.8/site-packages/blib2to3/pgen2/driver.py", ".venv/lib/python3.8/site-packages/blib2to3/pgen2/parse.py", ".venv/lib/python3.8/site-packages/blib2to3/pgen2/tokenize.py", ".venv/lib/python3.8/site-packages/blib2to3/pgen2/token.py", ".venv/lib/python3.8/site-packages/snowballstemmer/swedish_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/basque_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/norwegian_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/__init__.py", ".venv/lib/python3.8/site-packages/snowballstemmer/english_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/finnish_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/portuguese_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/lithuanian_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/danish_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/romanian_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/catalan_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/greek_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/german_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/dutch_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/spanish_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/hungarian_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/italian_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/russian_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/french_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/turkish_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/irish_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/porter_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/indonesian_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/tamil_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/nepali_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/basestemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/arabic_stemmer.py", ".venv/lib/python3.8/site-packages/snowballstemmer/among.py", ".venv/lib/python3.8/site-packages/snowballstemmer/hindi_stemmer.py", ".venv/lib/python3.8/site-packages/toml/__init__.py", ".venv/lib/python3.8/site-packages/toml/encoder.py", ".venv/lib/python3.8/site-packages/toml/decoder.py", ".venv/lib/python3.8/site-packages/toml/tz.py", ".venv/lib/python3.8/site-packages/toml/ordered.py", ".venv/lib/python3.8/site-packages/isort/pie_slice.py", ".venv/lib/python3.8/site-packages/isort/__init__.py", ".venv/lib/python3.8/site-packages/isort/pylama_isort.py", ".venv/lib/python3.8/site-packages/isort/__main__.py", ".venv/lib/python3.8/site-packages/isort/hooks.py", ".venv/lib/python3.8/site-packages/isort/main.py", ".venv/lib/python3.8/site-packages/isort/settings.py", ".venv/lib/python3.8/site-packages/isort/isort.py", ".venv/lib/python3.8/site-packages/isort/utils.py", ".venv/lib/python3.8/site-packages/isort/finders.py", ".venv/lib/python3.8/site-packages/isort/natural.py", ".venv/lib/python3.8/site-packages/typed_ast/__init__.py", ".venv/lib/python3.8/site-packages/typed_ast/ast3.py", ".venv/lib/python3.8/site-packages/typed_ast/conversions.py", ".venv/lib/python3.8/site-packages/typed_ast/ast27.py", ".venv/lib/python3.8/site-packages/typed_ast/tests/test_basics.py", "tests/test_cell_handling.py", "tests/_helpers.py" ] } ================================================ FILE: .codecov.yml ================================================ ignore: - "lib/client" # ignore folders and all its contents ================================================ FILE: .flake8 ================================================ [flake8] ignore = # whitespace before ':' E203, # missing docstrings D100,D101,D102,D103,D104,D105,D106,D107, D203, W503 exclude = .git, .venv, __pycache__, build, dist, docs/source/conf.py, lib, old max-line-length= 88 ================================================ FILE: .github/workflows/ci.yml ================================================ name: CI on: [push] jobs: build: runs-on: ubuntu-latest strategy: matrix: python-version: [3.8] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install flake8 flake8-blind-except flake8-comprehensions flake8-docstrings flake8-eradicate - name: Lint with flake8 run: | flake8 . ================================================ FILE: .gitignore ================================================ # Python Stuff *.pyc .venv/ # Sublime Stuff *.sublime-project *.sublime-workspace ================================================ FILE: .pre-commit-config.yaml ================================================ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks default_stages: [push] default_language_version: python: python3.8 exclude: "lib/client" repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 hooks: - id: check-added-large-files - id: check-ast - id: check-case-conflict - id: check-merge-conflict - id: check-toml - id: check-yaml - repo: local hooks: - id: black name: black entry: black language: system types: [python] - id: flake8 name: flake8 entry: flake8 language: system types: [python] args: [--config, pyproject.toml] - id: isort name: isort entry: isort language: system types: [python] ================================================ FILE: .python-version ================================================ 3.8 ================================================ FILE: CONTRIBUTORS.md ================================================ This package would not exist without the original work done by NEGORO Tetsuya (https://github.com/ngr-t). ================================================ FILE: Default.sublime-commands ================================================ [ { "caption": "Helium: Connect Kernel", "command": "helium_connect_kernel" }, { "caption": "Helium: Start Kernel", "command": "helium_start_kernel" }, { "caption": "Helium: Shutdown Kernel", "command": "helium_shutdown_kernel" }, { "caption": "Helium: Restart Kernel", "command": "helium_restart_kernel" }, { "caption": "Helium: Interrupt Kernel", "command": "helium_interrupt_kernel" }, { "caption": "Helium: Execute Block", "command": "helium_execute_block" }, { "caption": "Helium: Execute Cell", "command": "helium_execute_cell" }, { "caption": "Helium: Execute Cell and Move", "command": "helium_execute_cell", "args": { "move_cursor": "True" } }, { "caption": "Helium: List Kernels", "command": "helium_list_kernels" }, { "caption": "Helium: Get Object Inspection", "command": "helium_get_object_inspection" }, { "caption": "Helium: Clear All Cells", "command": "helium_clear_all_cells" }, { "caption": "Helium: Settings", "command": "edit_settings", "args": { "base_file": "${packages}/Helium/Helium.sublime-settings", "default": "{\n \"connections\": [\n $0\n ]\n}" } } ] ================================================ FILE: Default.sublime-keymap ================================================ [ {"keys": ["ctrl+alt+enter"], "command": "helium_execute_cell", "args": {"move_cursor": "True"}} ] ================================================ FILE: Helium.sublime-settings ================================================ { "cell_delimiter_pattern": "^(#\\s?%%)|(# )\\s*$", // Regex pattern to find code cell blocks delimiters // The default value hits the patterns below. // #%% // # %% // # // Whether use Helium' autocomplete powered by Jupyter kernel. "complete": true, // Timeout to get completion (in seconds). "complete_timeout": 0.5, // Set to true to show output in current view, like Jupyter "inline_output": false, // Set true to show the executed code in the output "output_code": false, // "image_size" controls the size of the image. It can take one three values: // "original": The image is displayed with its original size. // "resize_to_window": The image is rescaled to match the size of the window. // "optimal": The image is either resized or has its original size, // whichever is smaller. // The default is "optimal". "image_size": "optimal", // If you use kernels located in the directory other than default search path // (see http://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs), // set their path as below. // "jupyter_path": "/path/to/kernel" } ================================================ FILE: Helium.sublime-syntax ================================================ %YAML 1.2 --- name: Helium IPython scope: source.helium.output variables: command_start: "In\\[\\d+\\]:" error_start: "Error\\[[^\\]]+\\]:" # Not sure what should go inside the brackets of Error streams: "\\(stdout|display data|stderr\\):" contexts: main: - match: |- \b({{command_start}}) comment: Cell contents formatted in python name: cell.content embed: scope:source.python escape: (?={{streams}}|{{command_start}}|{{error_start}}) # Outputs are formatted captures: 1: markup.bold - match: |- ({{streams}}) name: streams comment: Streams' names in bold captures: 1: markup.italic ================================================ FILE: LICENSE.TXT ================================================ Helium package is licensed under GNU GENERAL PUBLIC LICENSE Version 2. YEAR: 2016-2018 COPYRIGHT HOLDER: NEGORO Tetsuya This package includes libraries below. - ipython_genutils - jupyter_core - traitlets - decorator This package includes a modified version of library below. - jupyter_client ipython_genutils is licensed under the terms of the Modified BSD License (also known as New or Revised or 3-Clause BSD), as follows: - Copyright (c) 2001-, IPython Development Team jupyter_core is licensed under the terms of the Modified BSD License (also known as New or Revised or 3-Clause BSD). - Copyright (c) 2015-, Jupyter Development Team jupyter_client is licensed under the terms of the Modified BSD License (also known as New or Revised or 3-Clause BSD), as follows: - Copyright (c) 2001-2015, IPython Development Team - Copyright (c) 2015-, Jupyter Development Team traitlets is licensed under the terms of the Modified BSD License (also known as New or Revised or 3-Clause BSD), as follows: - Copyright (c) 2001-, IPython Development Team decorator is licensed under BSD 2-Clause "Simplified" License. - Copyright (c) 2005-2018, Michele Simionato BSD 2-Clause License ---- Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in bytecode form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. the Modified BSD License: ---- Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of the IPython Development Team nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: Main.sublime-menu ================================================ [ { "caption": "Preferences", "mnemonic": "n", "id": "preferences", "children": [ { "caption": "Package Settings", "mnemonic": "P", "id": "package-settings", "children": [ { "caption": "Helium", "children": [ { "command": "open_file", "args": { "file": "${packages}/Helium/Helium.sublime-settings" }, "caption": "Settings – Default" }, { "command": "open_file", "args": { "file": "${packages}/User/Helium.sublime-settings" }, "caption": "Settings – User" }, ] } ] } ] } ] ================================================ FILE: README.md ================================================ Helium package for Sublime Text === Helium is a package for Sublime Text, which provides in-editor code execution and autocomplete in interaction with Jupyter kernels. The concept of an editor extension communicating Jupyter kernels is inspired by @nteract's splendid Atom package [Hydrogen](https://github.com/nteract/Hydrogen). I want something like it in Sublime Text, too. Any feedback is highly welcome. I hope this package will help your life with ST3! ![Introduction image](raw/images/README/animated.gif) ## Installation Now this package is under the package control channel! You can install it with Package Control plugin, run `Package Control: Install Package`, then choose `Helium` from the package list. ## Usage ### Connecting to Jupyter kernels #### 1. The most basic way, start a kernelspec installed locally, as a subprocess of ST3 (the process stops when Sublime stops) 1. Run `Helium: connect kernel` command. 2. Choose `New kernel`. 3. Choose the kernelspec you want to run. #### 2. Connect to the kernel already runnning and connected to Helium 1. Run `Helium: connect kernel` command. 2. Choose the kernel you want to connect. #### 3. Connect to a kernel already running under some other Jupyter app (such as Notebook) 1. Get connection info of the kernel. The way to get connection info differ among kernels, see the doc of each kernel (in ipython kernel, you can get it by `%connect_info` magic.) 2. Run `Helium: connect kernel` command. 3. Choose `New kernel`. 4. Choose `(Enter connection info)`. 5. Enter the connection info (Helium accepts a path or connection info itself). #### Using Python kernel installed via Conda Python kernel installed via Conda is not found by Jupyter by default. You should add the path to kernel into the `jupyter_path` entry of the config file. ### Execution Execute code by `Helium: Execute Block` (whose command name is `helium_execute_block`). #### Code cell Regions surrounded by `# %%` or `# ` (you can configure it in `cell_delimiter_pattern` option item) are considered as "code cells". You can execute a region by `Helium: Execute cell` (`helium_execute_cell`) or `Helium: Execute Cell and Move` command. Each cell has a clickable "Run Cell" phantom that appears next to the cell markers to run the cell. ### Object inspection Get Object Inspection by `Helium: Get Object Inspection` (whose command name is `helium_get_object_inspection`). ### Autocomplete You should be able to get autocomplete from the kernel from the time you connected. If you don't want autocomplete, set `"complete"` as `false` in setting file. ### Other kernel manipulations You can restart, shutdown, and interrupt process via `Helium: Restart Kernel`, `Helium: Shutdown Kernel`, `Helium: Interrupt Kernel` commands. You can also run these command as a submenu of `Helium: List Kernels` command. ## Motivation of development ### Why using Jupyter? We can execute code, retrieve results including images, get completions and object inspections by the Jupyter protocol regardless of the interpreter implementation of languages if it has Jupyter kernel. If we try to do that by directly running interpreters there should be several interpreter-specific problems, but we can entrust the kernel maintainers on language-specific problems by using Jupyter. ### Why not using Jupyter Notebook? I admit Jupyter Notebook is a powerful tool for instantly sharing small analysis work, exploring data or APIs, or making executable tutorials. Yes, I often use it, too. However, in my opinion, it is not suited for projects with large code bases. I want to jumpt across files instantly, make modules organized (not saved as `.ipynb`s), kick scripts with various parameters, and make project code more reusable and reproducible... but still I want to edit them with interactive feedback. ================================================ FILE: dependencies.json ================================================ { "windows": { "<4000": [ "dateutil", "enum", "python-six", "pyzmq" ], ">=4000": [ "dateutil", "python-six", "pyzmq", "tornado" ] }, "osx": { "<4000": [ "dateutil", "enum", "pexpect", "ptyprocess", "python-six", "pyzmq" ], ">=4000": [ "dateutil", "pexpect", "ptyprocess", "python-six", "pyzmq", "tornado" ] }, "linux": { "<4000": [ "dateutil", "enum", "pexpect", "ptyprocess", "python-six", "pyzmq" ], ">=4000": [ "dateutil", "pexpect", "ptyprocess", "python-six", "pyzmq", "tornado" ] } } ================================================ FILE: helium.py ================================================ """Helium package for Sublime Text 3. The package provides code execution and completion in interaction with Jupyter. Copyright (c) 2016-2018, NEGORO Tetsuya (https://github.com/ngr-t) """ import json import os import re import uuid from functools import partial from logging import INFO, StreamHandler, getLogger from os.path import expanduser import sublime from sublime_plugin import EventListener, TextCommand, ViewEventListener from .lib.kernel import MAX_PHANTOMS, KernelConnection from .lib.utils import add_path, chain_callbacks, get_cell with add_path(os.path.join(os.path.dirname(__file__), "lib/client")): # Import jupyter_client related functions and classes. # Temporarily insert `lib` into sys.path not to affect other packages. from jupyter_client.connect import tunnel_to_kernel from jupyter_client.kernelspec import find_kernel_specs from jupyter_client.manager import KernelManager # Logger setting HELIUM_LOGGER = getLogger(__name__) HANDLER = StreamHandler() HANDLER.setLevel(INFO) if len(HELIUM_LOGGER.handlers) == 0: HELIUM_LOGGER.setLevel(INFO) HELIUM_LOGGER.addHandler(HANDLER) # Regex patterns to extract code lines. INDENT_PATTERN = re.compile(r"^([ \t]*)") # TODO: move CSS into separate file RUN_CELL_PHANTOM = """ Run cell """ RUN_CELL_PHANTOM_ID = "HeliumRunCell" ORG_JUPYTER_PATH = os.environ.get("JUPYTER_PATH") def _refresh_jupyter_path(): additional_jupyter_path = sublime.load_settings("Helium.sublime-settings").get( "jupyter_path" ) os.environ["JUPYTER_PATH"] = ":".join( [ path for path in [ORG_JUPYTER_PATH, additional_jupyter_path] if path is not None ] ) class ViewManager(object): """Manage the relation of views and kernels.""" view_kernel_table = {} def __new__(cls, *args, **kwargs): if not hasattr(cls, "__instance__"): cls.__instance__ = super(ViewManager, object).__new__(cls, *args, **kwargs) return cls.__instance__ def __init__(self, *args, **kwargs): pass @classmethod def connect_kernel(cls, buffer_id, lang, kernel_id): """Connect view to kernel.""" kernel = HeliumKernelManager.get_kernel(kernel_id) cls.view_kernel_table[buffer_id] = kernel inline_output = sublime.load_settings("Helium.sublime-settings").get( "inline_output" ) if not inline_output: kernel.activate_view() @classmethod def remove_view(cls, buffer_id): """Remove view from manager.""" if buffer_id in cls.view_kernel_table: del cls.view_kernel_table[buffer_id] @classmethod def get_kernel_for_view(cls, buffer_id) -> KernelConnection: """Get Kernel instance corresponding to the buffer_id.""" return cls.view_kernel_table[buffer_id] class HeliumKernelManager(object): """Manage Jupyter kernels.""" # The key is a tuple consisted of the name of kernelspec and kernel ID, # the value is a KernelConnection instance correspond to it. kernels = {} logger = HELIUM_LOGGER def __new__(cls, *args, **kwargs): """Make this class a singleton.""" if not hasattr(cls, "__instance__"): cls.__instance__ = super(HeliumKernelManager, object).__new__( cls, *args, **kwargs ) return cls.__instance__ @classmethod def list_kernelspecs(cls): """Get the kernelspecs.""" _refresh_jupyter_path() return find_kernel_specs() @classmethod def list_kernels(cls): """Get the list of kernels.""" return [ {"name": cls.get_kernel(kernel_id).lang, "id": kernel_id} for kernel_id in cls.kernels.keys() if cls.get_kernel(kernel_id).is_alive() ] @classmethod def list_kernel_reprs(cls): """Get the list of representations of kernels.""" def get_repr(kernel): key = (kernel["name"], kernel["id"]) try: return cls.kernels[key].repr except KeyError: return "[{lang}] {kernel_id}".format( lang=kernel["name"], kernel_id=kernel["id"] ) return list(map(get_repr, cls.list_kernels())) @classmethod def get_kernel(cls, kernel_id, connection_name=None): """Get KernelConnection object.""" return cls.kernels[kernel_id] @classmethod def start_kernel( cls, kernelspec_name=None, connection_info=None, connection_name=None, cwd=None ): """Start kernel and return a `Kernel` instance.""" kernel_id = uuid.uuid4() if not cwd: cwd = expanduser("~") if kernelspec_name: kernel_manager = KernelManager(kernel_name=kernelspec_name) kernel_manager.start_kernel(cwd=cwd) elif connection_info: kernel_manager = KernelManager() kernel_manager.load_connection_info(connection_info) # `KernelManager.kernel_name` is not automatically set from connection info. kernel_manager.kernel_name = connection_info.get("kernel_name", "") else: raise Exception( "You must specify any of {`kernelspec_name`, `connection_info`}." ) kernel = KernelConnection( kernel_id, kernel_manager, cls, connection_name=connection_name, logger=cls.logger, ) cls.kernels[kernel_id] = kernel return kernel @classmethod def shutdown_kernel(cls, kernel_id): """Shutdown kernel.""" cls.get_kernel(kernel_id).shutdown_kernel() @classmethod def restart_kernel(cls, kernel_id): """Restart kernel.""" cls.get_kernel(kernel_id).restart_kernel() @classmethod def interrupt_kernel(cls, kernel_id): """Interrupt kernel.""" cls.get_kernel(kernel_id).interrupt_kernel() @chain_callbacks def _enter_connection_info(window, continue_cb): connection_info_str = yield partial( window.show_input_panel, "Enter connection info or the path to connection file.", "", on_change=None, on_cancel=None, ) try: continue_cb(json.loads(connection_info_str)) except ValueError: try: with open(connection_info_str) as infs: continue_cb(json.loads(infs.read())) except FileNotFoundError: sublime.message_dialog( "The input string was neither a valid JSON string nor a file path." ) raise @chain_callbacks def _start_kernel(window, view, continue_cb=lambda: None, *, logger=HELIUM_LOGGER): kernelspecs = HeliumKernelManager.list_kernelspecs() menu_items = list(kernelspecs.keys()) + [ "(Enter connection info)", ] index = yield partial(window.show_quick_panel, menu_items) cwd = None if view: cwd = expanduser("~") if view and view.file_name(): cwd = os.path.dirname(view.file_name()) if index == -1: return elif index == len(kernelspecs): # Create a kernel from connection info. connection_info = yield partial(_enter_connection_info, window) connection_name = yield partial( window.show_input_panel, "connection name", "", on_change=None, on_cancel=None, ) if connection_name == "": connection_name = None kernel = HeliumKernelManager.start_kernel( connection_info=connection_info, connection_name=connection_name, cwd=cwd ) elif index == len(kernelspecs) + 1: # Create a kernel with SSH tunneling. servers = sublime.load_settings("Helium.sublime-settings").get("ssh_servers") if not servers: sublime.message_dialog( "Please set `ssh_servers` item of the config file via `Helium: ` " "to connect SSH servers." ) return menu_items = list(servers.keys()) server_index = yield partial(window.show_quick_panel, menu_items) server = servers[menu_items[server_index]] connection_info = yield partial(_enter_connection_info, window) shell_port, iopub_port, stdin_port, hb_port = tunnel_to_kernel( connection_info, server["server"], server.get("key", None) ) new_ports = { "shell_port": shell_port, "iopub_port": iopub_port, "stdin_port": stdin_port, "hb_port": hb_port, } connection_info.update(new_ports) connection_name = yield partial( window.show_input_panel, "connection name", "", on_change=None, on_cancel=None, ) kernel = HeliumKernelManager.start_kernel( connection_info=connection_info, connection_name=connection_name ) else: # Create a kernel from the kernelspec name. selected_kernelspec = menu_items[index] connection_name = yield partial( window.show_input_panel, "connection name", "", on_change=None, on_cancel=None, ) if connection_name == "": connection_name = None kernel = HeliumKernelManager.start_kernel( kernelspec_name=selected_kernelspec, connection_name=connection_name, cwd=cwd, ) ViewManager.connect_kernel(view.buffer_id(), kernel.lang, kernel.kernel_id) if view.file_name(): view_name = view.file_name() else: view_name = view.name() log_info_msg = ( "Connected view '{view_name}(id: {buffer_id})'" "to kernel {kernel_id}." ).format( view_name=view_name, buffer_id=view.buffer_id(), kernel_id=kernel.kernel_id ) logger.info(log_info_msg) continue_cb() class HeliumStartKernel(TextCommand): """Start a kernel and connect view to it.""" def run(self, edit, *, logger=HELIUM_LOGGER): """Command definition.""" _start_kernel(sublime.active_window(), self.view) self.view.run_command("helium_clear_all_cells") # TODO: Make this an enum class ListKernelsSubcommands(object): connect = "Connect" rename = "Rename" interrupt = "Interrupt" restart = "Restart" shutdown = "Shutdown" back = "Back to the kernel list" @chain_callbacks def _list_kernels(window, view, *, logger=HELIUM_LOGGER): sc = ListKernelsSubcommands selected_kernel = yield partial( _show_kernel_selection_menu, window, view, add_new=True ) subcommands = [ sc.connect, sc.rename, sc.interrupt, sc.restart, sc.shutdown, sc.back, ] try: if ( selected_kernel["id"] == ViewManager.get_kernel_for_view(view.buffer_id()).kernel_id ): subcommands = [sc.rename, sc.interrupt, sc.restart, sc.shutdown, sc.back] except KeyError: # No kernel is connected # `subcommands` includes "Connect" pass index = yield partial(window.show_quick_panel, subcommands) if index == -1: return elif subcommands[index] is sc.connect: # Connect ViewManager.connect_kernel( view.buffer_id(), selected_kernel["name"], selected_kernel["id"] ) if view.file_name(): view_name = view.file_name() else: view_name = view.name() log_info_msg = ( "Connected view '{view_name}(id: {buffer_id})'" "to kernel {kernel_id}." ).format( view_name=view_name, buffer_id=view.buffer_id(), kernel_id=selected_kernel["id"], ) logger.info(log_info_msg) elif subcommands[index] is sc.rename: # Rename conn = HeliumKernelManager.get_kernel(selected_kernel["id"]) curr_name = conn.connection_name if conn.connection_name is not None else "" new_name = yield partial( window.show_input_panel, "New name", curr_name, on_change=None, on_cancel=None, ) conn.connection_name = new_name elif subcommands[index] is sc.interrupt: # Interrupt HeliumKernelManager.interrupt_kernel(selected_kernel["id"]) log_info_msg = ("Interrupted kernel {kernel_id}.").format( kernel_id=selected_kernel["id"] ) logger.info(log_info_msg) elif subcommands[index] is sc.restart: # Restart HeliumKernelManager.restart_kernel(selected_kernel["id"]) log_info_msg = ("Restarted kernel {kernel_id}.").format( kernel_id=selected_kernel["id"] ) logger.info(log_info_msg) elif subcommands[index] is sc.shutdown: # Shutdown HeliumKernelManager.shutdown_kernel(selected_kernel["id"]) log_info_msg = ("Shutdown kernel {kernel_id}.").format( kernel_id=selected_kernel["id"] ) logger.info(log_info_msg) elif subcommands[index] is sc.back: # Back to the kernel list yield _list_kernels(window, view) sublime.set_timeout_async(lambda: StatusBar(view), 0) class HeliumListKernels(TextCommand): """Command that shows the list of kernels and do some action for chosen kernels.""" def run(self, edit, *, logger=HELIUM_LOGGER): _list_kernels(sublime.active_window(), self.view) @chain_callbacks def _connect_kernel(window, view, *, continue_cb=lambda: None, logger=HELIUM_LOGGER): kernel_list = HeliumKernelManager.list_kernels() menu_items = [ "[{lang}] {kernel_id}".format(lang=kernel["name"], kernel_id=kernel["id"]) for kernel in kernel_list ] menu_items += ["New kernel"] index = yield partial(window.show_quick_panel, menu_items) if index == -1: return elif index == len(kernel_list): yield partial(_start_kernel, window, view) else: selected_kernel = kernel_list[index] ViewManager.connect_kernel( view.buffer_id(), selected_kernel["name"], selected_kernel["id"] ) if view.file_name(): view_name = view.file_name() else: view_name = view.name() update_run_cell_phantoms(view) log_info_msg = ( "Connected view '{view_name}(id: {buffer_id})' to kernel {kernel_id}." ).format( view_name=view_name, buffer_id=view.buffer_id(), kernel_id=selected_kernel["id"], ) logger.info(log_info_msg) sublime.set_timeout_async(lambda: StatusBar(view), 0) continue_cb() class HeliumConnectKernel(TextCommand): """Connect to Jupyter kernel.""" def run(self, edit, *, logger=HELIUM_LOGGER): """Command definition.""" _connect_kernel(sublime.active_window(), self.view, logger=logger) self.view.run_command("helium_clear_all_cells") @chain_callbacks def _show_kernel_selection_menu(window, view, cb, *, add_new=False): # Get the kernel ID related to `view` if exists. try: current_kernel_id = ViewManager.get_kernel_for_view(view.buffer_id()).kernel_id except KeyError: # TODO fix to use property of views. result = re.match(r"\*Helium Output\* .*?\(\[.*?\] ([\w-]*)\)", view.name()) if result: current_kernel_id = result.group(1) else: current_kernel_id = "" # It's better to pass the list of (connection_name, kernel_id) tuples # to improve the appearane of the menu. kernel_list = HeliumKernelManager.list_kernels() menu_items = [ "* " + repr if kernel["id"] == current_kernel_id else repr for repr, kernel in zip(HeliumKernelManager.list_kernel_reprs(), kernel_list) ] if add_new: menu_items += ["New kernel"] index = yield partial(window.show_quick_panel, menu_items) if index == -1: selected_kernel = None elif index == len(kernel_list): yield partial(_start_kernel, window, view) else: selected_kernel = kernel_list[index] cb(selected_kernel) @chain_callbacks def _interrupt_kernel(window, view, *, continue_cb=lambda: None, logger=HELIUM_LOGGER): selected_kernel = yield partial(_show_kernel_selection_menu, window, view) if selected_kernel is not None: HeliumKernelManager.interrupt_kernel(selected_kernel["id"]) log_info_msg = ("Interrupted kernel {kernel_id}.").format( kernel_id=selected_kernel["id"] ) logger.info(log_info_msg) continue_cb() class HeliumInterruptKernel(TextCommand): """Interrupt Jupyter kernel.""" def is_enabled(self, *, logger=HELIUM_LOGGER): try: kernel = ViewManager.get_kernel_for_view(self.view.buffer_id()) except KeyError: return False return HeliumKernelManager.get_kernel(kernel.kernel_id).is_alive() def is_visible(self, *, logger=HELIUM_LOGGER): return self.is_enabled() def run(self, edit, *, logger=HELIUM_LOGGER): """Command definition.""" _interrupt_kernel(sublime.active_window(), self.view, logger=logger) @chain_callbacks def _restart_kernel(window, view, *, continue_cb=lambda: None, logger=HELIUM_LOGGER): selected_kernel = yield partial(_show_kernel_selection_menu, window, view) if selected_kernel is not None: HeliumKernelManager.restart_kernel(selected_kernel["id"]) log_info_msg = ("Restarted kernel {kernel_id}.").format( kernel_id=selected_kernel["id"] ) logger.info(log_info_msg) continue_cb() class HeliumRestartKernel(TextCommand): """Restart Jupyter kernel.""" def is_enabled(self, *, logger=HELIUM_LOGGER): try: kernel = ViewManager.get_kernel_for_view(self.view.buffer_id()) except KeyError: return False return HeliumKernelManager.get_kernel(kernel.kernel_id).is_alive() def is_visible(self, *, logger=HELIUM_LOGGER): return self.is_enabled() def run(self, edit, *, logger=HELIUM_LOGGER): """Command definition.""" _restart_kernel(sublime.active_window(), self.view, logger=logger) self.view.run_command("helium_clear_all_cells") @chain_callbacks def _shutdown_kernel(window, view, *, continue_cb=lambda: None, logger=HELIUM_LOGGER): selected_kernel = yield partial(_show_kernel_selection_menu, window, view) if selected_kernel is not None: HeliumKernelManager.shutdown_kernel(selected_kernel["id"]) log_info_msg = ("Shutdown kernel {kernel_id}.").format( kernel_id=selected_kernel["id"] ) logger.info(log_info_msg) ViewManager.remove_view(view.buffer_id()) view.set_status("helium_connected_kernel", "") continue_cb() class HeliumShutdownKernel(TextCommand): """Shutdown Jupyter kernel.""" def is_enabled(self, *, logger=HELIUM_LOGGER): try: kernel = ViewManager.get_kernel_for_view(self.view.buffer_id()) except KeyError: return False return HeliumKernelManager.get_kernel(kernel.kernel_id).is_alive() def is_visible(self, *, logger=HELIUM_LOGGER): return self.is_enabled() def run(self, edit, *, logger=HELIUM_LOGGER): """Command definition.""" _shutdown_kernel(sublime.active_window(), self.view, logger=logger) class HeliumRunCellManager(ViewEventListener): """Manage 'Run cell' phantoms.""" def __init__(self, view): self.view = view self.timeout_scheduled = False self.needs_update = False def on_modified(self, *, logger=HELIUM_LOGGER): try: kernel = ViewManager.get_kernel_for_view(self.view.buffer_id()) if not kernel.is_alive(): return except KeyError: return # Call update_run_cell_phantoms(), but not any more than 10 times a second if self.timeout_scheduled: self.needs_update = True else: sublime.set_timeout(lambda: self.handle_timeout(), 100) self.timeout_scheduled = True update_run_cell_phantoms(self.view, logger=logger) def handle_timeout(self): self.timeout_scheduled = False if self.needs_update: self.needs_update = False update_run_cell_phantoms(self.view) def update_run_cell_phantoms(view, *, logger=HELIUM_LOGGER): """Add "Run Cell" links to each code cell.""" # find all cell delimiters: cell_delimiter_pattern = sublime.load_settings("Helium.sublime-settings").get( "cell_delimiter_pattern" ) limits = view.find_all(cell_delimiter_pattern) # append a virtual delimiter at EOF limits.append(sublime.Region(view.size(), view.size())) # remove existing Run cell phantoms, we'll recreate all of them view.erase_phantoms(RUN_CELL_PHANTOM_ID) for i in range(len(limits) - 1): code_region = sublime.Region(limits[i].end() + 1, limits[i + 1].begin() + 0) phantom_region = sublime.Region(limits[i].end(), limits[i].end()) view.add_phantom( RUN_CELL_PHANTOM_ID, phantom_region, RUN_CELL_PHANTOM, sublime.LAYOUT_INLINE, on_navigate=lambda href, view=view, region=code_region: _execute_cell( view, region ), ) def get_line(view: sublime.View, row: int) -> str: """Get the code line under the cursor.""" point = view.text_point(row, 0) line_region = view.line(point) return view.substr(line_region) def get_indent(view: sublime.View, row: int) -> str: line = get_line(view, row) return INDENT_PATTERN.match(line).group() def get_block(view: sublime.View, s: sublime.Region) -> (str, sublime.Region): """Get the code block under the cursor. The code block is the lines satisfying the following conditions: - Includes the line under the cursor. - Includes no blank line. - More indented than that of the line under the cursor. If `s` is a selected region, the code block is it. """ if not s.empty(): return (view.substr(s), s) view_end_row = view.rowcol(view.size())[0] current_row = view.rowcol(s.begin())[0] current_indent = get_indent(view, current_row) start_point = 0 for first_row in range(current_row, -1, -1): indent = get_indent(view, first_row) if ( not indent.startswith(current_indent) or get_line(view, first_row).strip() == "" ): start_point = view.text_point(first_row + 1, 0) break end_point = view.size() for last_row in range(current_row, view_end_row + 1): indent = get_indent(view, last_row) if ( not indent.startswith(current_indent) or get_line(view, last_row).strip() == "" ): end_point = view.text_point(last_row, 0) - 1 break block_region = sublime.Region(start_point, end_point) return (view.substr(block_region), block_region) @chain_callbacks def _execute_block(view, *, logger=HELIUM_LOGGER): try: kernel = ViewManager.get_kernel_for_view(view.buffer_id()) except KeyError: sublime.message_dialog("No kernel is connected to this view.") yield lambda cb: _connect_kernel(sublime.active_window(), view, continue_cb=cb) kernel = ViewManager.get_kernel_for_view(view.buffer_id()) pre_code = [] for s in view.sel(): code, region = get_block(view, s) if code == pre_code: continue kernel.execute_code(code, region, view) log_info_msg = "Executed code {code} with kernel {kernel_id}".format( code=code, kernel_id=kernel.kernel_id ) logger.info(log_info_msg) pre_code = code @chain_callbacks def _execute_cell(view, region: sublime.Region, *, logger=HELIUM_LOGGER): try: kernel = ViewManager.get_kernel_for_view(view.buffer_id()) except KeyError: sublime.message_dialog("No kernel is connected to this view.") yield lambda cb: _connect_kernel(sublime.active_window(), view, continue_cb=cb) kernel = ViewManager.get_kernel_for_view(view.buffer_id()) code, cell = get_cell(view, region, logger=logger) kernel.execute_code(code, cell, view) log_info_msg = "Executed code {code} with kernel {kernel_id}".format( code=code, kernel_id=kernel.kernel_id ) logger.info(log_info_msg) class HeliumExecuteBlock(TextCommand): """Execute code.""" def is_enabled(self, *, logger=HELIUM_LOGGER): try: kernel = ViewManager.get_kernel_for_view(self.view.buffer_id()) except KeyError: return False return HeliumKernelManager.get_kernel(kernel.kernel_id).is_alive() def is_visible(self, *, logger=HELIUM_LOGGER): return self.is_enabled() def run(self, edit, *, logger=HELIUM_LOGGER): """Command definition.""" _execute_block(self.view, logger=logger) class HeliumExecuteCell(TextCommand): """Execute code cell.""" def is_enabled(self, *, logger=HELIUM_LOGGER): try: kernel = ViewManager.get_kernel_for_view(self.view.buffer_id()) except KeyError: return False return kernel.is_alive() def is_visible(self, *, logger=HELIUM_LOGGER): return self.is_enabled() def run(self, edit, move_cursor=False, *, logger=HELIUM_LOGGER): """If move_cursor is true, move the cursor to the next cell after execution.""" for s in self.view.sel(): _execute_cell(self.view, s, logger=logger) if move_cursor: _, cell = get_cell(self.view, self.view.sel()[-1], logger=logger) pt = sublime.Region(cell.end() + 1, cell.end() + 1) self.view.sel().clear() self.view.sel().add(pt) # TODO: scroll to cursor after phantoms after Jupyter callback # rather than fixed time sublime.set_timeout(lambda: self.view.show(pt), 500) class HeliumClearAllCells(TextCommand): """Clear all phantoms.""" def is_enabled(self, *, logger=HELIUM_LOGGER): try: kernel = ViewManager.get_kernel_for_view(self.view.buffer_id()) # if view has an attached kernel, return its status return kernel.is_alive() except KeyError: # if view doesn't have an attached kernel, check if view was created by # kernel parent_view = self._get_parent_view() return parent_view is not None def is_visible(self, *, logger=HELIUM_LOGGER): return self.is_enabled() def run(self, edit, *, logger=HELIUM_LOGGER): # get correct kernel try: kernel = ViewManager.get_kernel_for_view(self.view.buffer_id()) except KeyError: view = self._get_parent_view() kernel = ViewManager.get_kernel_for_view(view.buffer_id()) def cb(): if kernel._show_inline_output: for k in range(MAX_PHANTOMS): self.view.erase_phantom_by_id(k) else: kernel.get_view().close() kernel.activate_view() # clear the old phantoms async sublime.set_timeout_async(cb, 0) def _get_parent_view(self) -> sublime.View: for window in sublime.windows(): for view in window.views(): try: kernel = ViewManager.get_kernel_for_view(view.buffer_id()) except KeyError: continue if kernel.get_view() == self.view: return view return None class StatusBar(object): """Status Bar with animation. This class is based on the one by @randy3k. """ def __init__(self, view, width=10, interval=500): self.view = view self.width = width self.buffer_id = view.buffer_id() self.interval = interval self.pos = 0 try: self.kernel = ViewManager.get_kernel_for_view(self.buffer_id) self.start() except KeyError: # When view is not connected. self.stop() def start(self): self.update() def stop(self): self.view.set_status("helium_connected_kernel", "") def update(self, pos=0): # `pos` can't be a property of `StatusBar` because it's not updated # when `update()` is called by `sublime.set_timeout[_async]()`. if self.buffer_id != sublime.active_window().active_view().buffer_id(): # Stop when view is unfocused. self.stop() return execution_state = self.kernel.execution_state if execution_state == "dead": # Stop when kernel is dead. self.view.set_status("helium_connected_kernel", "") return elif execution_state == "busy": pos = pos % (2 * self.width) before = min(pos, (2 * self.width) - pos) after = self.width - before progress_bar = " [{}={}]".format(" " * before, " " * after) else: # Make progress bar always start with pos=0. pos = -1 progress_bar = "" status = ( "{repr} (state: {execution_state})".format( repr=self.kernel.repr, execution_state=self.kernel.execution_state ) + progress_bar ) self.view.set_status("helium_connected_kernel", status) sublime.set_timeout_async(lambda: self.update(pos + 1), self.interval) class HeliumStatusUpdater(ViewEventListener): """Listen to the heartbeat of kernel and update status of view.""" def on_activated_async(self): sublime.set_timeout_async(lambda: StatusBar(self.view), 0) class HeliumGetObjectInspection(TextCommand): """Get object inspection.""" def is_enabled(self, *, logger=HELIUM_LOGGER): try: kernel = ViewManager.get_kernel_for_view(self.view.buffer_id()) except KeyError: return False return HeliumKernelManager.get_kernel(kernel.kernel_id).is_alive() def is_visible(self, *, logger=HELIUM_LOGGER): return self.is_enabled() @chain_callbacks def run(self, edit, *, logger=HELIUM_LOGGER): view = self.view try: kernel = ViewManager.get_kernel_for_view(view.buffer_id()) except KeyError: sublime.message_dialog("No kernel is connected to this view.") yield lambda cb: _connect_kernel( sublime.active_window(), view, continue_cb=cb ) kernel = ViewManager.get_kernel_for_view(view.buffer_id()) pre_code = [] for s in view.sel(): code, region = get_block(view, s) cursor_pos = s.end() - region.begin() if code == pre_code: continue kernel.get_inspection(code, cursor_pos) log_info_msg = ( "Requested object inspection for code {code} with kernel {kernel_id}" ).format(code=code, kernel_id=kernel.kernel_id) logger.info(log_info_msg) pre_code = code class HeliumCompleter(EventListener): """Completer.""" def on_query_completions(self, view, prefix, locations, *, logger=HELIUM_LOGGER): """Get completions from the Jupyter kernel.""" use_complete = sublime.load_settings("Helium.sublime-settings").get("complete") if not use_complete: return None timeout = sublime.load_settings("Helium.sublime-settings").get( "complete_timeout" ) try: kernel = ViewManager.get_kernel_for_view(view.buffer_id()) location = locations[0] code = view.substr(view.line(location)) log_info_msg = ( "Requested completion for code {code} with kernel {kernel_id}" ).format(code=code, kernel_id=kernel.kernel_id) logger.info(log_info_msg) _, col = view.rowcol(location) return kernel.get_complete(code, col, timeout) except Exception: # noqa return None ================================================ FILE: lib/__init__.py ================================================ ================================================ FILE: lib/client/__init__.py ================================================ ================================================ FILE: lib/client/decorator.py ================================================ # ######################### LICENSE ############################ # # Copyright (c) 2005-2018, Michele Simionato # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # Redistributions in bytecode form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS # OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH # DAMAGE. """ Decorator module, see http://pypi.python.org/pypi/decorator for the documentation. """ from __future__ import print_function import re import sys import inspect import operator import itertools import collections __version__ = '4.4.2' if sys.version_info >= (3,): from inspect import getfullargspec def get_init(cls): return cls.__init__ else: FullArgSpec = collections.namedtuple( 'FullArgSpec', 'args varargs varkw defaults ' 'kwonlyargs kwonlydefaults annotations') def getfullargspec(f): "A quick and dirty replacement for getfullargspec for Python 2.X" return FullArgSpec._make(inspect.getargspec(f) + ([], None, {})) def get_init(cls): return cls.__init__.__func__ try: iscoroutinefunction = inspect.iscoroutinefunction except AttributeError: # let's assume there are no coroutine functions in old Python def iscoroutinefunction(f): return False try: from inspect import isgeneratorfunction except ImportError: # assume no generator function in old Python versions def isgeneratorfunction(caller): return False DEF = re.compile(r'\s*def\s*([_\w][_\w\d]*)\s*\(') # basic functionality class FunctionMaker(object): """ An object with the ability to create functions with a given signature. It has attributes name, doc, module, signature, defaults, dict and methods update and make. """ # Atomic get-and-increment provided by the GIL _compile_count = itertools.count() # make pylint happy args = varargs = varkw = defaults = kwonlyargs = kwonlydefaults = () def __init__(self, func=None, name=None, signature=None, defaults=None, doc=None, module=None, funcdict=None): self.shortsignature = signature if func: # func can be a class or a callable, but not an instance method self.name = func.__name__ if self.name == '': # small hack for lambda functions self.name = '_lambda_' self.doc = func.__doc__ self.module = func.__module__ if inspect.isfunction(func): argspec = getfullargspec(func) self.annotations = getattr(func, '__annotations__', {}) for a in ('args', 'varargs', 'varkw', 'defaults', 'kwonlyargs', 'kwonlydefaults'): setattr(self, a, getattr(argspec, a)) for i, arg in enumerate(self.args): setattr(self, 'arg%d' % i, arg) allargs = list(self.args) allshortargs = list(self.args) if self.varargs: allargs.append('*' + self.varargs) allshortargs.append('*' + self.varargs) elif self.kwonlyargs: allargs.append('*') # single star syntax for a in self.kwonlyargs: allargs.append('%s=None' % a) allshortargs.append('%s=%s' % (a, a)) if self.varkw: allargs.append('**' + self.varkw) allshortargs.append('**' + self.varkw) self.signature = ', '.join(allargs) self.shortsignature = ', '.join(allshortargs) self.dict = func.__dict__.copy() # func=None happens when decorating a caller if name: self.name = name if signature is not None: self.signature = signature if defaults: self.defaults = defaults if doc: self.doc = doc if module: self.module = module if funcdict: self.dict = funcdict # check existence required attributes assert hasattr(self, 'name') if not hasattr(self, 'signature'): raise TypeError('You are decorating a non function: %s' % func) def update(self, func, **kw): "Update the signature of func with the data in self" func.__name__ = self.name func.__doc__ = getattr(self, 'doc', None) func.__dict__ = getattr(self, 'dict', {}) func.__defaults__ = self.defaults func.__kwdefaults__ = self.kwonlydefaults or None func.__annotations__ = getattr(self, 'annotations', None) try: frame = sys._getframe(3) except AttributeError: # for IronPython and similar implementations callermodule = '?' else: callermodule = frame.f_globals.get('__name__', '?') func.__module__ = getattr(self, 'module', callermodule) func.__dict__.update(kw) def make(self, src_templ, evaldict=None, addsource=False, **attrs): "Make a new function from a given template and update the signature" src = src_templ % vars(self) # expand name and signature evaldict = evaldict or {} mo = DEF.search(src) if mo is None: raise SyntaxError('not a valid function template\n%s' % src) name = mo.group(1) # extract the function name names = set([name] + [arg.strip(' *') for arg in self.shortsignature.split(',')]) for n in names: if n in ('_func_', '_call_'): raise NameError('%s is overridden in\n%s' % (n, src)) if not src.endswith('\n'): # add a newline for old Pythons src += '\n' # Ensure each generated function has a unique filename for profilers # (such as cProfile) that depend on the tuple of (, # , ) being unique. filename = '' % next(self._compile_count) try: code = compile(src, filename, 'single') exec(code, evaldict) except Exception: print('Error in generated code:', file=sys.stderr) print(src, file=sys.stderr) raise func = evaldict[name] if addsource: attrs['__source__'] = src self.update(func, **attrs) return func @classmethod def create(cls, obj, body, evaldict, defaults=None, doc=None, module=None, addsource=True, **attrs): """ Create a function from the strings name, signature and body. evaldict is the evaluation dictionary. If addsource is true an attribute __source__ is added to the result. The attributes attrs are added, if any. """ if isinstance(obj, str): # "name(signature)" name, rest = obj.strip().split('(', 1) signature = rest[:-1] # strip a right parens func = None else: # a function name = None signature = None func = obj self = cls(func, name, signature, defaults, doc, module) ibody = '\n'.join(' ' + line for line in body.splitlines()) caller = evaldict.get('_call_') # when called from `decorate` if caller and iscoroutinefunction(caller): body = ('async def %(name)s(%(signature)s):\n' + ibody).replace( 'return', 'return await') else: body = 'def %(name)s(%(signature)s):\n' + ibody return self.make(body, evaldict, addsource, **attrs) def decorate(func, caller, extras=()): """ decorate(func, caller) decorates a function using a caller. If the caller is a generator function, the resulting function will be a generator function. """ evaldict = dict(_call_=caller, _func_=func) es = '' for i, extra in enumerate(extras): ex = '_e%d_' % i evaldict[ex] = extra es += ex + ', ' if '3.5' <= sys.version < '3.6': # with Python 3.5 isgeneratorfunction returns True for all coroutines # however we know that it is NOT possible to have a generator # coroutine in python 3.5: PEP525 was not there yet generatorcaller = isgeneratorfunction( caller) and not iscoroutinefunction(caller) else: generatorcaller = isgeneratorfunction(caller) if generatorcaller: fun = FunctionMaker.create( func, "for res in _call_(_func_, %s%%(shortsignature)s):\n" " yield res" % es, evaldict, __wrapped__=func) else: fun = FunctionMaker.create( func, "return _call_(_func_, %s%%(shortsignature)s)" % es, evaldict, __wrapped__=func) if hasattr(func, '__qualname__'): fun.__qualname__ = func.__qualname__ return fun def decorator(caller, _func=None): """decorator(caller) converts a caller function into a decorator""" if _func is not None: # return a decorated function # this is obsolete behavior; you should use decorate instead return decorate(_func, caller) # else return a decorator function defaultargs, defaults = '', () if inspect.isclass(caller): name = caller.__name__.lower() doc = 'decorator(%s) converts functions/generators into ' \ 'factories of %s objects' % (caller.__name__, caller.__name__) elif inspect.isfunction(caller): if caller.__name__ == '': name = '_lambda_' else: name = caller.__name__ doc = caller.__doc__ nargs = caller.__code__.co_argcount ndefs = len(caller.__defaults__ or ()) defaultargs = ', '.join(caller.__code__.co_varnames[nargs-ndefs:nargs]) if defaultargs: defaultargs += ',' defaults = caller.__defaults__ else: # assume caller is an object with a __call__ method name = caller.__class__.__name__.lower() doc = caller.__call__.__doc__ evaldict = dict(_call=caller, _decorate_=decorate) dec = FunctionMaker.create( '%s(func, %s)' % (name, defaultargs), 'if func is None: return lambda func: _decorate_(func, _call, (%s))\n' 'return _decorate_(func, _call, (%s))' % (defaultargs, defaultargs), evaldict, doc=doc, module=caller.__module__, __wrapped__=caller) if defaults: dec.__defaults__ = (None,) + defaults return dec # ####################### contextmanager ####################### # try: # Python >= 3.2 from contextlib import _GeneratorContextManager except ImportError: # Python >= 2.5 from contextlib import GeneratorContextManager as _GeneratorContextManager class ContextManager(_GeneratorContextManager): def __call__(self, func): """Context manager decorator""" return FunctionMaker.create( func, "with _self_: return _func_(%(shortsignature)s)", dict(_self_=self, _func_=func), __wrapped__=func) init = getfullargspec(_GeneratorContextManager.__init__) n_args = len(init.args) if n_args == 2 and not init.varargs: # (self, genobj) Python 2.7 def __init__(self, g, *a, **k): return _GeneratorContextManager.__init__(self, g(*a, **k)) ContextManager.__init__ = __init__ elif n_args == 2 and init.varargs: # (self, gen, *a, **k) Python 3.4 pass elif n_args == 4: # (self, gen, args, kwds) Python 3.5 def __init__(self, g, *a, **k): return _GeneratorContextManager.__init__(self, g, a, k) ContextManager.__init__ = __init__ _contextmanager = decorator(ContextManager) def contextmanager(func): # Enable Pylint config: contextmanager-decorators=decorator.contextmanager return _contextmanager(func) # ############################ dispatch_on ############################ # def append(a, vancestors): """ Append ``a`` to the list of the virtual ancestors, unless it is already included. """ add = True for j, va in enumerate(vancestors): if issubclass(va, a): add = False break if issubclass(a, va): vancestors[j] = a add = False if add: vancestors.append(a) # inspired from simplegeneric by P.J. Eby and functools.singledispatch def dispatch_on(*dispatch_args): """ Factory of decorators turning a function into a generic function dispatching on the given arguments. """ assert dispatch_args, 'No dispatch args passed' dispatch_str = '(%s,)' % ', '.join(dispatch_args) def check(arguments, wrong=operator.ne, msg=''): """Make sure one passes the expected number of arguments""" if wrong(len(arguments), len(dispatch_args)): raise TypeError('Expected %d arguments, got %d%s' % (len(dispatch_args), len(arguments), msg)) def gen_func_dec(func): """Decorator turning a function into a generic function""" # first check the dispatch arguments argset = set(getfullargspec(func).args) if not set(dispatch_args) <= argset: raise NameError('Unknown dispatch arguments %s' % dispatch_str) typemap = {} def vancestors(*types): """ Get a list of sets of virtual ancestors for the given types """ check(types) ras = [[] for _ in range(len(dispatch_args))] for types_ in typemap: for t, type_, ra in zip(types, types_, ras): if issubclass(t, type_) and type_ not in t.mro(): append(type_, ra) return [set(ra) for ra in ras] def ancestors(*types): """ Get a list of virtual MROs, one for each type """ check(types) lists = [] for t, vas in zip(types, vancestors(*types)): n_vas = len(vas) if n_vas > 1: raise RuntimeError( 'Ambiguous dispatch for %s: %s' % (t, vas)) elif n_vas == 1: va, = vas mro = type('t', (t, va), {}).mro()[1:] else: mro = t.mro() lists.append(mro[:-1]) # discard t and object return lists def register(*types): """ Decorator to register an implementation for the given types """ check(types) def dec(f): check(getfullargspec(f).args, operator.lt, ' in ' + f.__name__) typemap[types] = f return f return dec def dispatch_info(*types): """ An utility to introspect the dispatch algorithm """ check(types) lst = [] for anc in itertools.product(*ancestors(*types)): lst.append(tuple(a.__name__ for a in anc)) return lst def _dispatch(dispatch_args, *args, **kw): types = tuple(type(arg) for arg in dispatch_args) try: # fast path f = typemap[types] except KeyError: pass else: return f(*args, **kw) combinations = itertools.product(*ancestors(*types)) next(combinations) # the first one has been already tried for types_ in combinations: f = typemap.get(types_) if f is not None: return f(*args, **kw) # else call the default implementation return func(*args, **kw) return FunctionMaker.create( func, 'return _f_(%s, %%(shortsignature)s)' % dispatch_str, dict(_f_=_dispatch), register=register, default=func, typemap=typemap, vancestors=vancestors, ancestors=ancestors, dispatch_info=dispatch_info, __wrapped__=func) gen_func_dec.__name__ = 'dispatch_on' + dispatch_str return gen_func_dec ================================================ FILE: lib/client/decorator_version ================================================ 4.4.2 ================================================ FILE: lib/client/ipython_genutils/__init__.py ================================================ from ._version import __version__, version_info ================================================ FILE: lib/client/ipython_genutils/_version.py ================================================ version_info = (0, 2, 0) __version__ = '.'.join(map(str, version_info)) ================================================ FILE: lib/client/ipython_genutils/encoding.py ================================================ # coding: utf-8 """ Utilities for dealing with text encodings """ #----------------------------------------------------------------------------- # Copyright (C) 2008-2012 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- import sys import locale import warnings # to deal with the possibility of sys.std* not being a stream at all def get_stream_enc(stream, default=None): """Return the given stream's encoding or a default. There are cases where ``sys.std*`` might not actually be a stream, so check for the encoding attribute prior to returning it, and return a default if it doesn't exist or evaluates as False. ``default`` is None if not provided. """ if not hasattr(stream, 'encoding') or not stream.encoding: return default else: return stream.encoding # Less conservative replacement for sys.getdefaultencoding, that will try # to match the environment. # Defined here as central function, so if we find better choices, we # won't need to make changes all over IPython. def getdefaultencoding(prefer_stream=True): """Return IPython's guess for the default encoding for bytes as text. If prefer_stream is True (default), asks for stdin.encoding first, to match the calling Terminal, but that is often None for subprocesses. Then fall back on locale.getpreferredencoding(), which should be a sensible platform default (that respects LANG environment), and finally to sys.getdefaultencoding() which is the most conservative option, and usually ASCII on Python 2 or UTF8 on Python 3. """ enc = None if prefer_stream: enc = get_stream_enc(sys.stdin) if not enc or enc=='ascii': try: # There are reports of getpreferredencoding raising errors # in some cases, which may well be fixed, but let's be conservative here. enc = locale.getpreferredencoding() except Exception: pass enc = enc or sys.getdefaultencoding() # On windows `cp0` can be returned to indicate that there is no code page. # Since cp0 is an invalid encoding return instead cp1252 which is the # Western European default. if enc == 'cp0': warnings.warn( "Invalid code page cp0 detected - using cp1252 instead." "If cp1252 is incorrect please ensure a valid code page " "is defined for the process.", RuntimeWarning) return 'cp1252' return enc DEFAULT_ENCODING = getdefaultencoding() ================================================ FILE: lib/client/ipython_genutils/importstring.py ================================================ # encoding: utf-8 """ A simple utility to import something by its string name. """ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. def import_item(name): """Import and return ``bar`` given the string ``foo.bar``. Calling ``bar = import_item("foo.bar")`` is the functional equivalent of executing the code ``from foo import bar``. Parameters ---------- name : string The fully qualified name of the module/package being imported. Returns ------- mod : module object The module that was imported. """ parts = name.rsplit('.', 1) if len(parts) == 2: # called with 'foo.bar....' package, obj = parts module = __import__(package, fromlist=[obj]) try: pak = getattr(module, obj) except AttributeError: raise ImportError('No module named %s' % obj) return pak else: # called with un-dotted string return __import__(parts[0]) ================================================ FILE: lib/client/ipython_genutils/ipstruct.py ================================================ # encoding: utf-8 """A dict subclass that supports attribute style access. Can probably be replaced by types.SimpleNamespace from Python 3.3 """ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. __all__ = ['Struct'] class Struct(dict): """A dict subclass with attribute style access. This dict subclass has a a few extra features: * Attribute style access. * Protection of class members (like keys, items) when using attribute style access. * The ability to restrict assignment to only existing keys. * Intelligent merging. * Overloaded operators. """ _allownew = True def __init__(self, *args, **kw): """Initialize with a dictionary, another Struct, or data. Parameters ---------- args : dict, Struct Initialize with one dict or Struct kw : dict Initialize with key, value pairs. Examples -------- >>> s = Struct(a=10,b=30) >>> s.a 10 >>> s.b 30 >>> s2 = Struct(s,c=30) >>> sorted(s2.keys()) ['a', 'b', 'c'] """ object.__setattr__(self, '_allownew', True) dict.__init__(self, *args, **kw) def __setitem__(self, key, value): """Set an item with check for allownew. Examples -------- >>> s = Struct() >>> s['a'] = 10 >>> s.allow_new_attr(False) >>> s['a'] = 10 >>> s['a'] 10 >>> try: ... s['b'] = 20 ... except KeyError: ... print('this is not allowed') ... this is not allowed """ if not self._allownew and key not in self: raise KeyError( "can't create new attribute %s when allow_new_attr(False)" % key) dict.__setitem__(self, key, value) def __setattr__(self, key, value): """Set an attr with protection of class members. This calls :meth:`self.__setitem__` but convert :exc:`KeyError` to :exc:`AttributeError`. Examples -------- >>> s = Struct() >>> s.a = 10 >>> s.a 10 >>> try: ... s.get = 10 ... except AttributeError: ... print("you can't set a class member") ... you can't set a class member """ # If key is an str it might be a class member or instance var if isinstance(key, str): # I can't simply call hasattr here because it calls getattr, which # calls self.__getattr__, which returns True for keys in # self._data. But I only want keys in the class and in # self.__dict__ if key in self.__dict__ or hasattr(Struct, key): raise AttributeError( 'attr %s is a protected member of class Struct.' % key ) try: self.__setitem__(key, value) except KeyError as e: raise AttributeError(e) def __getattr__(self, key): """Get an attr by calling :meth:`dict.__getitem__`. Like :meth:`__setattr__`, this method converts :exc:`KeyError` to :exc:`AttributeError`. Examples -------- >>> s = Struct(a=10) >>> s.a 10 >>> type(s.get) <... 'builtin_function_or_method'> >>> try: ... s.b ... except AttributeError: ... print("I don't have that key") ... I don't have that key """ try: result = self[key] except KeyError: raise AttributeError(key) else: return result def __iadd__(self, other): """s += s2 is a shorthand for s.merge(s2). Examples -------- >>> s = Struct(a=10,b=30) >>> s2 = Struct(a=20,c=40) >>> s += s2 >>> sorted(s.keys()) ['a', 'b', 'c'] """ self.merge(other) return self def __add__(self,other): """s + s2 -> New Struct made from s.merge(s2). Examples -------- >>> s1 = Struct(a=10,b=30) >>> s2 = Struct(a=20,c=40) >>> s = s1 + s2 >>> sorted(s.keys()) ['a', 'b', 'c'] """ sout = self.copy() sout.merge(other) return sout def __sub__(self,other): """s1 - s2 -> remove keys in s2 from s1. Examples -------- >>> s1 = Struct(a=10,b=30) >>> s2 = Struct(a=40) >>> s = s1 - s2 >>> s {'b': 30} """ sout = self.copy() sout -= other return sout def __isub__(self,other): """Inplace remove keys from self that are in other. Examples -------- >>> s1 = Struct(a=10,b=30) >>> s2 = Struct(a=40) >>> s1 -= s2 >>> s1 {'b': 30} """ for k in other.keys(): if k in self: del self[k] return self def __dict_invert(self, data): """Helper function for merge. Takes a dictionary whose values are lists and returns a dict with the elements of each list as keys and the original keys as values. """ outdict = {} for k,lst in data.items(): if isinstance(lst, str): lst = lst.split() for entry in lst: outdict[entry] = k return outdict def dict(self): return self def copy(self): """Return a copy as a Struct. Examples -------- >>> s = Struct(a=10,b=30) >>> s2 = s.copy() >>> type(s2) is Struct True """ return Struct(dict.copy(self)) def hasattr(self, key): """hasattr function available as a method. Implemented like has_key. Examples -------- >>> s = Struct(a=10) >>> s.hasattr('a') True >>> s.hasattr('b') False >>> s.hasattr('get') False """ return key in self def allow_new_attr(self, allow = True): """Set whether new attributes can be created in this Struct. This can be used to catch typos by verifying that the attribute user tries to change already exists in this Struct. """ object.__setattr__(self, '_allownew', allow) def merge(self, __loc_data__=None, __conflict_solve=None, **kw): """Merge two Structs with customizable conflict resolution. This is similar to :meth:`update`, but much more flexible. First, a dict is made from data+key=value pairs. When merging this dict with the Struct S, the optional dictionary 'conflict' is used to decide what to do. If conflict is not given, the default behavior is to preserve any keys with their current value (the opposite of the :meth:`update` method's behavior). Parameters ---------- __loc_data : dict, Struct The data to merge into self __conflict_solve : dict The conflict policy dict. The keys are binary functions used to resolve the conflict and the values are lists of strings naming the keys the conflict resolution function applies to. Instead of a list of strings a space separated string can be used, like 'a b c'. kw : dict Additional key, value pairs to merge in Notes ----- The `__conflict_solve` dict is a dictionary of binary functions which will be used to solve key conflicts. Here is an example:: __conflict_solve = dict( func1=['a','b','c'], func2=['d','e'] ) In this case, the function :func:`func1` will be used to resolve keys 'a', 'b' and 'c' and the function :func:`func2` will be used for keys 'd' and 'e'. This could also be written as:: __conflict_solve = dict(func1='a b c',func2='d e') These functions will be called for each key they apply to with the form:: func1(self['a'], other['a']) The return value is used as the final merged value. As a convenience, merge() provides five (the most commonly needed) pre-defined policies: preserve, update, add, add_flip and add_s. The easiest explanation is their implementation:: preserve = lambda old,new: old update = lambda old,new: new add = lambda old,new: old + new add_flip = lambda old,new: new + old # note change of order! add_s = lambda old,new: old + ' ' + new # only for str! You can use those four words (as strings) as keys instead of defining them as functions, and the merge method will substitute the appropriate functions for you. For more complicated conflict resolution policies, you still need to construct your own functions. Examples -------- This show the default policy: >>> s = Struct(a=10,b=30) >>> s2 = Struct(a=20,c=40) >>> s.merge(s2) >>> sorted(s.items()) [('a', 10), ('b', 30), ('c', 40)] Now, show how to specify a conflict dict: >>> s = Struct(a=10,b=30) >>> s2 = Struct(a=20,b=40) >>> conflict = {'update':'a','add':'b'} >>> s.merge(s2,conflict) >>> sorted(s.items()) [('a', 20), ('b', 70)] """ data_dict = dict(__loc_data__,**kw) # policies for conflict resolution: two argument functions which return # the value that will go in the new struct preserve = lambda old,new: old update = lambda old,new: new add = lambda old,new: old + new add_flip = lambda old,new: new + old # note change of order! add_s = lambda old,new: old + ' ' + new # default policy is to keep current keys when there's a conflict conflict_solve = dict.fromkeys(self, preserve) # the conflict_solve dictionary is given by the user 'inverted': we # need a name-function mapping, it comes as a function -> names # dict. Make a local copy (b/c we'll make changes), replace user # strings for the three builtin policies and invert it. if __conflict_solve: inv_conflict_solve_user = __conflict_solve.copy() for name, func in [('preserve',preserve), ('update',update), ('add',add), ('add_flip',add_flip), ('add_s',add_s)]: if name in inv_conflict_solve_user.keys(): inv_conflict_solve_user[func] = inv_conflict_solve_user[name] del inv_conflict_solve_user[name] conflict_solve.update(self.__dict_invert(inv_conflict_solve_user)) for key in data_dict: if key not in self: self[key] = data_dict[key] else: self[key] = conflict_solve[key](self[key],data_dict[key]) ================================================ FILE: lib/client/ipython_genutils/path.py ================================================ # encoding: utf-8 """ Utilities for path handling. """ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. import os import sys import errno import shutil import random from . import py3compat fs_encoding = sys.getfilesystemencoding() def filefind(filename, path_dirs=None): """Find a file by looking through a sequence of paths. This iterates through a sequence of paths looking for a file and returns the full, absolute path of the first occurence of the file. If no set of path dirs is given, the filename is tested as is, after running through :func:`expandvars` and :func:`expanduser`. Thus a simple call:: filefind('myfile.txt') will find the file in the current working dir, but:: filefind('~/myfile.txt') Will find the file in the users home directory. This function does not automatically try any paths, such as the cwd or the user's home directory. Parameters ---------- filename : str The filename to look for. path_dirs : str, None or sequence of str The sequence of paths to look for the file in. If None, the filename need to be absolute or be in the cwd. If a string, the string is put into a sequence and the searched. If a sequence, walk through each element and join with ``filename``, calling :func:`expandvars` and :func:`expanduser` before testing for existence. Returns ------- Raises :exc:`IOError` or returns absolute path to file. """ # If paths are quoted, abspath gets confused, strip them... filename = filename.strip('"').strip("'") # If the input is an absolute path, just check it exists if os.path.isabs(filename) and os.path.isfile(filename): return filename if path_dirs is None: path_dirs = ("",) elif isinstance(path_dirs, py3compat.string_types): path_dirs = (path_dirs,) for path in path_dirs: if path == '.': path = py3compat.getcwd() testname = expand_path(os.path.join(path, filename)) if os.path.isfile(testname): return os.path.abspath(testname) raise IOError("File %r does not exist in any of the search paths: %r" % (filename, path_dirs) ) def expand_path(s): """Expand $VARS and ~names in a string, like a shell :Examples: In [2]: os.environ['FOO']='test' In [3]: expand_path('variable FOO is $FOO') Out[3]: 'variable FOO is test' """ # This is a pretty subtle hack. When expand user is given a UNC path # on Windows (\\server\share$\%username%), os.path.expandvars, removes # the $ to get (\\server\share\%username%). I think it considered $ # alone an empty var. But, we need the $ to remains there (it indicates # a hidden share). if os.name=='nt': s = s.replace('$\\', 'IPYTHON_TEMP') s = os.path.expandvars(os.path.expanduser(s)) if os.name=='nt': s = s.replace('IPYTHON_TEMP', '$\\') return s try: ENOLINK = errno.ENOLINK except AttributeError: ENOLINK = 1998 def link(src, dst): """Hard links ``src`` to ``dst``, returning 0 or errno. Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't supported by the operating system. """ if not hasattr(os, "link"): return ENOLINK link_errno = 0 try: os.link(src, dst) except OSError as e: link_errno = e.errno return link_errno def link_or_copy(src, dst): """Attempts to hardlink ``src`` to ``dst``, copying if the link fails. Attempts to maintain the semantics of ``shutil.copy``. Because ``os.link`` does not overwrite files, a unique temporary file will be used if the target already exists, then that file will be moved into place. """ if os.path.isdir(dst): dst = os.path.join(dst, os.path.basename(src)) link_errno = link(src, dst) if link_errno == errno.EEXIST: if os.stat(src).st_ino == os.stat(dst).st_ino: # dst is already a hard link to the correct file, so we don't need # to do anything else. If we try to link and rename the file # anyway, we get duplicate files - see http://bugs.python.org/issue21876 return new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), ) try: link_or_copy(src, new_dst) except: try: os.remove(new_dst) except OSError: pass raise os.rename(new_dst, dst) elif link_errno != 0: # Either link isn't supported, or the filesystem doesn't support # linking, or 'src' and 'dst' are on different filesystems. shutil.copy(src, dst) def ensure_dir_exists(path, mode=0o755): """ensure that a directory exists If it doesn't exist, try to create it and protect against a race condition if another process is doing the same. The default permissions are 755, which differ from os.makedirs default of 777. """ if not os.path.exists(path): try: os.makedirs(path, mode=mode) except OSError as e: if e.errno != errno.EEXIST: raise elif not os.path.isdir(path): raise IOError("%r exists but is not a directory" % path) ================================================ FILE: lib/client/ipython_genutils/py3compat.py ================================================ # coding: utf-8 """Compatibility tricks for Python 3. Mainly to do with unicode.""" import functools import os import sys import re import shutil import types from .encoding import DEFAULT_ENCODING def no_code(x, encoding=None): return x def decode(s, encoding=None): encoding = encoding or DEFAULT_ENCODING return s.decode(encoding, "replace") def encode(u, encoding=None): encoding = encoding or DEFAULT_ENCODING return u.encode(encoding, "replace") def cast_unicode(s, encoding=None): if isinstance(s, bytes): return decode(s, encoding) return s def cast_bytes(s, encoding=None): if not isinstance(s, bytes): return encode(s, encoding) return s def buffer_to_bytes(buf): """Cast a buffer or memoryview object to bytes""" if isinstance(buf, memoryview): return buf.tobytes() if not isinstance(buf, bytes): return bytes(buf) return buf def _modify_str_or_docstring(str_change_func): @functools.wraps(str_change_func) def wrapper(func_or_str): if isinstance(func_or_str, string_types): func = None doc = func_or_str else: func = func_or_str doc = func.__doc__ doc = str_change_func(doc) if func: func.__doc__ = doc return func return doc return wrapper def safe_unicode(e): """unicode(e) with various fallbacks. Used for exceptions, which may not be safe to call unicode() on. """ try: return unicode_type(e) except UnicodeError: pass try: return str_to_unicode(str(e)) except UnicodeError: pass try: return str_to_unicode(repr(e)) except UnicodeError: pass return u'Unrecoverably corrupt evalue' # shutil.which from Python 3.4 def _shutil_which(cmd, mode=os.F_OK | os.X_OK, path=None): """Given a command, mode, and a PATH string, return the path which conforms to the given mode on the PATH, or None if there is no such file. `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result of os.environ.get("PATH"), or can be overridden with a custom search path. This is a backport of shutil.which from Python 3.4 """ # Check that a given file can be accessed with the correct mode. # Additionally check that `file` is not a directory, as on Windows # directories pass the os.access check. def _access_check(fn, mode): return (os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn)) # If we're given a path with a directory part, look it up directly rather # than referring to PATH directories. This includes checking relative to the # current directory, e.g. ./script if os.path.dirname(cmd): if _access_check(cmd, mode): return cmd return None if path is None: path = os.environ.get("PATH", os.defpath) if not path: return None path = path.split(os.pathsep) if sys.platform == "win32": # The current directory takes precedence on Windows. if not os.curdir in path: path.insert(0, os.curdir) # PATHEXT is necessary to check on Windows. pathext = os.environ.get("PATHEXT", "").split(os.pathsep) # See if the given file matches any of the expected path extensions. # This will allow us to short circuit when given "python.exe". # If it does match, only test that one, otherwise we have to try # others. if any(cmd.lower().endswith(ext.lower()) for ext in pathext): files = [cmd] else: files = [cmd + ext for ext in pathext] else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. files = [cmd] seen = set() for dir in path: normdir = os.path.normcase(dir) if not normdir in seen: seen.add(normdir) for thefile in files: name = os.path.join(dir, thefile) if _access_check(name, mode): return name return None import platform if sys.version_info[0] >= 3 or platform.python_implementation() == 'IronPython': str_to_unicode = no_code unicode_to_str = no_code str_to_bytes = encode bytes_to_str = decode cast_bytes_py2 = no_code cast_unicode_py2 = no_code buffer_to_bytes_py2 = no_code string_types = (str,) unicode_type = str else: str_to_unicode = decode unicode_to_str = encode str_to_bytes = no_code bytes_to_str = no_code cast_bytes_py2 = cast_bytes cast_unicode_py2 = cast_unicode buffer_to_bytes_py2 = buffer_to_bytes string_types = (str, unicode) unicode_type = unicode if sys.version_info[0] >= 3: PY3 = True # keep reference to builtin_mod because the kernel overrides that value # to forward requests to a frontend. def input(prompt=''): return builtin_mod.input(prompt) builtin_mod_name = "builtins" import builtins as builtin_mod which = shutil.which def isidentifier(s, dotted=False): if dotted: return all(isidentifier(a) for a in s.split(".")) return s.isidentifier() xrange = range def iteritems(d): return iter(d.items()) def itervalues(d): return iter(d.values()) getcwd = os.getcwd MethodType = types.MethodType def execfile(fname, glob, loc=None, compiler=None): loc = loc if (loc is not None) else glob with open(fname, 'rb') as f: compiler = compiler or compile exec(compiler(f.read(), fname, 'exec'), glob, loc) # Refactor print statements in doctests. _print_statement_re = re.compile(r"\bprint (?P.*)$", re.MULTILINE) def _print_statement_sub(match): expr = match.groups('expr') return "print(%s)" % expr @_modify_str_or_docstring def doctest_refactor_print(doc): """Refactor 'print x' statements in a doctest to print(x) style. 2to3 unfortunately doesn't pick up on our doctests. Can accept a string or a function, so it can be used as a decorator.""" return _print_statement_re.sub(_print_statement_sub, doc) # Abstract u'abc' syntax: @_modify_str_or_docstring def u_format(s): """"{u}'abc'" --> "'abc'" (Python 3) Accepts a string or a function, so it can be used as a decorator.""" return s.format(u='') def get_closure(f): """Get a function's closure attribute""" return f.__closure__ else: PY3 = False # keep reference to builtin_mod because the kernel overrides that value # to forward requests to a frontend. def input(prompt=''): return builtin_mod.raw_input(prompt) builtin_mod_name = "__builtin__" import __builtin__ as builtin_mod import re _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$") def isidentifier(s, dotted=False): if dotted: return all(isidentifier(a) for a in s.split(".")) return bool(_name_re.match(s)) xrange = xrange def iteritems(d): return d.iteritems() def itervalues(d): return d.itervalues() getcwd = os.getcwdu def MethodType(func, instance): return types.MethodType(func, instance, type(instance)) def doctest_refactor_print(func_or_str): return func_or_str def get_closure(f): """Get a function's closure attribute""" return f.func_closure which = _shutil_which # Abstract u'abc' syntax: @_modify_str_or_docstring def u_format(s): """"{u}'abc'" --> "u'abc'" (Python 2) Accepts a string or a function, so it can be used as a decorator.""" return s.format(u='u') if sys.platform == 'win32': def execfile(fname, glob=None, loc=None, compiler=None): loc = loc if (loc is not None) else glob scripttext = builtin_mod.open(fname).read()+ '\n' # compile converts unicode filename to str assuming # ascii. Let's do the conversion before calling compile if isinstance(fname, unicode): filename = unicode_to_str(fname) else: filename = fname compiler = compiler or compile exec(compiler(scripttext, filename, 'exec'), glob, loc) else: def execfile(fname, glob=None, loc=None, compiler=None): if isinstance(fname, unicode): filename = fname.encode(sys.getfilesystemencoding()) else: filename = fname where = [ns for ns in [glob, loc] if ns is not None] if compiler is None: builtin_mod.execfile(filename, *where) else: scripttext = builtin_mod.open(fname).read().rstrip() + '\n' exec(compiler(scripttext, filename, 'exec'), glob, loc) def annotate(**kwargs): """Python 3 compatible function annotation for Python 2.""" if not kwargs: raise ValueError('annotations must be provided as keyword arguments') def dec(f): if hasattr(f, '__annotations__'): for k, v in kwargs.items(): f.__annotations__[k] = v else: f.__annotations__ = kwargs return f return dec # Parts below taken from six: # Copyright (c) 2010-2013 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. def with_metaclass(meta, *bases): """Create a base class with a metaclass.""" return meta("_NewBase", bases, {}) ================================================ FILE: lib/client/ipython_genutils/tempdir.py ================================================ """TemporaryDirectory class, copied from Python 3 This is copied from the stdlib and will be standard in Python 3.2 and onwards. """ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import print_function import os as _os import warnings as _warnings import sys as _sys # This code should only be used in Python versions < 3.2, since after that we # can rely on the stdlib itself. try: from tempfile import TemporaryDirectory except ImportError: from tempfile import mkdtemp, template class TemporaryDirectory(object): """Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For example: with TemporaryDirectory() as tmpdir: ... Upon exiting the context, the directory and everthing contained in it are removed. """ def __init__(self, suffix="", prefix=template, dir=None): self.name = mkdtemp(suffix, prefix, dir) self._closed = False def __enter__(self): return self.name def cleanup(self, _warn=False): if self.name and not self._closed: try: self._rmtree(self.name) except (TypeError, AttributeError) as ex: # Issue #10188: Emit a warning on stderr # if the directory could not be cleaned # up due to missing globals if "None" not in str(ex): raise print("ERROR: {!r} while cleaning up {!r}".format(ex, self,), file=_sys.stderr) return self._closed = True if _warn: self._warn("Implicitly cleaning up {!r}".format(self), Warning) def __exit__(self, exc, value, tb): self.cleanup() def __del__(self): # Issue a ResourceWarning if implicit cleanup needed self.cleanup(_warn=True) # XXX (ncoghlan): The following code attempts to make # this class tolerant of the module nulling out process # that happens during CPython interpreter shutdown # Alas, it doesn't actually manage it. See issue #10188 _listdir = staticmethod(_os.listdir) _path_join = staticmethod(_os.path.join) _isdir = staticmethod(_os.path.isdir) _remove = staticmethod(_os.remove) _rmdir = staticmethod(_os.rmdir) _os_error = _os.error _warn = _warnings.warn def _rmtree(self, path): # Essentially a stripped down version of shutil.rmtree. We can't # use globals because they may be None'ed out at shutdown. for name in self._listdir(path): fullname = self._path_join(path, name) try: isdir = self._isdir(fullname) except self._os_error: isdir = False if isdir: self._rmtree(fullname) else: try: self._remove(fullname) except self._os_error: pass try: self._rmdir(path) except self._os_error: pass # extra temp-dir-related context managers class NamedFileInTemporaryDirectory(object): def __init__(self, filename, mode='w+b', bufsize=-1, **kwds): """ Open a file named `filename` in a temporary directory. This context manager is preferred over `NamedTemporaryFile` in stdlib `tempfile` when one needs to reopen the file. Arguments `mode` and `bufsize` are passed to `open`. Rest of the arguments are passed to `TemporaryDirectory`. """ self._tmpdir = TemporaryDirectory(**kwds) path = _os.path.join(self._tmpdir.name, filename) self.file = open(path, mode, bufsize) def cleanup(self): self.file.close() self._tmpdir.cleanup() __del__ = cleanup def __enter__(self): return self.file def __exit__(self, type, value, traceback): self.cleanup() class TemporaryWorkingDirectory(TemporaryDirectory): """ Creates a temporary directory and sets the cwd to that directory. Automatically reverts to previous cwd upon cleanup. Usage example: with TemporaryWorkingDirectory() as tmpdir: ... """ def __enter__(self): self.old_wd = _os.getcwd() _os.chdir(self.name) return super(TemporaryWorkingDirectory, self).__enter__() def __exit__(self, exc, value, tb): _os.chdir(self.old_wd) return super(TemporaryWorkingDirectory, self).__exit__(exc, value, tb) ================================================ FILE: lib/client/ipython_genutils/text.py ================================================ # encoding: utf-8 """ Utilities for working with strings and text. Inheritance diagram: .. inheritance-diagram:: IPython.utils.text :parts: 3 """ import os import re import sys import textwrap from string import Formatter # datetime.strftime date format for ipython if sys.platform == 'win32': date_format = "%B %d, %Y" else: date_format = "%B %-d, %Y" def indent(instr,nspaces=4, ntabs=0, flatten=False): """Indent a string a given number of spaces or tabstops. indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces. Parameters ---------- instr : basestring The string to be indented. nspaces : int (default: 4) The number of spaces to be indented. ntabs : int (default: 0) The number of tabs to be indented. flatten : bool (default: False) Whether to scrub existing indentation. If True, all lines will be aligned to the same indentation. If False, existing indentation will be strictly increased. Returns ------- str|unicode : string indented by ntabs and nspaces. """ if instr is None: return ind = '\t'*ntabs+' '*nspaces if flatten: pat = re.compile(r'^\s*', re.MULTILINE) else: pat = re.compile(r'^', re.MULTILINE) outstr = re.sub(pat, ind, instr) if outstr.endswith(os.linesep+ind): return outstr[:-len(ind)] else: return outstr def dedent(text): """Equivalent of textwrap.dedent that ignores unindented first line. This means it will still dedent strings like: '''foo is a bar ''' For use in wrap_paragraphs. """ if text.startswith('\n'): # text starts with blank line, don't ignore the first line return textwrap.dedent(text) # split first line splits = text.split('\n',1) if len(splits) == 1: # only one line return textwrap.dedent(text) first, rest = splits # dedent everything but the first line rest = textwrap.dedent(rest) return '\n'.join([first, rest]) def wrap_paragraphs(text, ncols=80): """Wrap multiple paragraphs to fit a specified width. This is equivalent to textwrap.wrap, but with support for multiple paragraphs, as separated by empty lines. Returns ------- list of complete paragraphs, wrapped to fill `ncols` columns. """ paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE) text = dedent(text).strip() paragraphs = paragraph_re.split(text)[::2] # every other entry is space out_ps = [] indent_re = re.compile(r'\n\s+', re.MULTILINE) for p in paragraphs: # presume indentation that survives dedent is meaningful formatting, # so don't fill unless text is flush. if indent_re.search(p) is None: # wrap paragraph p = textwrap.fill(p, ncols) out_ps.append(p) return out_ps def strip_ansi(source): """ Remove ansi escape codes from text. Parameters ---------- source : str Source to remove the ansi from """ return re.sub(r'\033\[(\d|;)+?m', '', source) #----------------------------------------------------------------------------- # Utils to columnize a list of string #----------------------------------------------------------------------------- def _chunks(l, n): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i+n] def _find_optimal(rlist , separator_size=2 , displaywidth=80): """Calculate optimal info to columnize a list of string""" for nrow in range(1, len(rlist)+1) : chk = list(map(max,_chunks(rlist, nrow))) sumlength = sum(chk) ncols = len(chk) if sumlength+separator_size*(ncols-1) <= displaywidth : break; return {'columns_numbers' : ncols, 'optimal_separator_width':(displaywidth - sumlength)/(ncols-1) if (ncols -1) else 0, 'rows_numbers' : nrow, 'columns_width' : chk } def _get_or_default(mylist, i, default=None): """return list item number, or default if don't exist""" if i >= len(mylist): return default else : return mylist[i] def compute_item_matrix(items, empty=None, *args, **kwargs) : """Returns a nested list, and info to columnize items Parameters ---------- items list of strings to columize empty : (default None) default value to fill list if needed separator_size : int (default=2) How much caracters will be used as a separation between each columns. displaywidth : int (default=80) The width of the area onto wich the columns should enter Returns ------- strings_matrix nested list of string, the outer most list contains as many list as rows, the innermost lists have each as many element as colums. If the total number of elements in `items` does not equal the product of rows*columns, the last element of some lists are filled with `None`. dict_info some info to make columnize easier: columns_numbers number of columns rows_numbers number of rows columns_width list of with of each columns optimal_separator_width best separator width between columns Examples -------- :: In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l'] ...: compute_item_matrix(l,displaywidth=12) Out[1]: ([['aaa', 'f', 'k'], ['b', 'g', 'l'], ['cc', 'h', None], ['d', 'i', None], ['eeeee', 'j', None]], {'columns_numbers': 3, 'columns_width': [5, 1, 1], 'optimal_separator_width': 2, 'rows_numbers': 5}) """ info = _find_optimal(list(map(len, items)), *args, **kwargs) nrow, ncol = info['rows_numbers'], info['columns_numbers'] return ([[ _get_or_default(items, c*nrow+i, default=empty) for c in range(ncol) ] for i in range(nrow) ], info) def columnize(items, separator=' ', displaywidth=80): """ Transform a list of strings into a single string with columns. Parameters ---------- items : sequence of strings The strings to process. separator : str, optional [default is two spaces] The string that separates columns. displaywidth : int, optional [default is 80] Width of the display in number of characters. Returns ------- The formatted string. """ if not items : return '\n' matrix, info = compute_item_matrix(items, separator_size=len(separator), displaywidth=displaywidth) fmatrix = [filter(None, x) for x in matrix] sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['columns_width'])]) return '\n'.join(map(sjoin, fmatrix))+'\n' ================================================ FILE: lib/client/jupyter_client/__init__.py ================================================ """Client-side implementations of the Jupyter protocol""" from ._version import version_info, __version__, protocol_version_info, protocol_version from .connect import * from .launcher import * from .client import KernelClient from .manager import KernelManager, run_kernel from .blocking import BlockingKernelClient from .multikernelmanager import MultiKernelManager ================================================ FILE: lib/client/jupyter_client/_version.py ================================================ version_info = (5, 2, 4) __version__ = '.'.join(map(str, version_info)) protocol_version_info = (5, 3) protocol_version = "%i.%i" % protocol_version_info ================================================ FILE: lib/client/jupyter_client/adapter.py ================================================ """Adapters for Jupyter msg spec versions.""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import re import json from jupyter_client import protocol_version_info def code_to_line(code, cursor_pos): """Turn a multiline code block and cursor position into a single line and new cursor position. For adapting ``complete_`` and ``object_info_request``. """ if not code: return "", 0 for line in code.splitlines(True): n = len(line) if cursor_pos > n: cursor_pos -= n else: break return line, cursor_pos _match_bracket = re.compile(r'\([^\(\)]+\)', re.UNICODE) _end_bracket = re.compile(r'\([^\(]*$', re.UNICODE) _identifier = re.compile(r'[a-z_][0-9a-z._]*', re.I|re.UNICODE) def extract_oname_v4(code, cursor_pos): """Reimplement token-finding logic from IPython 2.x javascript for adapting object_info_request from v5 to v4 """ line, _ = code_to_line(code, cursor_pos) oldline = line line = _match_bracket.sub(u'', line) while oldline != line: oldline = line line = _match_bracket.sub(u'', line) # remove everything after last open bracket line = _end_bracket.sub('', line) matches = _identifier.findall(line) if matches: return matches[-1] else: return '' class Adapter(object): """Base class for adapting messages Override message_type(msg) methods to create adapters. """ msg_type_map = {} def update_header(self, msg): return msg def update_metadata(self, msg): return msg def update_msg_type(self, msg): header = msg['header'] msg_type = header['msg_type'] if msg_type in self.msg_type_map: msg['msg_type'] = header['msg_type'] = self.msg_type_map[msg_type] return msg def handle_reply_status_error(self, msg): """This will be called *instead of* the regular handler on any reply with status != ok """ return msg def __call__(self, msg): msg = self.update_header(msg) msg = self.update_metadata(msg) msg = self.update_msg_type(msg) header = msg['header'] handler = getattr(self, header['msg_type'], None) if handler is None: return msg # handle status=error replies separately (no change, at present) if msg['content'].get('status', None) in {'error', 'aborted'}: return self.handle_reply_status_error(msg) return handler(msg) def _version_str_to_list(version): """convert a version string to a list of ints non-int segments are excluded """ v = [] for part in version.split('.'): try: v.append(int(part)) except ValueError: pass return v class V5toV4(Adapter): """Adapt msg protocol v5 to v4""" version = '4.1' msg_type_map = { 'execute_result' : 'pyout', 'execute_input' : 'pyin', 'error' : 'pyerr', 'inspect_request' : 'object_info_request', 'inspect_reply' : 'object_info_reply', } def update_header(self, msg): msg['header'].pop('version', None) msg['parent_header'].pop('version', None) return msg # shell channel def kernel_info_reply(self, msg): v4c = {} content = msg['content'] for key in ('language_version', 'protocol_version'): if key in content: v4c[key] = _version_str_to_list(content[key]) if content.get('implementation', '') == 'ipython' \ and 'implementation_version' in content: v4c['ipython_version'] = _version_str_to_list(content['implementation_version']) language_info = content.get('language_info', {}) language = language_info.get('name', '') v4c.setdefault('language', language) if 'version' in language_info: v4c.setdefault('language_version', _version_str_to_list(language_info['version'])) msg['content'] = v4c return msg def execute_request(self, msg): content = msg['content'] content.setdefault('user_variables', []) return msg def execute_reply(self, msg): content = msg['content'] content.setdefault('user_variables', {}) # TODO: handle payloads return msg def complete_request(self, msg): content = msg['content'] code = content['code'] cursor_pos = content['cursor_pos'] line, cursor_pos = code_to_line(code, cursor_pos) new_content = msg['content'] = {} new_content['text'] = '' new_content['line'] = line new_content['block'] = None new_content['cursor_pos'] = cursor_pos return msg def complete_reply(self, msg): content = msg['content'] cursor_start = content.pop('cursor_start') cursor_end = content.pop('cursor_end') match_len = cursor_end - cursor_start content['matched_text'] = content['matches'][0][:match_len] content.pop('metadata', None) return msg def object_info_request(self, msg): content = msg['content'] code = content['code'] cursor_pos = content['cursor_pos'] line, _ = code_to_line(code, cursor_pos) new_content = msg['content'] = {} new_content['oname'] = extract_oname_v4(code, cursor_pos) new_content['detail_level'] = content['detail_level'] return msg def object_info_reply(self, msg): """inspect_reply can't be easily backward compatible""" msg['content'] = {'found' : False, 'oname' : 'unknown'} return msg # iopub channel def stream(self, msg): content = msg['content'] content['data'] = content.pop('text') return msg def display_data(self, msg): content = msg['content'] content.setdefault("source", "display") data = content['data'] if 'application/json' in data: try: data['application/json'] = json.dumps(data['application/json']) except Exception: # warn? pass return msg # stdin channel def input_request(self, msg): msg['content'].pop('password', None) return msg class V4toV5(Adapter): """Convert msg spec V4 to V5""" version = '5.0' # invert message renames above msg_type_map = {v:k for k,v in V5toV4.msg_type_map.items()} def update_header(self, msg): msg['header']['version'] = self.version if msg['parent_header']: msg['parent_header']['version'] = self.version return msg # shell channel def kernel_info_reply(self, msg): content = msg['content'] for key in ('protocol_version', 'ipython_version'): if key in content: content[key] = '.'.join(map(str, content[key])) content.setdefault('protocol_version', '4.1') if content['language'].startswith('python') and 'ipython_version' in content: content['implementation'] = 'ipython' content['implementation_version'] = content.pop('ipython_version') language = content.pop('language') language_info = content.setdefault('language_info', {}) language_info.setdefault('name', language) if 'language_version' in content: language_version = '.'.join(map(str, content.pop('language_version'))) language_info.setdefault('version', language_version) content['banner'] = '' return msg def execute_request(self, msg): content = msg['content'] user_variables = content.pop('user_variables', []) user_expressions = content.setdefault('user_expressions', {}) for v in user_variables: user_expressions[v] = v return msg def execute_reply(self, msg): content = msg['content'] user_expressions = content.setdefault('user_expressions', {}) user_variables = content.pop('user_variables', {}) if user_variables: user_expressions.update(user_variables) # Pager payloads became a mime bundle for payload in content.get('payload', []): if payload.get('source', None) == 'page' and ('text' in payload): if 'data' not in payload: payload['data'] = {} payload['data']['text/plain'] = payload.pop('text') return msg def complete_request(self, msg): old_content = msg['content'] new_content = msg['content'] = {} new_content['code'] = old_content['line'] new_content['cursor_pos'] = old_content['cursor_pos'] return msg def complete_reply(self, msg): # complete_reply needs more context than we have to get cursor_start and end. # use special end=null to indicate current cursor position and negative offset # for start relative to the cursor. # start=None indicates that start == end (accounts for no -0). content = msg['content'] new_content = msg['content'] = {'status' : 'ok'} new_content['matches'] = content['matches'] if content['matched_text']: new_content['cursor_start'] = -len(content['matched_text']) else: # no -0, use None to indicate that start == end new_content['cursor_start'] = None new_content['cursor_end'] = None new_content['metadata'] = {} return msg def inspect_request(self, msg): content = msg['content'] name = content['oname'] new_content = msg['content'] = {} new_content['code'] = name new_content['cursor_pos'] = len(name) new_content['detail_level'] = content['detail_level'] return msg def inspect_reply(self, msg): """inspect_reply can't be easily backward compatible""" content = msg['content'] new_content = msg['content'] = {'status' : 'ok'} found = new_content['found'] = content['found'] new_content['data'] = data = {} new_content['metadata'] = {} if found: lines = [] for key in ('call_def', 'init_definition', 'definition'): if content.get(key, False): lines.append(content[key]) break for key in ('call_docstring', 'init_docstring', 'docstring'): if content.get(key, False): lines.append(content[key]) break if not lines: lines.append("") data['text/plain'] = '\n'.join(lines) return msg # iopub channel def stream(self, msg): content = msg['content'] content['text'] = content.pop('data') return msg def display_data(self, msg): content = msg['content'] content.pop("source", None) data = content['data'] if 'application/json' in data: try: data['application/json'] = json.loads(data['application/json']) except Exception: # warn? pass return msg # stdin channel def input_request(self, msg): msg['content'].setdefault('password', False) return msg def adapt(msg, to_version=protocol_version_info[0]): """Adapt a single message to a target version Parameters ---------- msg : dict A Jupyter message. to_version : int, optional The target major version. If unspecified, adapt to the current version. Returns ------- msg : dict A Jupyter message appropriate in the new version. """ from .session import utcnow header = msg['header'] if 'date' not in header: header['date'] = utcnow() if 'version' in header: from_version = int(header['version'].split('.')[0]) else: # assume last version before adding the key to the header from_version = 4 adapter = adapters.get((from_version, to_version), None) if adapter is None: return msg return adapter(msg) # one adapter per major version from,to adapters = { (5,4) : V5toV4(), (4,5) : V4toV5(), } ================================================ FILE: lib/client/jupyter_client/blocking/__init__.py ================================================ from .client import BlockingKernelClient ================================================ FILE: lib/client/jupyter_client/blocking/channels.py ================================================ """Blocking channels Useful for test suites and blocking terminal interfaces. """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. try: from queue import Queue, Empty # Py 3 except ImportError: from Queue import Queue, Empty # Py 2 class ZMQSocketChannel(object): """A ZMQ socket in a simple blocking API""" session = None socket = None stream = None _exiting = False proxy_methods = [] def __init__(self, socket, session, loop=None): """Create a channel. Parameters ---------- socket : :class:`zmq.Socket` The ZMQ socket to use. session : :class:`session.Session` The session to use. loop Unused here, for other implementations """ super(ZMQSocketChannel, self).__init__() self.socket = socket self.session = session def _recv(self, **kwargs): msg = self.socket.recv_multipart(**kwargs) ident,smsg = self.session.feed_identities(msg) return self.session.deserialize(smsg) def get_msg(self, block=True, timeout=None): """ Gets a message if there is one that is ready. """ if block: if timeout is not None: timeout *= 1000 # seconds to ms ready = self.socket.poll(timeout) else: ready = self.socket.poll(timeout=0) if ready: return self._recv() else: raise Empty def get_msgs(self): """ Get all messages that are currently ready. """ msgs = [] while True: try: msgs.append(self.get_msg(block=False)) except Empty: break return msgs def msg_ready(self): """ Is there a message that has been received? """ return bool(self.socket.poll(timeout=0)) def close(self): if self.socket is not None: try: self.socket.close(linger=0) except Exception: pass self.socket = None stop = close def is_alive(self): return (self.socket is not None) def send(self, msg): """Pass a message to the ZMQ socket to send """ self.session.send(self.socket, msg) def start(self): pass ================================================ FILE: lib/client/jupyter_client/blocking/client.py ================================================ """Implements a fully blocking kernel client. Useful for test suites and blocking terminal interfaces. """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import print_function from functools import partial from getpass import getpass try: from queue import Empty # Python 3 except ImportError: from Queue import Empty # Python 2 import sys import time import zmq from traitlets import Type from jupyter_client.channels import HBChannel from jupyter_client.client import KernelClient from .channels import ZMQSocketChannel try: monotonic = time.monotonic except AttributeError: # py2 monotonic = time.time # close enough try: TimeoutError except NameError: # py2 TimeoutError = RuntimeError def reqrep(meth): def wrapped(self, *args, **kwargs): reply = kwargs.pop('reply', False) timeout = kwargs.pop('timeout', None) msg_id = meth(self, *args, **kwargs) if not reply: return msg_id return self._recv_reply(msg_id, timeout=timeout) if not meth.__doc__: # python -OO removes docstrings, # so don't bother building the wrapped docstring return wrapped basedoc, _ = meth.__doc__.split('Returns\n', 1) parts = [basedoc.strip()] if 'Parameters' not in basedoc: parts.append(""" Parameters ---------- """) parts.append(""" reply: bool (default: False) Whether to wait for and return reply timeout: float or None (default: None) Timeout to use when waiting for a reply Returns ------- msg_id: str The msg_id of the request sent, if reply=False (default) reply: dict The reply message for this request, if reply=True """) wrapped.__doc__ = '\n'.join(parts) return wrapped class BlockingKernelClient(KernelClient): """A KernelClient with blocking APIs ``get_[channel]_msg()`` methods wait for and return messages on channels, raising :exc:`queue.Empty` if no message arrives within ``timeout`` seconds. """ def wait_for_ready(self, timeout=None): """Waits for a response when a client is blocked - Sets future time for timeout - Blocks on shell channel until a message is received - Exit if the kernel has died - If client times out before receiving a message from the kernel, send RuntimeError - Flush the IOPub channel """ if timeout is None: abs_timeout = float('inf') else: abs_timeout = time.time() + timeout from ..manager import KernelManager if not isinstance(self.parent, KernelManager): # This Client was not created by a KernelManager, # so wait for kernel to become responsive to heartbeats # before checking for kernel_info reply while not self.is_alive(): if time.time() > abs_timeout: raise RuntimeError("Kernel didn't respond to heartbeats in %d seconds and timed out" % timeout) time.sleep(0.2) # Wait for kernel info reply on shell channel while True: try: msg = self.shell_channel.get_msg(block=True, timeout=1) except Empty: pass else: if msg['msg_type'] == 'kernel_info_reply': self._handle_kernel_info_reply(msg) break if not self.is_alive(): raise RuntimeError('Kernel died before replying to kernel_info') # Check if current time is ready check time plus timeout if time.time() > abs_timeout: raise RuntimeError("Kernel didn't respond in %d seconds" % timeout) # Flush IOPub channel while True: try: msg = self.iopub_channel.get_msg(block=True, timeout=0.2) except Empty: break # The classes to use for the various channels shell_channel_class = Type(ZMQSocketChannel) iopub_channel_class = Type(ZMQSocketChannel) stdin_channel_class = Type(ZMQSocketChannel) hb_channel_class = Type(HBChannel) def _recv_reply(self, msg_id, timeout=None): """Receive and return the reply for a given request""" if timeout is not None: deadline = monotonic() + timeout while True: if timeout is not None: timeout = max(0, deadline - monotonic()) try: reply = self.get_shell_msg(timeout=timeout) except Empty: raise TimeoutError("Timeout waiting for reply") if reply['parent_header'].get('msg_id') != msg_id: # not my reply, someone may have forgotten to retrieve theirs continue return reply execute = reqrep(KernelClient.execute) history = reqrep(KernelClient.history) complete = reqrep(KernelClient.complete) inspect = reqrep(KernelClient.inspect) kernel_info = reqrep(KernelClient.kernel_info) comm_info = reqrep(KernelClient.comm_info) shutdown = reqrep(KernelClient.shutdown) def _stdin_hook_default(self, msg): """Handle an input request""" content = msg['content'] if content.get('password', False): prompt = getpass elif sys.version_info < (3,): prompt = raw_input else: prompt = input try: raw_data = prompt(content["prompt"]) except EOFError: # turn EOFError into EOF character raw_data = '\x04' except KeyboardInterrupt: sys.stdout.write('\n') return # only send stdin reply if there *was not* another request # or execution finished while we were reading. if not (self.stdin_channel.msg_ready() or self.shell_channel.msg_ready()): self.input(raw_data) def _output_hook_default(self, msg): """Default hook for redisplaying plain-text output""" msg_type = msg['header']['msg_type'] content = msg['content'] if msg_type == 'stream': stream = getattr(sys, content['name']) stream.write(content['text']) elif msg_type in ('display_data', 'execute_result'): sys.stdout.write(content['data'].get('text/plain', '')) elif msg_type == 'error': print('\n'.join(content['traceback']), file=sys.stderr) def _output_hook_kernel(self, session, socket, parent_header, msg): """Output hook when running inside an IPython kernel adds rich output support. """ msg_type = msg['header']['msg_type'] if msg_type in ('display_data', 'execute_result', 'error'): session.send(socket, msg_type, msg['content'], parent=parent_header) else: self._output_hook_default(msg) def execute_interactive(self, code, silent=False, store_history=True, user_expressions=None, allow_stdin=None, stop_on_error=True, timeout=None, output_hook=None, stdin_hook=None, ): """Execute code in the kernel interactively Output will be redisplayed, and stdin prompts will be relayed as well. If an IPython kernel is detected, rich output will be displayed. You can pass a custom output_hook callable that will be called with every IOPub message that is produced instead of the default redisplay. .. versionadded:: 5.0 Parameters ---------- code : str A string of code in the kernel's language. silent : bool, optional (default False) If set, the kernel will execute the code as quietly possible, and will force store_history to be False. store_history : bool, optional (default True) If set, the kernel will store command history. This is forced to be False if silent is True. user_expressions : dict, optional A dict mapping names to expressions to be evaluated in the user's dict. The expression values are returned as strings formatted using :func:`repr`. allow_stdin : bool, optional (default self.allow_stdin) Flag for whether the kernel can send stdin requests to frontends. Some frontends (e.g. the Notebook) do not support stdin requests. If raw_input is called from code executed from such a frontend, a StdinNotImplementedError will be raised. stop_on_error: bool, optional (default True) Flag whether to abort the execution queue, if an exception is encountered. timeout: float or None (default: None) Timeout to use when waiting for a reply output_hook: callable(msg) Function to be called with output messages. If not specified, output will be redisplayed. stdin_hook: callable(msg) Function to be called with stdin_request messages. If not specified, input/getpass will be called. Returns ------- reply: dict The reply message for this request """ if not self.iopub_channel.is_alive(): raise RuntimeError("IOPub channel must be running to receive output") if allow_stdin is None: allow_stdin = self.allow_stdin if allow_stdin and not self.stdin_channel.is_alive(): raise RuntimeError("stdin channel must be running to allow input") msg_id = self.execute(code, silent=silent, store_history=store_history, user_expressions=user_expressions, allow_stdin=allow_stdin, stop_on_error=stop_on_error, ) if stdin_hook is None: stdin_hook = self._stdin_hook_default if output_hook is None: # detect IPython kernel if 'IPython' in sys.modules: from IPython import get_ipython ip = get_ipython() in_kernel = getattr(ip, 'kernel', False) if in_kernel: output_hook = partial( self._output_hook_kernel, ip.display_pub.session, ip.display_pub.pub_socket, ip.display_pub.parent_header, ) if output_hook is None: # default: redisplay plain-text outputs output_hook = self._output_hook_default # set deadline based on timeout if timeout is not None: deadline = monotonic() + timeout else: timeout_ms = None poller = zmq.Poller() iopub_socket = self.iopub_channel.socket poller.register(iopub_socket, zmq.POLLIN) if allow_stdin: stdin_socket = self.stdin_channel.socket poller.register(stdin_socket, zmq.POLLIN) else: stdin_socket = None # wait for output and redisplay it while True: if timeout is not None: timeout = max(0, deadline - monotonic()) timeout_ms = 1e3 * timeout events = dict(poller.poll(timeout_ms)) if not events: raise TimeoutError("Timeout waiting for output") if stdin_socket in events: req = self.stdin_channel.get_msg(timeout=0) stdin_hook(req) continue if iopub_socket not in events: continue msg = self.iopub_channel.get_msg(timeout=0) if msg['parent_header'].get('msg_id') != msg_id: # not from my request continue output_hook(msg) # stop on idle if msg['header']['msg_type'] == 'status' and \ msg['content']['execution_state'] == 'idle': break # output is done, get the reply if timeout is not None: timeout = max(0, deadline - monotonic()) return self._recv_reply(msg_id, timeout=timeout) ================================================ FILE: lib/client/jupyter_client/channels.py ================================================ """Base classes to manage a Client's interaction with a running kernel""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import absolute_import import atexit import errno from threading import Thread import time import zmq # import ZMQError in top-level namespace, to avoid ugly attribute-error messages # during garbage collection of threads at exit: from zmq import ZMQError from jupyter_client import protocol_version_info from .channelsabc import HBChannelABC #----------------------------------------------------------------------------- # Constants and exceptions #----------------------------------------------------------------------------- major_protocol_version = protocol_version_info[0] class InvalidPortNumber(Exception): pass class HBChannel(Thread): """The heartbeat channel which monitors the kernel heartbeat. Note that the heartbeat channel is paused by default. As long as you start this channel, the kernel manager will ensure that it is paused and un-paused as appropriate. """ context = None session = None socket = None address = None _exiting = False time_to_dead = 1. poller = None _running = None _pause = None _beating = None def __init__(self, context=None, session=None, address=None): """Create the heartbeat monitor thread. Parameters ---------- context : :class:`zmq.Context` The ZMQ context to use. session : :class:`session.Session` The session to use. address : zmq url Standard (ip, port) tuple that the kernel is listening on. """ super(HBChannel, self).__init__() self.daemon = True self.context = context self.session = session if isinstance(address, tuple): if address[1] == 0: message = 'The port number for a channel cannot be 0.' raise InvalidPortNumber(message) address = "tcp://%s:%i" % address self.address = address # running is False until `.start()` is called self._running = False # don't start paused self._pause = False self.poller = zmq.Poller() @staticmethod @atexit.register def _notice_exit(): # Class definitions can be torn down during interpreter shutdown. # We only need to set _exiting flag if this hasn't happened. if HBChannel is not None: HBChannel._exiting = True def _create_socket(self): if self.socket is not None: # close previous socket, before opening a new one self.poller.unregister(self.socket) self.socket.close() self.socket = self.context.socket(zmq.REQ) self.socket.linger = 1000 self.socket.connect(self.address) self.poller.register(self.socket, zmq.POLLIN) def _poll(self, start_time): """poll for heartbeat replies until we reach self.time_to_dead. Ignores interrupts, and returns the result of poll(), which will be an empty list if no messages arrived before the timeout, or the event tuple if there is a message to receive. """ until_dead = self.time_to_dead - (time.time() - start_time) # ensure poll at least once until_dead = max(until_dead, 1e-3) events = [] while True: try: events = self.poller.poll(1000 * until_dead) except ZMQError as e: if e.errno == errno.EINTR: # ignore interrupts during heartbeat # this may never actually happen until_dead = self.time_to_dead - (time.time() - start_time) until_dead = max(until_dead, 1e-3) pass else: raise except Exception: if self._exiting: break else: raise else: break return events def run(self): """The thread's main activity. Call start() instead.""" self._create_socket() self._running = True self._beating = True while self._running: if self._pause: # just sleep, and skip the rest of the loop time.sleep(self.time_to_dead) continue since_last_heartbeat = 0.0 # io.rprint('Ping from HB channel') # dbg # no need to catch EFSM here, because the previous event was # either a recv or connect, which cannot be followed by EFSM self.socket.send(b'ping') request_time = time.time() ready = self._poll(request_time) if ready: self._beating = True # the poll above guarantees we have something to recv self.socket.recv() # sleep the remainder of the cycle remainder = self.time_to_dead - (time.time() - request_time) if remainder > 0: time.sleep(remainder) continue else: # nothing was received within the time limit, signal heart failure self._beating = False since_last_heartbeat = time.time() - request_time self.call_handlers(since_last_heartbeat) # and close/reopen the socket, because the REQ/REP cycle has been broken self._create_socket() continue def pause(self): """Pause the heartbeat.""" self._pause = True def unpause(self): """Unpause the heartbeat.""" self._pause = False def is_beating(self): """Is the heartbeat running and responsive (and not paused).""" if self.is_alive() and not self._pause and self._beating: return True else: return False def stop(self): """Stop the channel's event loop and join its thread.""" self._running = False self.join() self.close() def close(self): if self.socket is not None: try: self.socket.close(linger=0) except Exception: pass self.socket = None def call_handlers(self, since_last_heartbeat): """This method is called in the ioloop thread when a message arrives. Subclasses should override this method to handle incoming messages. It is important to remember that this method is called in the thread so that some logic must be done to ensure that the application level handlers are called in the application thread. """ pass HBChannelABC.register(HBChannel) ================================================ FILE: lib/client/jupyter_client/channelsabc.py ================================================ """Abstract base classes for kernel client channels""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import abc from ipython_genutils.py3compat import with_metaclass class ChannelABC(with_metaclass(abc.ABCMeta, object)): """A base class for all channel ABCs.""" @abc.abstractmethod def start(self): pass @abc.abstractmethod def stop(self): pass @abc.abstractmethod def is_alive(self): pass class HBChannelABC(ChannelABC): """HBChannel ABC. The docstrings for this class can be found in the base implementation: `jupyter_client.channels.HBChannel` """ @abc.abstractproperty def time_to_dead(self): pass @abc.abstractmethod def pause(self): pass @abc.abstractmethod def unpause(self): pass @abc.abstractmethod def is_beating(self): pass ================================================ FILE: lib/client/jupyter_client/client.py ================================================ """Base class to manage the interaction with a running kernel""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import absolute_import from jupyter_client.channels import major_protocol_version from ipython_genutils.py3compat import string_types, iteritems import zmq from traitlets import ( Any, Instance, Type, ) from .channelsabc import (ChannelABC, HBChannelABC) from .clientabc import KernelClientABC from .connect import ConnectionFileMixin # some utilities to validate message structure, these might get moved elsewhere # if they prove to have more generic utility def validate_string_dict(dct): """Validate that the input is a dict with string keys and values. Raises ValueError if not.""" for k,v in iteritems(dct): if not isinstance(k, string_types): raise ValueError('key %r in dict must be a string' % k) if not isinstance(v, string_types): raise ValueError('value %r in dict must be a string' % v) class KernelClient(ConnectionFileMixin): """Communicates with a single kernel on any host via zmq channels. There are four channels associated with each kernel: * shell: for request/reply calls to the kernel. * iopub: for the kernel to publish results to frontends. * hb: for monitoring the kernel's heartbeat. * stdin: for frontends to reply to raw_input calls in the kernel. The messages that can be sent on these channels are exposed as methods of the client (KernelClient.execute, complete, history, etc.). These methods only send the message, they don't wait for a reply. To get results, use e.g. :meth:`get_shell_msg` to fetch messages from the shell channel. """ # The PyZMQ Context to use for communication with the kernel. context = Instance(zmq.Context) def _context_default(self): return zmq.Context.instance() # The classes to use for the various channels shell_channel_class = Type(ChannelABC) iopub_channel_class = Type(ChannelABC) stdin_channel_class = Type(ChannelABC) hb_channel_class = Type(HBChannelABC) # Protected traits _shell_channel = Any() _iopub_channel = Any() _stdin_channel = Any() _hb_channel = Any() # flag for whether execute requests should be allowed to call raw_input: allow_stdin = True #-------------------------------------------------------------------------- # Channel proxy methods #-------------------------------------------------------------------------- def get_shell_msg(self, *args, **kwargs): """Get a message from the shell channel""" return self.shell_channel.get_msg(*args, **kwargs) def get_iopub_msg(self, *args, **kwargs): """Get a message from the iopub channel""" return self.iopub_channel.get_msg(*args, **kwargs) def get_stdin_msg(self, *args, **kwargs): """Get a message from the stdin channel""" return self.stdin_channel.get_msg(*args, **kwargs) #-------------------------------------------------------------------------- # Channel management methods #-------------------------------------------------------------------------- def start_channels(self, shell=True, iopub=True, stdin=True, hb=True): """Starts the channels for this kernel. This will create the channels if they do not exist and then start them (their activity runs in a thread). If port numbers of 0 are being used (random ports) then you must first call :meth:`start_kernel`. If the channels have been stopped and you call this, :class:`RuntimeError` will be raised. """ if shell: self.shell_channel.start() self.kernel_info() if iopub: self.iopub_channel.start() if stdin: self.stdin_channel.start() self.allow_stdin = True else: self.allow_stdin = False if hb: self.hb_channel.start() def stop_channels(self): """Stops all the running channels for this kernel. This stops their event loops and joins their threads. """ if self.shell_channel.is_alive(): self.shell_channel.stop() if self.iopub_channel.is_alive(): self.iopub_channel.stop() if self.stdin_channel.is_alive(): self.stdin_channel.stop() if self.hb_channel.is_alive(): self.hb_channel.stop() @property def channels_running(self): """Are any of the channels created and running?""" return (self.shell_channel.is_alive() or self.iopub_channel.is_alive() or self.stdin_channel.is_alive() or self.hb_channel.is_alive()) ioloop = None # Overridden in subclasses that use pyzmq event loop @property def shell_channel(self): """Get the shell channel object for this kernel.""" if self._shell_channel is None: url = self._make_url('shell') self.log.debug("connecting shell channel to %s", url) socket = self.connect_shell(identity=self.session.bsession) self._shell_channel = self.shell_channel_class( socket, self.session, self.ioloop ) return self._shell_channel @property def iopub_channel(self): """Get the iopub channel object for this kernel.""" if self._iopub_channel is None: url = self._make_url('iopub') self.log.debug("connecting iopub channel to %s", url) socket = self.connect_iopub() self._iopub_channel = self.iopub_channel_class( socket, self.session, self.ioloop ) return self._iopub_channel @property def stdin_channel(self): """Get the stdin channel object for this kernel.""" if self._stdin_channel is None: url = self._make_url('stdin') self.log.debug("connecting stdin channel to %s", url) socket = self.connect_stdin(identity=self.session.bsession) self._stdin_channel = self.stdin_channel_class( socket, self.session, self.ioloop ) return self._stdin_channel @property def hb_channel(self): """Get the hb channel object for this kernel.""" if self._hb_channel is None: url = self._make_url('hb') self.log.debug("connecting heartbeat channel to %s", url) self._hb_channel = self.hb_channel_class( self.context, self.session, url ) return self._hb_channel def is_alive(self): """Is the kernel process still running?""" from .manager import KernelManager if isinstance(self.parent, KernelManager): # This KernelClient was created by a KernelManager, # we can ask the parent KernelManager: return self.parent.is_alive() if self._hb_channel is not None: # We don't have access to the KernelManager, # so we use the heartbeat. return self._hb_channel.is_beating() else: # no heartbeat and not local, we can't tell if it's running, # so naively return True return True # Methods to send specific messages on channels def execute(self, code, silent=False, store_history=True, user_expressions=None, allow_stdin=None, stop_on_error=True): """Execute code in the kernel. Parameters ---------- code : str A string of code in the kernel's language. silent : bool, optional (default False) If set, the kernel will execute the code as quietly possible, and will force store_history to be False. store_history : bool, optional (default True) If set, the kernel will store command history. This is forced to be False if silent is True. user_expressions : dict, optional A dict mapping names to expressions to be evaluated in the user's dict. The expression values are returned as strings formatted using :func:`repr`. allow_stdin : bool, optional (default self.allow_stdin) Flag for whether the kernel can send stdin requests to frontends. Some frontends (e.g. the Notebook) do not support stdin requests. If raw_input is called from code executed from such a frontend, a StdinNotImplementedError will be raised. stop_on_error: bool, optional (default True) Flag whether to abort the execution queue, if an exception is encountered. Returns ------- The msg_id of the message sent. """ if user_expressions is None: user_expressions = {} if allow_stdin is None: allow_stdin = self.allow_stdin # Don't waste network traffic if inputs are invalid if not isinstance(code, string_types): raise ValueError('code %r must be a string' % code) validate_string_dict(user_expressions) # Create class for content/msg creation. Related to, but possibly # not in Session. content = dict(code=code, silent=silent, store_history=store_history, user_expressions=user_expressions, allow_stdin=allow_stdin, stop_on_error=stop_on_error ) msg = self.session.msg('execute_request', content) self.shell_channel.send(msg) return msg['header']['msg_id'] def complete(self, code, cursor_pos=None): """Tab complete text in the kernel's namespace. Parameters ---------- code : str The context in which completion is requested. Can be anything between a variable name and an entire cell. cursor_pos : int, optional The position of the cursor in the block of code where the completion was requested. Default: ``len(code)`` Returns ------- The msg_id of the message sent. """ if cursor_pos is None: cursor_pos = len(code) content = dict(code=code, cursor_pos=cursor_pos) msg = self.session.msg('complete_request', content) self.shell_channel.send(msg) return msg['header']['msg_id'] def inspect(self, code, cursor_pos=None, detail_level=0): """Get metadata information about an object in the kernel's namespace. It is up to the kernel to determine the appropriate object to inspect. Parameters ---------- code : str The context in which info is requested. Can be anything between a variable name and an entire cell. cursor_pos : int, optional The position of the cursor in the block of code where the info was requested. Default: ``len(code)`` detail_level : int, optional The level of detail for the introspection (0-2) Returns ------- The msg_id of the message sent. """ if cursor_pos is None: cursor_pos = len(code) content = dict(code=code, cursor_pos=cursor_pos, detail_level=detail_level, ) msg = self.session.msg('inspect_request', content) self.shell_channel.send(msg) return msg['header']['msg_id'] def history(self, raw=True, output=False, hist_access_type='range', **kwargs): """Get entries from the kernel's history list. Parameters ---------- raw : bool If True, return the raw input. output : bool If True, then return the output as well. hist_access_type : str 'range' (fill in session, start and stop params), 'tail' (fill in n) or 'search' (fill in pattern param). session : int For a range request, the session from which to get lines. Session numbers are positive integers; negative ones count back from the current session. start : int The first line number of a history range. stop : int The final (excluded) line number of a history range. n : int The number of lines of history to get for a tail request. pattern : str The glob-syntax pattern for a search request. Returns ------- The ID of the message sent. """ if hist_access_type == 'range': kwargs.setdefault('session', 0) kwargs.setdefault('start', 0) content = dict(raw=raw, output=output, hist_access_type=hist_access_type, **kwargs) msg = self.session.msg('history_request', content) self.shell_channel.send(msg) return msg['header']['msg_id'] def kernel_info(self): """Request kernel info Returns ------- The msg_id of the message sent """ msg = self.session.msg('kernel_info_request') self.shell_channel.send(msg) return msg['header']['msg_id'] def comm_info(self, target_name=None): """Request comm info Returns ------- The msg_id of the message sent """ if target_name is None: content = {} else: content = dict(target_name=target_name) msg = self.session.msg('comm_info_request', content) self.shell_channel.send(msg) return msg['header']['msg_id'] def _handle_kernel_info_reply(self, msg): """handle kernel info reply sets protocol adaptation version. This might be run from a separate thread. """ adapt_version = int(msg['content']['protocol_version'].split('.')[0]) if adapt_version != major_protocol_version: self.session.adapt_version = adapt_version def shutdown(self, restart=False): """Request an immediate kernel shutdown. Upon receipt of the (empty) reply, client code can safely assume that the kernel has shut down and it's safe to forcefully terminate it if it's still alive. The kernel will send the reply via a function registered with Python's atexit module, ensuring it's truly done as the kernel is done with all normal operation. Returns ------- The msg_id of the message sent """ # Send quit message to kernel. Once we implement kernel-side setattr, # this should probably be done that way, but for now this will do. msg = self.session.msg('shutdown_request', {'restart':restart}) self.shell_channel.send(msg) return msg['header']['msg_id'] def is_complete(self, code): """Ask the kernel whether some code is complete and ready to execute.""" msg = self.session.msg('is_complete_request', {'code': code}) self.shell_channel.send(msg) return msg['header']['msg_id'] def input(self, string): """Send a string of raw input to the kernel. This should only be called in response to the kernel sending an ``input_request`` message on the stdin channel. """ content = dict(value=string) msg = self.session.msg('input_reply', content) self.stdin_channel.send(msg) KernelClientABC.register(KernelClient) ================================================ FILE: lib/client/jupyter_client/clientabc.py ================================================ """Abstract base class for kernel clients""" #----------------------------------------------------------------------------- # Copyright (c) The Jupyter Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- import abc from ipython_genutils.py3compat import with_metaclass #----------------------------------------------------------------------------- # Main kernel client class #----------------------------------------------------------------------------- class KernelClientABC(with_metaclass(abc.ABCMeta, object)): """KernelManager ABC. The docstrings for this class can be found in the base implementation: `jupyter_client.client.KernelClient` """ @abc.abstractproperty def kernel(self): pass @abc.abstractproperty def shell_channel_class(self): pass @abc.abstractproperty def iopub_channel_class(self): pass @abc.abstractproperty def hb_channel_class(self): pass @abc.abstractproperty def stdin_channel_class(self): pass #-------------------------------------------------------------------------- # Channel management methods #-------------------------------------------------------------------------- @abc.abstractmethod def start_channels(self, shell=True, iopub=True, stdin=True, hb=True): pass @abc.abstractmethod def stop_channels(self): pass @abc.abstractproperty def channels_running(self): pass @abc.abstractproperty def shell_channel(self): pass @abc.abstractproperty def iopub_channel(self): pass @abc.abstractproperty def stdin_channel(self): pass @abc.abstractproperty def hb_channel(self): pass ================================================ FILE: lib/client/jupyter_client/connect.py ================================================ """Utilities for connecting to jupyter kernels The :class:`ConnectionFileMixin` class in this module encapsulates the logic related to writing and reading connections files. """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import absolute_import import errno import glob import json import os import socket import stat import tempfile import warnings from getpass import getpass import zmq from traitlets.config import LoggingConfigurable from .localinterfaces import localhost from ipython_genutils.path import filefind from ipython_genutils.py3compat import ( bytes_to_str, cast_bytes, cast_bytes_py2, string_types, ) from traitlets import ( Bool, Integer, Unicode, CaselessStrEnum, Instance, Type, ) from jupyter_core.paths import jupyter_data_dir, jupyter_runtime_dir def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0, control_port=0, ip='', key=b'', transport='tcp', signature_scheme='hmac-sha256', kernel_name='' ): """Generates a JSON config file, including the selection of random ports. Parameters ---------- fname : unicode The path to the file to write shell_port : int, optional The port to use for ROUTER (shell) channel. iopub_port : int, optional The port to use for the SUB channel. stdin_port : int, optional The port to use for the ROUTER (raw input) channel. control_port : int, optional The port to use for the ROUTER (control) channel. hb_port : int, optional The port to use for the heartbeat REP channel. ip : str, optional The ip address the kernel will bind to. key : str, optional The Session key used for message authentication. signature_scheme : str, optional The scheme used for message authentication. This has the form 'digest-hash', where 'digest' is the scheme used for digests, and 'hash' is the name of the hash function used by the digest scheme. Currently, 'hmac' is the only supported digest scheme, and 'sha256' is the default hash function. kernel_name : str, optional The name of the kernel currently connected to. """ if not ip: ip = localhost() # default to temporary connector file if not fname: fd, fname = tempfile.mkstemp('.json') os.close(fd) # Find open ports as necessary. ports = [] ports_needed = int(shell_port <= 0) + \ int(iopub_port <= 0) + \ int(stdin_port <= 0) + \ int(control_port <= 0) + \ int(hb_port <= 0) if transport == 'tcp': for i in range(ports_needed): sock = socket.socket() # struct.pack('ii', (0,0)) is 8 null bytes sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8) sock.bind((ip, 0)) ports.append(sock) for i, sock in enumerate(ports): port = sock.getsockname()[1] sock.close() ports[i] = port else: N = 1 for i in range(ports_needed): while os.path.exists("%s-%s" % (ip, str(N))): N += 1 ports.append(N) N += 1 if shell_port <= 0: shell_port = ports.pop(0) if iopub_port <= 0: iopub_port = ports.pop(0) if stdin_port <= 0: stdin_port = ports.pop(0) if control_port <= 0: control_port = ports.pop(0) if hb_port <= 0: hb_port = ports.pop(0) cfg = dict( shell_port=shell_port, iopub_port=iopub_port, stdin_port=stdin_port, control_port=control_port, hb_port=hb_port, ) cfg['ip'] = ip cfg['key'] = bytes_to_str(key) cfg['transport'] = transport cfg['signature_scheme'] = signature_scheme cfg['kernel_name'] = kernel_name with open(fname, 'w') as f: f.write(json.dumps(cfg, indent=2)) if hasattr(stat, 'S_ISVTX'): # set the sticky bit on the file and its parent directory # to avoid periodic cleanup paths = [fname] runtime_dir = os.path.dirname(fname) if runtime_dir: paths.append(runtime_dir) for path in paths: permissions = os.stat(path).st_mode new_permissions = permissions | stat.S_ISVTX if new_permissions != permissions: try: os.chmod(path, new_permissions) except OSError as e: if e.errno == errno.EPERM and path == runtime_dir: # suppress permission errors setting sticky bit on runtime_dir, # which we may not own. pass else: # failed to set sticky bit, probably not a big deal warnings.warn( "Failed to set sticky bit on %r: %s" "\nProbably not a big deal, but runtime files may be cleaned up periodically." % (path, e), RuntimeWarning, ) return fname, cfg def find_connection_file(filename='kernel-*.json', path=None, profile=None): """find a connection file, and return its absolute path. The current working directory and optional search path will be searched for the file if it is not given by absolute path. If the argument does not match an existing file, it will be interpreted as a fileglob, and the matching file in the profile's security dir with the latest access time will be used. Parameters ---------- filename : str The connection file or fileglob to search for. path : str or list of strs[optional] Paths in which to search for connection files. Returns ------- str : The absolute path of the connection file. """ if profile is not None: warnings.warn("Jupyter has no profiles. profile=%s has been ignored." % profile) if path is None: path = ['.', jupyter_runtime_dir()] if isinstance(path, string_types): path = [path] try: # first, try explicit name return filefind(filename, path) except IOError: pass # not found by full name if '*' in filename: # given as a glob already pat = filename else: # accept any substring match pat = '*%s*' % filename matches = [] for p in path: matches.extend(glob.glob(os.path.join(p, pat))) matches = [ os.path.abspath(m) for m in matches ] if not matches: raise IOError("Could not find %r in %r" % (filename, path)) elif len(matches) == 1: return matches[0] else: # get most recent match, by access time: return sorted(matches, key=lambda f: os.stat(f).st_atime)[-1] def tunnel_to_kernel(connection_info, sshserver, sshkey=None): """tunnel connections to a kernel via ssh This will open four SSH tunnels from localhost on this machine to the ports associated with the kernel. They can be either direct localhost-localhost tunnels, or if an intermediate server is necessary, the kernel must be listening on a public IP. Parameters ---------- connection_info : dict or str (path) Either a connection dict, or the path to a JSON connection file sshserver : str The ssh sever to use to tunnel to the kernel. Can be a full `user@server:port` string. ssh config aliases are respected. sshkey : str [optional] Path to file containing ssh key to use for authentication. Only necessary if your ssh config does not already associate a keyfile with the host. Returns ------- (shell, iopub, stdin, hb) : ints The four ports on localhost that have been forwarded to the kernel. """ from zmq.ssh import tunnel if isinstance(connection_info, string_types): # it's a path, unpack it with open(connection_info) as f: connection_info = json.loads(f.read()) cf = connection_info lports = tunnel.select_random_ports(4) rports = cf['shell_port'], cf['iopub_port'], cf['stdin_port'], cf['hb_port'] remote_ip = cf['ip'] if tunnel.try_passwordless_ssh(sshserver, sshkey): password=False else: password = getpass("SSH Password for %s: " % cast_bytes_py2(sshserver)) for lp,rp in zip(lports, rports): tunnel.ssh_tunnel(lp, rp, sshserver, remote_ip, sshkey, password) return tuple(lports) #----------------------------------------------------------------------------- # Mixin for classes that work with connection files #----------------------------------------------------------------------------- channel_socket_types = { 'hb' : zmq.REQ, 'shell' : zmq.DEALER, 'iopub' : zmq.SUB, 'stdin' : zmq.DEALER, 'control': zmq.DEALER, } port_names = [ "%s_port" % channel for channel in ('shell', 'stdin', 'iopub', 'hb', 'control')] class ConnectionFileMixin(LoggingConfigurable): """Mixin for configurable classes that work with connection files""" data_dir = Unicode() def _data_dir_default(self): return jupyter_data_dir() # The addresses for the communication channels connection_file = Unicode('', config=True, help="""JSON file in which to store connection info [default: kernel-.json] This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security dir of the current profile, but can be specified by absolute path. """) _connection_file_written = Bool(False) transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True) kernel_name = Unicode() ip = Unicode(config=True, help="""Set the kernel\'s IP address [default localhost]. If the IP address is something other than localhost, then Consoles on other machines will be able to connect to the Kernel, so be careful!""" ) def _ip_default(self): if self.transport == 'ipc': if self.connection_file: return os.path.splitext(self.connection_file)[0] + '-ipc' else: return 'kernel-ipc' else: return localhost() def _ip_changed(self, name, old, new): if new == '*': self.ip = '0.0.0.0' # protected traits hb_port = Integer(0, config=True, help="set the heartbeat port [default: random]") shell_port = Integer(0, config=True, help="set the shell (ROUTER) port [default: random]") iopub_port = Integer(0, config=True, help="set the iopub (PUB) port [default: random]") stdin_port = Integer(0, config=True, help="set the stdin (ROUTER) port [default: random]") control_port = Integer(0, config=True, help="set the control (ROUTER) port [default: random]") # names of the ports with random assignment _random_port_names = None @property def ports(self): return [ getattr(self, name) for name in port_names ] # The Session to use for communication with the kernel. session = Instance('jupyter_client.session.Session') def _session_default(self): from jupyter_client.session import Session return Session(parent=self) #-------------------------------------------------------------------------- # Connection and ipc file management #-------------------------------------------------------------------------- def get_connection_info(self, session=False): """Return the connection info as a dict Parameters ---------- session : bool [default: False] If True, return our session object will be included in the connection info. If False (default), the configuration parameters of our session object will be included, rather than the session object itself. Returns ------- connect_info : dict dictionary of connection information. """ info = dict( transport=self.transport, ip=self.ip, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port, control_port=self.control_port, ) if session: # add *clone* of my session, # so that state such as digest_history is not shared. info['session'] = self.session.clone() else: # add session info info.update(dict( signature_scheme=self.session.signature_scheme, key=self.session.key, )) return info # factory for blocking clients blocking_class = Type(klass=object, default_value='jupyter_client.BlockingKernelClient') def blocking_client(self): """Make a blocking client connected to my kernel""" info = self.get_connection_info() info['parent'] = self bc = self.blocking_class(**info) bc.session.key = self.session.key return bc def cleanup_connection_file(self): """Cleanup connection file *if we wrote it* Will not raise if the connection file was already removed somehow. """ if self._connection_file_written: # cleanup connection files on full shutdown of kernel we started self._connection_file_written = False try: os.remove(self.connection_file) except (IOError, OSError, AttributeError): pass def cleanup_ipc_files(self): """Cleanup ipc files if we wrote them.""" if self.transport != 'ipc': return for port in self.ports: ipcfile = "%s-%i" % (self.ip, port) try: os.remove(ipcfile) except (IOError, OSError): pass def _record_random_port_names(self): """Records which of the ports are randomly assigned. Records on first invocation, if the transport is tcp. Does nothing on later invocations.""" if self.transport != 'tcp': return if self._random_port_names is not None: return self._random_port_names = [] for name in port_names: if getattr(self, name) <= 0: self._random_port_names.append(name) def cleanup_random_ports(self): """Forgets randomly assigned port numbers and cleans up the connection file. Does nothing if no port numbers have been randomly assigned. In particular, does nothing unless the transport is tcp. """ if not self._random_port_names: return for name in self._random_port_names: setattr(self, name, 0) self.cleanup_connection_file() def write_connection_file(self): """Write connection info to JSON dict in self.connection_file.""" if self._connection_file_written and os.path.exists(self.connection_file): return self.connection_file, cfg = write_connection_file(self.connection_file, transport=self.transport, ip=self.ip, key=self.session.key, stdin_port=self.stdin_port, iopub_port=self.iopub_port, shell_port=self.shell_port, hb_port=self.hb_port, control_port=self.control_port, signature_scheme=self.session.signature_scheme, kernel_name=self.kernel_name ) # write_connection_file also sets default ports: self._record_random_port_names() for name in port_names: setattr(self, name, cfg[name]) self._connection_file_written = True def load_connection_file(self, connection_file=None): """Load connection info from JSON dict in self.connection_file. Parameters ---------- connection_file: unicode, optional Path to connection file to load. If unspecified, use self.connection_file """ if connection_file is None: connection_file = self.connection_file self.log.debug(u"Loading connection file %s", connection_file) with open(connection_file) as f: info = json.load(f) self.load_connection_info(info) def load_connection_info(self, info): """Load connection info from a dict containing connection info. Typically this data comes from a connection file and is called by load_connection_file. Parameters ---------- info: dict Dictionary containing connection_info. See the connection_file spec for details. """ self.transport = info.get('transport', self.transport) self.ip = info.get('ip', self._ip_default()) self._record_random_port_names() for name in port_names: if getattr(self, name) == 0 and name in info: # not overridden by config or cl_args setattr(self, name, info[name]) if 'key' in info: self.session.key = cast_bytes(info['key']) if 'signature_scheme' in info: self.session.signature_scheme = info['signature_scheme'] #-------------------------------------------------------------------------- # Creating connected sockets #-------------------------------------------------------------------------- def _make_url(self, channel): """Make a ZeroMQ URL for a given channel.""" transport = self.transport ip = self.ip port = getattr(self, '%s_port' % channel) if transport == 'tcp': return "tcp://%s:%i" % (ip, port) else: return "%s://%s-%s" % (transport, ip, port) def _create_connected_socket(self, channel, identity=None): """Create a zmq Socket and connect it to the kernel.""" url = self._make_url(channel) socket_type = channel_socket_types[channel] self.log.debug("Connecting to: %s" % url) sock = self.context.socket(socket_type) # set linger to 1s to prevent hangs at exit sock.linger = 1000 if identity: sock.identity = identity sock.connect(url) return sock def connect_iopub(self, identity=None): """return zmq Socket connected to the IOPub channel""" sock = self._create_connected_socket('iopub', identity=identity) sock.setsockopt(zmq.SUBSCRIBE, b'') return sock def connect_shell(self, identity=None): """return zmq Socket connected to the Shell channel""" return self._create_connected_socket('shell', identity=identity) def connect_stdin(self, identity=None): """return zmq Socket connected to the StdIn channel""" return self._create_connected_socket('stdin', identity=identity) def connect_hb(self, identity=None): """return zmq Socket connected to the Heartbeat channel""" return self._create_connected_socket('hb', identity=identity) def connect_control(self, identity=None): """return zmq Socket connected to the Control channel""" return self._create_connected_socket('control', identity=identity) __all__ = [ 'write_connection_file', 'find_connection_file', 'tunnel_to_kernel', ] ================================================ FILE: lib/client/jupyter_client/consoleapp.py ================================================ """ A minimal application base mixin for all ZMQ based IPython frontends. This is not a complete console app, as subprocess will not be able to receive input, there is no real readline support, among other limitations. This is a refactoring of what used to be the IPython/qt/console/qtconsoleapp.py """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import atexit import os import signal import sys import uuid import warnings from traitlets.config.application import boolean_flag from ipython_genutils.path import filefind from traitlets import ( Dict, List, Unicode, CUnicode, CBool, Any ) from jupyter_core.application import base_flags, base_aliases from .blocking import BlockingKernelClient from .restarter import KernelRestarter from . import KernelManager, tunnel_to_kernel, find_connection_file, connect from .kernelspec import NoSuchKernel from .session import Session ConnectionFileMixin = connect.ConnectionFileMixin from .localinterfaces import localhost #----------------------------------------------------------------------------- # Aliases and Flags #----------------------------------------------------------------------------- flags = {} flags.update(base_flags) # the flags that are specific to the frontend # these must be scrubbed before being passed to the kernel, # or it will raise an error on unrecognized flags app_flags = { 'existing' : ({'JupyterConsoleApp' : {'existing' : 'kernel*.json'}}, "Connect to an existing kernel. If no argument specified, guess most recent"), } app_flags.update(boolean_flag( 'confirm-exit', 'JupyterConsoleApp.confirm_exit', """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', to force a direct exit without any confirmation. This can also be set in the config file by setting `c.JupyterConsoleApp.confirm_exit`. """, """Don't prompt the user when exiting. This will terminate the kernel if it is owned by the frontend, and leave it alive if it is external. This can also be set in the config file by setting `c.JupyterConsoleApp.confirm_exit`. """ )) flags.update(app_flags) aliases = {} aliases.update(base_aliases) # also scrub aliases from the frontend app_aliases = dict( ip = 'JupyterConsoleApp.ip', transport = 'JupyterConsoleApp.transport', hb = 'JupyterConsoleApp.hb_port', shell = 'JupyterConsoleApp.shell_port', iopub = 'JupyterConsoleApp.iopub_port', stdin = 'JupyterConsoleApp.stdin_port', existing = 'JupyterConsoleApp.existing', f = 'JupyterConsoleApp.connection_file', kernel = 'JupyterConsoleApp.kernel_name', ssh = 'JupyterConsoleApp.sshserver', ) aliases.update(app_aliases) #----------------------------------------------------------------------------- # Classes #----------------------------------------------------------------------------- classes = [KernelManager, KernelRestarter, Session] class JupyterConsoleApp(ConnectionFileMixin): name = 'jupyter-console-mixin' description = """ The Jupyter Console Mixin. This class contains the common portions of console client (QtConsole, ZMQ-based terminal console, etc). It is not a full console, in that launched terminal subprocesses will not be able to accept input. The Console using this mixing supports various extra features beyond the single-process Terminal IPython shell, such as connecting to existing kernel, via: jupyter console --existing as well as tunnel via SSH """ classes = classes flags = Dict(flags) aliases = Dict(aliases) kernel_manager_class = KernelManager kernel_client_class = BlockingKernelClient kernel_argv = List(Unicode()) # connection info: sshserver = Unicode('', config=True, help="""The SSH server to use to connect to the kernel.""") sshkey = Unicode('', config=True, help="""Path to the ssh key to use for logging in to the ssh server.""") def _connection_file_default(self): return 'kernel-%i.json' % os.getpid() existing = CUnicode('', config=True, help="""Connect to an already running kernel""") kernel_name = Unicode('python', config=True, help="""The name of the default kernel to start.""") confirm_exit = CBool(True, config=True, help=""" Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', to force a direct exit without any confirmation.""", ) def build_kernel_argv(self, argv=None): """build argv to be passed to kernel subprocess Override in subclasses if any args should be passed to the kernel """ self.kernel_argv = self.extra_args def init_connection_file(self): """find the connection file, and load the info if found. The current working directory and the current profile's security directory will be searched for the file if it is not given by absolute path. When attempting to connect to an existing kernel and the `--existing` argument does not match an existing file, it will be interpreted as a fileglob, and the matching file in the current profile's security dir with the latest access time will be used. After this method is called, self.connection_file contains the *full path* to the connection file, never just its name. """ if self.existing: try: cf = find_connection_file(self.existing, ['.', self.runtime_dir]) except Exception: self.log.critical("Could not find existing kernel connection file %s", self.existing) self.exit(1) self.log.debug("Connecting to existing kernel: %s" % cf) self.connection_file = cf else: # not existing, check if we are going to write the file # and ensure that self.connection_file is a full path, not just the shortname try: cf = find_connection_file(self.connection_file, [self.runtime_dir]) except Exception: # file might not exist if self.connection_file == os.path.basename(self.connection_file): # just shortname, put it in security dir cf = os.path.join(self.runtime_dir, self.connection_file) else: cf = self.connection_file self.connection_file = cf try: self.connection_file = filefind(self.connection_file, ['.', self.runtime_dir]) except IOError: self.log.debug("Connection File not found: %s", self.connection_file) return # should load_connection_file only be used for existing? # as it is now, this allows reusing ports if an existing # file is requested try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_ssh(self): """set up ssh tunnels, if needed.""" if not self.existing or (not self.sshserver and not self.sshkey): return self.load_connection_file() transport = self.transport ip = self.ip if transport != 'tcp': self.log.error("Can only use ssh tunnels with TCP sockets, not %s", transport) sys.exit(-1) if self.sshkey and not self.sshserver: # specifying just the key implies that we are connecting directly self.sshserver = ip ip = localhost() # build connection dict for tunnels: info = dict(ip=ip, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port ) self.log.info("Forwarding connections to %s via %s"%(ip, self.sshserver)) # tunnels return a new set of ports, which will be on localhost: self.ip = localhost() try: newports = tunnel_to_kernel(info, self.sshserver, self.sshkey) except: # even catch KeyboardInterrupt self.log.error("Could not setup tunnels", exc_info=True) self.exit(1) self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports cf = self.connection_file root, ext = os.path.splitext(cf) self.connection_file = root + '-ssh' + ext self.write_connection_file() # write the new connection file self.log.info("To connect another client via this tunnel, use:") self.log.info("--existing %s" % os.path.basename(self.connection_file)) def _new_connection_file(self): cf = '' while not cf: # we don't need a 128b id to distinguish kernels, use more readable # 48b node segment (12 hex chars). Users running more than 32k simultaneous # kernels can subclass. ident = str(uuid.uuid4()).split('-')[-1] cf = os.path.join(self.runtime_dir, 'kernel-%s.json' % ident) # only keep if it's actually new. Protect against unlikely collision # in 48b random search space cf = cf if not os.path.exists(cf) else '' return cf def init_kernel_manager(self): # Don't let Qt or ZMQ swallow KeyboardInterupts. if self.existing: self.kernel_manager = None return signal.signal(signal.SIGINT, signal.SIG_DFL) # Create a KernelManager and start a kernel. try: self.kernel_manager = self.kernel_manager_class( ip=self.ip, session=self.session, transport=self.transport, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port, connection_file=self.connection_file, kernel_name=self.kernel_name, parent=self, data_dir=self.data_dir, ) except NoSuchKernel: self.log.critical("Could not find kernel %s", self.kernel_name) self.exit(1) self.kernel_manager.client_factory = self.kernel_client_class # FIXME: remove special treatment of IPython kernels kwargs = {} if self.kernel_manager.ipykernel: kwargs['extra_arguments'] = self.kernel_argv self.kernel_manager.start_kernel(**kwargs) atexit.register(self.kernel_manager.cleanup_ipc_files) if self.sshserver: # ssh, write new connection file self.kernel_manager.write_connection_file() # in case KM defaults / ssh writing changes things: km = self.kernel_manager self.shell_port=km.shell_port self.iopub_port=km.iopub_port self.stdin_port=km.stdin_port self.hb_port=km.hb_port self.connection_file = km.connection_file atexit.register(self.kernel_manager.cleanup_connection_file) def init_kernel_client(self): if self.kernel_manager is not None: self.kernel_client = self.kernel_manager.client() else: self.kernel_client = self.kernel_client_class( session=self.session, ip=self.ip, transport=self.transport, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port, connection_file=self.connection_file, parent=self, ) self.kernel_client.start_channels() def initialize(self, argv=None): """ Classes which mix this class in should call: JupyterConsoleApp.initialize(self,argv) """ if self._dispatching: return self.init_connection_file() self.init_ssh() self.init_kernel_manager() self.init_kernel_client() class IPythonConsoleApp(JupyterConsoleApp): def __init__(self, *args, **kwargs): warnings.warn("IPythonConsoleApp is deprecated. Use JupyterConsoleApp") super(IPythonConsoleApp, self).__init__(*args, **kwargs) ================================================ FILE: lib/client/jupyter_client/ioloop/__init__.py ================================================ from .manager import IOLoopKernelManager from .restarter import IOLoopKernelRestarter ================================================ FILE: lib/client/jupyter_client/ioloop/manager.py ================================================ """A kernel manager with a tornado IOLoop""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import absolute_import from zmq.eventloop import ioloop from zmq.eventloop.zmqstream import ZMQStream from traitlets import ( Instance, Type, ) from jupyter_client.manager import KernelManager from .restarter import IOLoopKernelRestarter def as_zmqstream(f): def wrapped(self, *args, **kwargs): socket = f(self, *args, **kwargs) return ZMQStream(socket, self.loop) return wrapped class IOLoopKernelManager(KernelManager): loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): return ioloop.IOLoop.current() restarter_class = Type( default_value=IOLoopKernelRestarter, klass=IOLoopKernelRestarter, help=( 'Type of KernelRestarter to use. ' 'Must be a subclass of IOLoopKernelRestarter.\n' 'Override this to customize how kernel restarts are managed.' ), config=True, ) _restarter = Instance('jupyter_client.ioloop.IOLoopKernelRestarter', allow_none=True) def start_restarter(self): if self.autorestart and self.has_kernel: if self._restarter is None: self._restarter = self.restarter_class( kernel_manager=self, loop=self.loop, parent=self, log=self.log ) self._restarter.start() def stop_restarter(self): if self.autorestart: if self._restarter is not None: self._restarter.stop() connect_shell = as_zmqstream(KernelManager.connect_shell) connect_iopub = as_zmqstream(KernelManager.connect_iopub) connect_stdin = as_zmqstream(KernelManager.connect_stdin) connect_hb = as_zmqstream(KernelManager.connect_hb) ================================================ FILE: lib/client/jupyter_client/ioloop/restarter.py ================================================ """A basic in process kernel monitor with autorestarting. This watches a kernel's state using KernelManager.is_alive and auto restarts the kernel if it dies. """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import absolute_import import warnings from zmq.eventloop import ioloop from jupyter_client.restarter import KernelRestarter from traitlets import ( Instance, ) class IOLoopKernelRestarter(KernelRestarter): """Monitor and autorestart a kernel.""" loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): warnings.warn("IOLoopKernelRestarter.loop is deprecated in jupyter-client 5.2", DeprecationWarning, stacklevel=4, ) return ioloop.IOLoop.current() _pcallback = None def start(self): """Start the polling of the kernel.""" if self._pcallback is None: self._pcallback = ioloop.PeriodicCallback( self.poll, 1000*self.time_to_dead, ) self._pcallback.start() def stop(self): """Stop the kernel polling.""" if self._pcallback is not None: self._pcallback.stop() self._pcallback = None ================================================ FILE: lib/client/jupyter_client/jsonutil.py ================================================ # coding: utf-8 """Utilities to manipulate JSON objects.""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from datetime import datetime import re import warnings from dateutil.parser import parse as _dateutil_parse from dateutil.tz import tzlocal from ipython_genutils import py3compat from ipython_genutils.py3compat import string_types, iteritems next_attr_name = '__next__' if py3compat.PY3 else 'next' #----------------------------------------------------------------------------- # Globals and constants #----------------------------------------------------------------------------- # timestamp formats ISO8601 = "%Y-%m-%dT%H:%M:%S.%f" ISO8601_PAT = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(\.\d{1,6})?(Z|([\+\-]\d{2}:?\d{2}))?$") # holy crap, strptime is not threadsafe. # Calling it once at import seems to help. datetime.strptime("1", "%d") #----------------------------------------------------------------------------- # Classes and functions #----------------------------------------------------------------------------- def _ensure_tzinfo(dt): """Ensure a datetime object has tzinfo If no tzinfo is present, add tzlocal """ if not dt.tzinfo: # No more naïve datetime objects! warnings.warn(u"Interpreting naive datetime as local %s. Please add timezone info to timestamps." % dt, DeprecationWarning, stacklevel=4) dt = dt.replace(tzinfo=tzlocal()) return dt def parse_date(s): """parse an ISO8601 date string If it is None or not a valid ISO8601 timestamp, it will be returned unmodified. Otherwise, it will return a datetime object. """ if s is None: return s m = ISO8601_PAT.match(s) if m: dt = _dateutil_parse(s) return _ensure_tzinfo(dt) return s def extract_dates(obj): """extract ISO8601 dates from unpacked JSON""" if isinstance(obj, dict): new_obj = {} # don't clobber for k,v in iteritems(obj): new_obj[k] = extract_dates(v) obj = new_obj elif isinstance(obj, (list, tuple)): obj = [ extract_dates(o) for o in obj ] elif isinstance(obj, string_types): obj = parse_date(obj) return obj def squash_dates(obj): """squash datetime objects into ISO8601 strings""" if isinstance(obj, dict): obj = dict(obj) # don't clobber for k,v in iteritems(obj): obj[k] = squash_dates(v) elif isinstance(obj, (list, tuple)): obj = [ squash_dates(o) for o in obj ] elif isinstance(obj, datetime): obj = obj.isoformat() return obj def date_default(obj): """default function for packing datetime objects in JSON.""" if isinstance(obj, datetime): obj = _ensure_tzinfo(obj) return obj.isoformat().replace('+00:00', 'Z') else: raise TypeError("%r is not JSON serializable" % obj) ================================================ FILE: lib/client/jupyter_client/kernelapp.py ================================================ import os import signal import uuid from jupyter_core.application import JupyterApp, base_flags from tornado.ioloop import IOLoop from traitlets import Unicode from . import __version__ from .kernelspec import KernelSpecManager, NATIVE_KERNEL_NAME from .manager import KernelManager class KernelApp(JupyterApp): """Launch a kernel by name in a local subprocess. """ version = __version__ description = "Run a kernel locally in a subprocess" classes = [KernelManager, KernelSpecManager] aliases = { 'kernel': 'KernelApp.kernel_name', 'ip': 'KernelManager.ip', } flags = {'debug': base_flags['debug']} kernel_name = Unicode(NATIVE_KERNEL_NAME, help = 'The name of a kernel type to start' ).tag(config=True) def initialize(self, argv=None): super(KernelApp, self).initialize(argv) self.km = KernelManager(kernel_name=self.kernel_name, config=self.config) cf_basename = 'kernel-%s.json' % uuid.uuid4() self.km.connection_file = os.path.join(self.runtime_dir, cf_basename) self.loop = IOLoop.current() self.loop.add_callback(self._record_started) def setup_signals(self): """Shutdown on SIGTERM or SIGINT (Ctrl-C)""" if os.name == 'nt': return def shutdown_handler(signo, frame): self.loop.add_callback_from_signal(self.shutdown, signo) for sig in [signal.SIGTERM, signal.SIGINT]: signal.signal(sig, shutdown_handler) def shutdown(self, signo): self.log.info('Shutting down on signal %d' % signo) self.km.shutdown_kernel() self.loop.stop() def log_connection_info(self): cf = self.km.connection_file self.log.info('Connection file: %s', cf) self.log.info("To connect a client: --existing %s", os.path.basename(cf)) def _record_started(self): """For tests, create a file to indicate that we've started Do not rely on this except in our own tests! """ fn = os.environ.get('JUPYTER_CLIENT_TEST_RECORD_STARTUP_PRIVATE') if fn is not None: with open(fn, 'wb'): pass def start(self): self.log.info('Starting kernel %r', self.kernel_name) try: self.km.start_kernel() self.log_connection_info() self.setup_signals() self.loop.start() finally: self.km.cleanup() main = KernelApp.launch_instance ================================================ FILE: lib/client/jupyter_client/kernelspec.py ================================================ """Tools for managing kernel specs""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import errno import io import json import os import re import shutil import warnings pjoin = os.path.join from ipython_genutils.py3compat import PY3 from traitlets import ( HasTraits, List, Unicode, Dict, Set, Bool, Type, CaselessStrEnum ) from traitlets.config import LoggingConfigurable from jupyter_core.paths import jupyter_data_dir, jupyter_path, SYSTEM_JUPYTER_PATH NATIVE_KERNEL_NAME = 'python3' if PY3 else 'python2' class KernelSpec(HasTraits): argv = List() display_name = Unicode() language = Unicode() env = Dict() resource_dir = Unicode() interrupt_mode = CaselessStrEnum( ['message', 'signal'], default_value='signal' ) metadata = Dict() @classmethod def from_resource_dir(cls, resource_dir): """Create a KernelSpec object by reading kernel.json Pass the path to the *directory* containing kernel.json. """ kernel_file = pjoin(resource_dir, 'kernel.json') with io.open(kernel_file, 'r', encoding='utf-8') as f: kernel_dict = json.load(f) return cls(resource_dir=resource_dir, **kernel_dict) def to_dict(self): d = dict(argv=self.argv, env=self.env, display_name=self.display_name, language=self.language, interrupt_mode=self.interrupt_mode, metadata=self.metadata, ) return d def to_json(self): """Serialise this kernelspec to a JSON object. Returns a string. """ return json.dumps(self.to_dict()) _kernel_name_pat = re.compile(r'^[a-z0-9._\-]+$', re.IGNORECASE) def _is_valid_kernel_name(name): """Check that a kernel name is valid.""" # quote is not unicode-safe on Python 2 return _kernel_name_pat.match(name) _kernel_name_description = "Kernel names can only contain ASCII letters and numbers and these separators:" \ " - . _ (hyphen, period, and underscore)." def _is_kernel_dir(path): """Is ``path`` a kernel directory?""" return os.path.isdir(path) and os.path.isfile(pjoin(path, 'kernel.json')) def _list_kernels_in(dir): """Return a mapping of kernel names to resource directories from dir. If dir is None or does not exist, returns an empty dict. """ if dir is None or not os.path.isdir(dir): return {} kernels = {} for f in os.listdir(dir): path = pjoin(dir, f) if not _is_kernel_dir(path): continue key = f.lower() if not _is_valid_kernel_name(key): warnings.warn("Invalid kernelspec directory name (%s): %s" % (_kernel_name_description, path), stacklevel=3, ) kernels[key] = path return kernels class NoSuchKernel(KeyError): def __init__(self, name): self.name = name def __str__(self): return "No such kernel named {}".format(self.name) class KernelSpecManager(LoggingConfigurable): kernel_spec_class = Type(KernelSpec, config=True, help="""The kernel spec class. This is configurable to allow subclassing of the KernelSpecManager for customized behavior. """ ) ensure_native_kernel = Bool(True, config=True, help="""If there is no Python kernelspec registered and the IPython kernel is available, ensure it is added to the spec list. """ ) data_dir = Unicode() def _data_dir_default(self): return jupyter_data_dir() user_kernel_dir = Unicode() def _user_kernel_dir_default(self): return pjoin(self.data_dir, 'kernels') whitelist = Set(config=True, help="""Whitelist of allowed kernel names. By default, all installed kernels are allowed. """ ) kernel_dirs = List( help="List of kernel directories to search. Later ones take priority over earlier." ) def _kernel_dirs_default(self): dirs = jupyter_path('kernels') # At some point, we should stop adding .ipython/kernels to the path, # but the cost to keeping it is very small. try: from IPython.paths import get_ipython_dir except ImportError: try: from IPython.utils.path import get_ipython_dir except ImportError: # no IPython, no ipython dir get_ipython_dir = None if get_ipython_dir is not None: dirs.append(os.path.join(get_ipython_dir(), 'kernels')) return dirs def find_kernel_specs(self): """Returns a dict mapping kernel names to resource directories.""" d = {} for kernel_dir in self.kernel_dirs: kernels = _list_kernels_in(kernel_dir) for kname, spec in kernels.items(): if kname not in d: self.log.debug("Found kernel %s in %s", kname, kernel_dir) d[kname] = spec if self.ensure_native_kernel and NATIVE_KERNEL_NAME not in d: try: from ipykernel.kernelspec import RESOURCES self.log.debug("Native kernel (%s) available from %s", NATIVE_KERNEL_NAME, RESOURCES) d[NATIVE_KERNEL_NAME] = RESOURCES except ImportError: self.log.warning("Native kernel (%s) is not available", NATIVE_KERNEL_NAME) if self.whitelist: # filter if there's a whitelist d = {name:spec for name,spec in d.items() if name in self.whitelist} return d # TODO: Caching? def _get_kernel_spec_by_name(self, kernel_name, resource_dir): """ Returns a :class:`KernelSpec` instance for a given kernel_name and resource_dir. """ if kernel_name == NATIVE_KERNEL_NAME: try: from ipykernel.kernelspec import RESOURCES, get_kernel_dict except ImportError: # It should be impossible to reach this, but let's play it safe pass else: if resource_dir == RESOURCES: return self.kernel_spec_class(resource_dir=resource_dir, **get_kernel_dict()) return self.kernel_spec_class.from_resource_dir(resource_dir) def _find_spec_directory(self, kernel_name): """Find the resource directory of a named kernel spec""" for kernel_dir in self.kernel_dirs: try: files = os.listdir(kernel_dir) except OSError as e: if e.errno in (errno.ENOTDIR, errno.ENOENT): continue raise for f in files: path = pjoin(kernel_dir, f) if f.lower() == kernel_name and _is_kernel_dir(path): return path if kernel_name == NATIVE_KERNEL_NAME: try: from ipykernel.kernelspec import RESOURCES except ImportError: pass else: return RESOURCES def get_kernel_spec(self, kernel_name): """Returns a :class:`KernelSpec` instance for the given kernel_name. Raises :exc:`NoSuchKernel` if the given kernel name is not found. """ if not _is_valid_kernel_name(kernel_name): self.log.warning("Kernelspec name %r is invalid: %s", kernel_name, _kernel_name_description) resource_dir = self._find_spec_directory(kernel_name.lower()) if resource_dir is None: raise NoSuchKernel(kernel_name) return self._get_kernel_spec_by_name(kernel_name, resource_dir) def get_all_specs(self): """Returns a dict mapping kernel names to kernelspecs. Returns a dict of the form:: { 'kernel_name': { 'resource_dir': '/path/to/kernel_name', 'spec': {"the spec itself": ...} }, ... } """ d = self.find_kernel_specs() res = {} for kname, resource_dir in d.items(): try: if self.__class__ is KernelSpecManager: spec = self._get_kernel_spec_by_name(kname, resource_dir) else: # avoid calling private methods in subclasses, # which may have overridden find_kernel_specs # and get_kernel_spec, but not the newer get_all_specs spec = self.get_kernel_spec(kname) res[kname] = { "resource_dir": resource_dir, "spec": spec.to_dict() } except Exception: self.log.warning("Error loading kernelspec %r", kname, exc_info=True) return res def remove_kernel_spec(self, name): """Remove a kernel spec directory by name. Returns the path that was deleted. """ save_native = self.ensure_native_kernel try: self.ensure_native_kernel = False specs = self.find_kernel_specs() finally: self.ensure_native_kernel = save_native spec_dir = specs[name] self.log.debug("Removing %s", spec_dir) if os.path.islink(spec_dir): os.remove(spec_dir) else: shutil.rmtree(spec_dir) return spec_dir def _get_destination_dir(self, kernel_name, user=False, prefix=None): if user: return os.path.join(self.user_kernel_dir, kernel_name) elif prefix: return os.path.join(os.path.abspath(prefix), 'share', 'jupyter', 'kernels', kernel_name) else: return os.path.join(SYSTEM_JUPYTER_PATH[0], 'kernels', kernel_name) def install_kernel_spec(self, source_dir, kernel_name=None, user=False, replace=None, prefix=None): """Install a kernel spec by copying its directory. If ``kernel_name`` is not given, the basename of ``source_dir`` will be used. If ``user`` is False, it will attempt to install into the systemwide kernel registry. If the process does not have appropriate permissions, an :exc:`OSError` will be raised. If ``prefix`` is given, the kernelspec will be installed to PREFIX/share/jupyter/kernels/KERNEL_NAME. This can be sys.prefix for installation inside virtual or conda envs. """ source_dir = source_dir.rstrip('/\\') if not kernel_name: kernel_name = os.path.basename(source_dir) kernel_name = kernel_name.lower() if not _is_valid_kernel_name(kernel_name): raise ValueError("Invalid kernel name %r. %s" % (kernel_name, _kernel_name_description)) if user and prefix: raise ValueError("Can't specify both user and prefix. Please choose one or the other.") if replace is not None: warnings.warn( "replace is ignored. Installing a kernelspec always replaces an existing installation", DeprecationWarning, stacklevel=2, ) destination = self._get_destination_dir(kernel_name, user=user, prefix=prefix) self.log.debug('Installing kernelspec in %s', destination) kernel_dir = os.path.dirname(destination) if kernel_dir not in self.kernel_dirs: self.log.warning("Installing to %s, which is not in %s. The kernelspec may not be found.", kernel_dir, self.kernel_dirs, ) if os.path.isdir(destination): self.log.info('Removing existing kernelspec in %s', destination) shutil.rmtree(destination) shutil.copytree(source_dir, destination) self.log.info('Installed kernelspec %s in %s', kernel_name, destination) return destination def install_native_kernel_spec(self, user=False): """DEPRECATED: Use ipykernel.kenelspec.install""" warnings.warn("install_native_kernel_spec is deprecated." " Use ipykernel.kernelspec import install.", stacklevel=2) from ipykernel.kernelspec import install install(self, user=user) def find_kernel_specs(): """Returns a dict mapping kernel names to resource directories.""" return KernelSpecManager().find_kernel_specs() def get_kernel_spec(kernel_name): """Returns a :class:`KernelSpec` instance for the given kernel_name. Raises KeyError if the given kernel name is not found. """ return KernelSpecManager().get_kernel_spec(kernel_name) def install_kernel_spec(source_dir, kernel_name=None, user=False, replace=False, prefix=None): return KernelSpecManager().install_kernel_spec(source_dir, kernel_name, user, replace, prefix) install_kernel_spec.__doc__ = KernelSpecManager.install_kernel_spec.__doc__ def install_native_kernel_spec(user=False): return KernelSpecManager().install_native_kernel_spec(user=user) install_native_kernel_spec.__doc__ = KernelSpecManager.install_native_kernel_spec.__doc__ ================================================ FILE: lib/client/jupyter_client/kernelspecapp.py ================================================ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import print_function import errno import os.path import sys import json from traitlets.config.application import Application from jupyter_core.application import ( JupyterApp, base_flags, base_aliases ) from traitlets import Instance, Dict, Unicode, Bool, List from . import __version__ from .kernelspec import KernelSpecManager try: raw_input except NameError: # py3 raw_input = input class ListKernelSpecs(JupyterApp): version = __version__ description = """List installed kernel specifications.""" kernel_spec_manager = Instance(KernelSpecManager) json_output = Bool(False, help='output spec name and location as machine-readable json.', config=True) flags = {'json': ({'ListKernelSpecs': {'json_output': True}}, "output spec name and location as machine-readable json."), 'debug': base_flags['debug'], } def _kernel_spec_manager_default(self): return KernelSpecManager(parent=self, data_dir=self.data_dir) def start(self): paths = self.kernel_spec_manager.find_kernel_specs() specs = self.kernel_spec_manager.get_all_specs() if not self.json_output: if not specs: print("No kernels available") return # pad to width of longest kernel name name_len = len(sorted(paths, key=lambda name: len(name))[-1]) def path_key(item): """sort key function for Jupyter path priority""" path = item[1] for idx, prefix in enumerate(self.jupyter_path): if path.startswith(prefix): return (idx, path) # not in jupyter path, artificially added to the front return (-1, path) print("Available kernels:") for kernelname, path in sorted(paths.items(), key=path_key): print(" %s %s" % (kernelname.ljust(name_len), path)) else: print(json.dumps({ 'kernelspecs': specs }, indent=2)) class InstallKernelSpec(JupyterApp): version = __version__ description = """Install a kernel specification directory. Given a SOURCE DIRECTORY containing a kernel spec, jupyter will copy that directory into one of the Jupyter kernel directories. The default is to install kernelspecs for all users. `--user` can be specified to install a kernel only for the current user. """ examples = """ jupyter kernelspec install /path/to/my_kernel --user """ usage = "jupyter kernelspec install SOURCE_DIR [--options]" kernel_spec_manager = Instance(KernelSpecManager) def _kernel_spec_manager_default(self): return KernelSpecManager(data_dir=self.data_dir) sourcedir = Unicode() kernel_name = Unicode("", config=True, help="Install the kernel spec with this name" ) def _kernel_name_default(self): return os.path.basename(self.sourcedir) user = Bool(False, config=True, help=""" Try to install the kernel spec to the per-user directory instead of the system or environment directory. """ ) prefix = Unicode('', config=True, help="""Specify a prefix to install to, e.g. an env. The kernelspec will be installed in PREFIX/share/jupyter/kernels/ """ ) replace = Bool(False, config=True, help="Replace any existing kernel spec with this name." ) aliases = { 'name': 'InstallKernelSpec.kernel_name', 'prefix': 'InstallKernelSpec.prefix', } aliases.update(base_aliases) flags = {'user': ({'InstallKernelSpec': {'user': True}}, "Install to the per-user kernel registry"), 'replace': ({'InstallKernelSpec': {'replace': True}}, "Replace any existing kernel spec with this name."), 'sys-prefix': ({'InstallKernelSpec': {'prefix': sys.prefix}}, "Install to Python's sys.prefix. Useful in conda/virtual environments."), 'debug': base_flags['debug'], } def parse_command_line(self, argv): super(InstallKernelSpec, self).parse_command_line(argv) # accept positional arg as profile name if self.extra_args: self.sourcedir = self.extra_args[0] else: print("No source directory specified.") self.exit(1) def start(self): if self.user and self.prefix: self.exit("Can't specify both user and prefix. Please choose one or the other.") try: self.kernel_spec_manager.install_kernel_spec(self.sourcedir, kernel_name=self.kernel_name, user=self.user, prefix=self.prefix, replace=self.replace, ) except OSError as e: if e.errno == errno.EACCES: print(e, file=sys.stderr) if not self.user: print("Perhaps you want to install with `sudo` or `--user`?", file=sys.stderr) self.exit(1) elif e.errno == errno.EEXIST: print("A kernel spec is already present at %s" % e.filename, file=sys.stderr) self.exit(1) raise class RemoveKernelSpec(JupyterApp): version = __version__ description = """Remove one or more Jupyter kernelspecs by name.""" examples = """jupyter kernelspec remove python2 [my_kernel ...]""" force = Bool(False, config=True, help="""Force removal, don't prompt for confirmation.""" ) spec_names = List(Unicode()) kernel_spec_manager = Instance(KernelSpecManager) def _kernel_spec_manager_default(self): return KernelSpecManager(data_dir=self.data_dir, parent=self) flags = { 'f': ({'RemoveKernelSpec': {'force': True}}, force.get_metadata('help')), } flags.update(JupyterApp.flags) def parse_command_line(self, argv): super(RemoveKernelSpec, self).parse_command_line(argv) # accept positional arg as profile name if self.extra_args: self.spec_names = sorted(set(self.extra_args)) # remove duplicates else: self.exit("No kernelspec specified.") def start(self): self.kernel_spec_manager.ensure_native_kernel = False spec_paths = self.kernel_spec_manager.find_kernel_specs() missing = set(self.spec_names).difference(set(spec_paths)) if missing: self.exit("Couldn't find kernel spec(s): %s" % ', '.join(missing)) if not self.force: print("Kernel specs to remove:") for name in self.spec_names: print(" %s\t%s" % (name.ljust(20), spec_paths[name])) answer = raw_input("Remove %i kernel specs [y/N]: " % len(self.spec_names)) if not answer.lower().startswith('y'): return for kernel_name in self.spec_names: try: path = self.kernel_spec_manager.remove_kernel_spec(kernel_name) except OSError as e: if e.errno == errno.EACCES: print(e, file=sys.stderr) print("Perhaps you want sudo?", file=sys.stderr) self.exit(1) else: raise self.log.info("Removed %s", path) class InstallNativeKernelSpec(JupyterApp): version = __version__ description = """[DEPRECATED] Install the IPython kernel spec directory for this Python.""" kernel_spec_manager = Instance(KernelSpecManager) def _kernel_spec_manager_default(self): return KernelSpecManager(data_dir=self.data_dir) user = Bool(False, config=True, help=""" Try to install the kernel spec to the per-user directory instead of the system or environment directory. """ ) flags = {'user': ({'InstallNativeKernelSpec': {'user': True}}, "Install to the per-user kernel registry"), 'debug': base_flags['debug'], } def start(self): self.log.warning("`jupyter kernelspec install-self` is DEPRECATED as of 4.0." " You probably want `ipython kernel install` to install the IPython kernelspec.") try: from ipykernel import kernelspec except ImportError: print("ipykernel not available, can't install its spec.", file=sys.stderr) self.exit(1) try: kernelspec.install(self.kernel_spec_manager, user=self.user) except OSError as e: if e.errno == errno.EACCES: print(e, file=sys.stderr) if not self.user: print("Perhaps you want to install with `sudo` or `--user`?", file=sys.stderr) self.exit(1) self.exit(e) class KernelSpecApp(Application): version = __version__ name = "jupyter kernelspec" description = """Manage Jupyter kernel specifications.""" subcommands = Dict({ 'list': (ListKernelSpecs, ListKernelSpecs.description.splitlines()[0]), 'install': (InstallKernelSpec, InstallKernelSpec.description.splitlines()[0]), 'uninstall': (RemoveKernelSpec, "Alias for remove"), 'remove': (RemoveKernelSpec, RemoveKernelSpec.description.splitlines()[0]), 'install-self': (InstallNativeKernelSpec, InstallNativeKernelSpec.description.splitlines()[0]), }) aliases = {} flags = {} def start(self): if self.subapp is None: print("No subcommand specified. Must specify one of: %s"% list(self.subcommands)) print() self.print_description() self.print_subcommands() self.exit(1) else: return self.subapp.start() if __name__ == '__main__': KernelSpecApp.launch_instance() ================================================ FILE: lib/client/jupyter_client/launcher.py ================================================ """Utilities for launching kernels""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import os import sys from subprocess import Popen, PIPE from ipython_genutils.encoding import getdefaultencoding from ipython_genutils.py3compat import cast_bytes_py2, PY3 from traitlets.log import get_logger def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None, independent=False, cwd=None, **kw): """ Launches a localhost kernel, binding to the specified ports. Parameters ---------- cmd : Popen list, A string of Python code that imports and executes a kernel entry point. stdin, stdout, stderr : optional (default None) Standards streams, as defined in subprocess.Popen. env: dict, optional Environment variables passed to the kernel independent : bool, optional (default False) If set, the kernel process is guaranteed to survive if this process dies. If not set, an effort is made to ensure that the kernel is killed when this process dies. Note that in this case it is still good practice to kill kernels manually before exiting. cwd : path, optional The working dir of the kernel process (default: cwd of this process). **kw: optional Additional arguments for Popen Returns ------- Popen instance for the kernel subprocess """ # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr # are invalid. Unfortunately, there is in general no way to detect whether # they are valid. The following two blocks redirect them to (temporary) # pipes in certain important cases. # If this process has been backgrounded, our stdin is invalid. Since there # is no compelling reason for the kernel to inherit our stdin anyway, we'll # place this one safe and always redirect. redirect_in = True _stdin = PIPE if stdin is None else stdin # If this process in running on pythonw, we know that stdin, stdout, and # stderr are all invalid. redirect_out = sys.executable.endswith('pythonw.exe') if redirect_out: blackhole = open(os.devnull, 'w') _stdout = blackhole if stdout is None else stdout _stderr = blackhole if stderr is None else stderr else: _stdout, _stderr = stdout, stderr env = env if (env is not None) else os.environ.copy() encoding = getdefaultencoding(prefer_stream=False) kwargs = kw.copy() main_args = dict( stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env, ) kwargs.update(main_args) # Spawn a kernel. if sys.platform == 'win32': # Popen on Python 2 on Windows cannot handle unicode args or cwd cmd = [ cast_bytes_py2(c, encoding) for c in cmd ] if cwd: cwd = cast_bytes_py2(cwd, sys.getfilesystemencoding() or 'ascii') kwargs['cwd'] = cwd from .win_interrupt import create_interrupt_event # Create a Win32 event for interrupting the kernel # and store it in an environment variable. interrupt_event = create_interrupt_event() env["JPY_INTERRUPT_EVENT"] = str(interrupt_event) # deprecated old env name: env["IPY_INTERRUPT_EVENT"] = env["JPY_INTERRUPT_EVENT"] try: from _winapi import DuplicateHandle, GetCurrentProcess, \ DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP except: from _subprocess import DuplicateHandle, GetCurrentProcess, \ DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP # create a handle on the parent to be inherited if independent: kwargs['creationflags'] = CREATE_NEW_PROCESS_GROUP else: pid = GetCurrentProcess() handle = DuplicateHandle(pid, pid, pid, 0, True, # Inheritable by new processes. DUPLICATE_SAME_ACCESS) env['JPY_PARENT_PID'] = str(int(handle)) # Prevent creating new console window on pythonw if redirect_out: kwargs['creationflags'] = kwargs.setdefault('creationflags', 0) | 0x08000000 # CREATE_NO_WINDOW # Avoid closing the above parent and interrupt handles. # close_fds is True by default on Python >=3.7 # or when no stream is captured on Python <3.7 # (we always capture stdin, so this is already False by default on <3.7) kwargs['close_fds'] = False else: # Create a new session. # This makes it easier to interrupt the kernel, # because we want to interrupt the whole process group. # We don't use setpgrp, which is known to cause problems for kernels starting # certain interactive subprocesses, such as bash -i. if PY3: kwargs['start_new_session'] = True else: kwargs['preexec_fn'] = lambda: os.setsid() if not independent: env['JPY_PARENT_PID'] = str(os.getpid()) try: proc = Popen(cmd, **kwargs) except Exception as exc: msg = ( "Failed to run command:\n{}\n" " PATH={!r}\n" " with kwargs:\n{!r}\n" ) # exclude environment variables, # which may contain access tokens and the like. without_env = {key:value for key, value in kwargs.items() if key != 'env'} msg = msg.format(cmd, env.get('PATH', os.defpath), without_env) get_logger().error(msg) raise if sys.platform == 'win32': # Attach the interrupt event to the Popen objet so it can be used later. proc.win32_interrupt_event = interrupt_event # Clean up pipes created to work around Popen bug. if redirect_in: if stdin is None: proc.stdin.close() return proc __all__ = [ 'launch_kernel', ] ================================================ FILE: lib/client/jupyter_client/localinterfaces.py ================================================ """Utilities for identifying local IP addresses.""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import os import re import socket import subprocess from subprocess import Popen, PIPE from warnings import warn LOCAL_IPS = [] PUBLIC_IPS = [] LOCALHOST = '' def _uniq_stable(elems): """uniq_stable(elems) -> list Return from an iterable, a list of all the unique elements in the input, maintaining the order in which they first appear. From ipython_genutils.data """ seen = set() return [x for x in elems if x not in seen and not seen.add(x)] def _get_output(cmd): """Get output of a command, raising IOError if it fails""" startupinfo = None if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW p = Popen(cmd, stdout=PIPE, stderr=PIPE, startupinfo=startupinfo) stdout, stderr = p.communicate() if p.returncode: raise IOError("Failed to run %s: %s" % (cmd, stderr.decode('utf8', 'replace'))) return stdout.decode('utf8', 'replace') def _only_once(f): """decorator to only run a function once""" f.called = False def wrapped(**kwargs): if f.called: return ret = f(**kwargs) f.called = True return ret return wrapped def _requires_ips(f): """decorator to ensure load_ips has been run before f""" def ips_loaded(*args, **kwargs): _load_ips() return f(*args, **kwargs) return ips_loaded # subprocess-parsing ip finders class NoIPAddresses(Exception): pass def _populate_from_list(addrs): """populate local and public IPs from flat list of all IPs""" if not addrs: raise NoIPAddresses global LOCALHOST public_ips = [] local_ips = [] for ip in addrs: local_ips.append(ip) if not ip.startswith('127.'): public_ips.append(ip) elif not LOCALHOST: LOCALHOST = ip if not LOCALHOST: LOCALHOST = '127.0.0.1' local_ips.insert(0, LOCALHOST) local_ips.extend(['0.0.0.0', '']) LOCAL_IPS[:] = _uniq_stable(local_ips) PUBLIC_IPS[:] = _uniq_stable(public_ips) _ifconfig_ipv4_pat = re.compile(r'inet\b.*?(\d+\.\d+\.\d+\.\d+)', re.IGNORECASE) def _load_ips_ifconfig(): """load ip addresses from `ifconfig` output (posix)""" try: out = _get_output('ifconfig') except (IOError, OSError): # no ifconfig, it's usually in /sbin and /sbin is not on everyone's PATH out = _get_output('/sbin/ifconfig') lines = out.splitlines() addrs = [] for line in lines: m = _ifconfig_ipv4_pat.match(line.strip()) if m: addrs.append(m.group(1)) _populate_from_list(addrs) def _load_ips_ip(): """load ip addresses from `ip addr` output (Linux)""" out = _get_output(['ip', '-f', 'inet', 'addr']) lines = out.splitlines() addrs = [] for line in lines: blocks = line.lower().split() if (len(blocks) >= 2) and (blocks[0] == 'inet'): addrs.append(blocks[1].split('/')[0]) _populate_from_list(addrs) _ipconfig_ipv4_pat = re.compile(r'ipv4.*?(\d+\.\d+\.\d+\.\d+)$', re.IGNORECASE) def _load_ips_ipconfig(): """load ip addresses from `ipconfig` output (Windows)""" out = _get_output('ipconfig') lines = out.splitlines() addrs = [] for line in lines: m = _ipconfig_ipv4_pat.match(line.strip()) if m: addrs.append(m.group(1)) _populate_from_list(addrs) def _load_ips_netifaces(): """load ip addresses with netifaces""" import netifaces global LOCALHOST local_ips = [] public_ips = [] # list of iface names, 'lo0', 'eth0', etc. for iface in netifaces.interfaces(): # list of ipv4 addrinfo dicts ipv4s = netifaces.ifaddresses(iface).get(netifaces.AF_INET, []) for entry in ipv4s: addr = entry.get('addr') if not addr: continue if not (iface.startswith('lo') or addr.startswith('127.')): public_ips.append(addr) elif not LOCALHOST: LOCALHOST = addr local_ips.append(addr) if not LOCALHOST: # we never found a loopback interface (can this ever happen?), assume common default LOCALHOST = '127.0.0.1' local_ips.insert(0, LOCALHOST) local_ips.extend(['0.0.0.0', '']) LOCAL_IPS[:] = _uniq_stable(local_ips) PUBLIC_IPS[:] = _uniq_stable(public_ips) def _load_ips_gethostbyname(): """load ip addresses with socket.gethostbyname_ex This can be slow. """ global LOCALHOST try: LOCAL_IPS[:] = socket.gethostbyname_ex('localhost')[2] except socket.error: # assume common default LOCAL_IPS[:] = ['127.0.0.1'] try: hostname = socket.gethostname() PUBLIC_IPS[:] = socket.gethostbyname_ex(hostname)[2] # try hostname.local, in case hostname has been short-circuited to loopback if not hostname.endswith('.local') and all(ip.startswith('127') for ip in PUBLIC_IPS): PUBLIC_IPS[:] = socket.gethostbyname_ex(socket.gethostname() + '.local')[2] except socket.error: pass finally: PUBLIC_IPS[:] = _uniq_stable(PUBLIC_IPS) LOCAL_IPS.extend(PUBLIC_IPS) # include all-interface aliases: 0.0.0.0 and '' LOCAL_IPS.extend(['0.0.0.0', '']) LOCAL_IPS[:] = _uniq_stable(LOCAL_IPS) LOCALHOST = LOCAL_IPS[0] def _load_ips_dumb(): """Fallback in case of unexpected failure""" global LOCALHOST LOCALHOST = '127.0.0.1' LOCAL_IPS[:] = [LOCALHOST, '0.0.0.0', ''] PUBLIC_IPS[:] = [] @_only_once def _load_ips(suppress_exceptions=True): """load the IPs that point to this machine This function will only ever be called once. It will use netifaces to do it quickly if available. Then it will fallback on parsing the output of ifconfig / ip addr / ipconfig, as appropriate. Finally, it will fallback on socket.gethostbyname_ex, which can be slow. """ try: # first priority, use netifaces try: return _load_ips_netifaces() except ImportError: pass # second priority, parse subprocess output (how reliable is this?) if os.name == 'nt': try: return _load_ips_ipconfig() except (IOError, NoIPAddresses): pass else: try: return _load_ips_ip() except (IOError, OSError, NoIPAddresses): pass try: return _load_ips_ifconfig() except (IOError, OSError, NoIPAddresses): pass # lowest priority, use gethostbyname return _load_ips_gethostbyname() except Exception as e: if not suppress_exceptions: raise # unexpected error shouldn't crash, load dumb default values instead. warn("Unexpected error discovering local network interfaces: %s" % e) _load_ips_dumb() @_requires_ips def local_ips(): """return the IP addresses that point to this machine""" return LOCAL_IPS @_requires_ips def public_ips(): """return the IP addresses for this machine that are visible to other machines""" return PUBLIC_IPS @_requires_ips def localhost(): """return ip for localhost (almost always 127.0.0.1)""" return LOCALHOST @_requires_ips def is_local_ip(ip): """does `ip` point to this machine?""" return ip in LOCAL_IPS @_requires_ips def is_public_ip(ip): """is `ip` a publicly visible address?""" return ip in PUBLIC_IPS ================================================ FILE: lib/client/jupyter_client/manager.py ================================================ """Base class to manage a running kernel""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import absolute_import from contextlib import contextmanager import os import re import signal import sys import time import warnings import zmq from ipython_genutils.importstring import import_item from .localinterfaces import is_local_ip, local_ips from traitlets import ( Any, Float, Instance, Unicode, List, Bool, Type, DottedObjectName ) from jupyter_client import ( launch_kernel, kernelspec, ) from .connect import ConnectionFileMixin from .managerabc import ( KernelManagerABC ) class KernelManager(ConnectionFileMixin): """Manages a single kernel in a subprocess on this host. This version starts kernels with Popen. """ # The PyZMQ Context to use for communication with the kernel. context = Instance(zmq.Context) def _context_default(self): return zmq.Context.instance() # the class to create with our `client` method client_class = DottedObjectName('jupyter_client.blocking.BlockingKernelClient') client_factory = Type(klass='jupyter_client.KernelClient') def _client_factory_default(self): return import_item(self.client_class) def _client_class_changed(self, name, old, new): self.client_factory = import_item(str(new)) # The kernel process with which the KernelManager is communicating. # generally a Popen instance kernel = Any() kernel_spec_manager = Instance(kernelspec.KernelSpecManager) def _kernel_spec_manager_default(self): return kernelspec.KernelSpecManager(data_dir=self.data_dir) def _kernel_spec_manager_changed(self): self._kernel_spec = None shutdown_wait_time = Float( 5.0, config=True, help="Time to wait for a kernel to terminate before killing it, " "in seconds.") kernel_name = Unicode(kernelspec.NATIVE_KERNEL_NAME) def _kernel_name_changed(self, name, old, new): self._kernel_spec = None if new == 'python': self.kernel_name = kernelspec.NATIVE_KERNEL_NAME _kernel_spec = None @property def kernel_spec(self): if self._kernel_spec is None and self.kernel_name != '': self._kernel_spec = self.kernel_spec_manager.get_kernel_spec(self.kernel_name) return self._kernel_spec kernel_cmd = List(Unicode(), config=True, help="""DEPRECATED: Use kernel_name instead. The Popen Command to launch the kernel. Override this if you have a custom kernel. If kernel_cmd is specified in a configuration file, Jupyter does not pass any arguments to the kernel, because it cannot make any assumptions about the arguments that the kernel understands. In particular, this means that the kernel does not receive the option --debug if it given on the Jupyter command line. """ ) def _kernel_cmd_changed(self, name, old, new): warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to " "start different kernels.") @property def ipykernel(self): return self.kernel_name in {'python', 'python2', 'python3'} # Protected traits _launch_args = Any() _control_socket = Any() _restarter = Any() autorestart = Bool(True, config=True, help="""Should we autorestart the kernel if it dies.""" ) def __del__(self): self._close_control_socket() self.cleanup_connection_file() #-------------------------------------------------------------------------- # Kernel restarter #-------------------------------------------------------------------------- def start_restarter(self): pass def stop_restarter(self): pass def add_restart_callback(self, callback, event='restart'): """register a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.add_callback(callback, event) def remove_restart_callback(self, callback, event='restart'): """unregister a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.remove_callback(callback, event) #-------------------------------------------------------------------------- # create a Client connected to our Kernel #-------------------------------------------------------------------------- def client(self, **kwargs): """Create a client configured to connect to our kernel""" kw = {} kw.update(self.get_connection_info(session=True)) kw.update(dict( connection_file=self.connection_file, parent=self, )) # add kwargs last, for manual overrides kw.update(kwargs) return self.client_factory(**kw) #-------------------------------------------------------------------------- # Kernel management #-------------------------------------------------------------------------- def format_kernel_cmd(self, extra_arguments=None): """replace templated args (e.g. {connection_file})""" extra_arguments = extra_arguments or [] if self.kernel_cmd: cmd = self.kernel_cmd + extra_arguments else: cmd = self.kernel_spec.argv + extra_arguments if cmd and cmd[0] in {'python', 'python%i' % sys.version_info[0], 'python%i.%i' % sys.version_info[:2]}: # executable is 'python' or 'python3', use sys.executable. # These will typically be the same, # but if the current process is in an env # and has been launched by abspath without # activating the env, python on PATH may not be sys.executable, # but it should be. cmd[0] = sys.executable ns = dict(connection_file=self.connection_file, prefix=sys.prefix, ) if self.kernel_spec: ns["resource_dir"] = self.kernel_spec.resource_dir ns.update(self._launch_args) pat = re.compile(r'\{([A-Za-z0-9_]+)\}') def from_ns(match): """Get the key out of ns if it's there, otherwise no change.""" return ns.get(match.group(1), match.group()) return [ pat.sub(from_ns, arg) for arg in cmd ] def _launch_kernel(self, kernel_cmd, **kw): """actually launch the kernel override in a subclass to launch kernel subprocesses differently """ return launch_kernel(kernel_cmd, **kw) # Control socket used for polite kernel shutdown def _connect_control_socket(self): if self._control_socket is None: self._control_socket = self.connect_control() self._control_socket.linger = 100 def _close_control_socket(self): if self._control_socket is None: return self._control_socket.close() self._control_socket = None def start_kernel(self, **kw): """Starts a kernel on this host in a separate process. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- `**kw` : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError("Can only launch a kernel on a local interface. " "This one is not: %s." "Make sure that the '*_address' attributes are " "configured properly. " "Currently valid addresses are: %s" % (self.ip, local_ips()) ) # write connection file / get default ports self.write_connection_file() # save kwargs for use in restart self._launch_args = kw.copy() # build the Popen cmd extra_arguments = kw.pop('extra_arguments', []) kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments) env = kw.pop('env', os.environ).copy() # Don't allow PYTHONEXECUTABLE to be passed to kernel process. # If set, it can bork all the things. env.pop('PYTHONEXECUTABLE', None) if not self.kernel_cmd: # If kernel_cmd has been set manually, don't refer to a kernel spec # Environment variables from kernel spec are added to os.environ env.update(self.kernel_spec.env or {}) # launch the kernel subprocess self.log.debug("Starting kernel: %s", kernel_cmd) self.kernel = self._launch_kernel(kernel_cmd, env=env, **kw) self.start_restarter() self._connect_control_socket() def request_shutdown(self, restart=False): """Send a shutdown request via control channel """ content = dict(restart=restart) msg = self.session.msg("shutdown_request", content=content) # ensure control socket is connected self._connect_control_socket() self.session.send(self._control_socket, msg) def finish_shutdown(self, waittime=None, pollinterval=0.1): """Wait for kernel shutdown, then kill process if it doesn't shutdown. This does not send shutdown requests - use :meth:`request_shutdown` first. """ if waittime is None: waittime = max(self.shutdown_wait_time, 0) for i in range(int(waittime/pollinterval)): if self.is_alive(): time.sleep(pollinterval) else: break else: # OK, we've waited long enough. if self.has_kernel: self.log.debug("Kernel is taking too long to finish, killing") self._kill_kernel() def cleanup(self, connection_file=True): """Clean up resources when the kernel is shut down""" if connection_file: self.cleanup_connection_file() self.cleanup_ipc_files() self._close_control_socket() def shutdown_kernel(self, now=False, restart=False): """Attempts to stop the kernel process cleanly. This attempts to shutdown the kernels cleanly by: 1. Sending it a shutdown message over the shell channel. 2. If that fails, the kernel is shutdown forcibly by sending it a signal. Parameters ---------- now : bool Should the kernel be forcible killed *now*. This skips the first, nice shutdown attempt. restart: bool Will this kernel be restarted after it is shutdown. When this is True, connection files will not be cleaned up. """ # Stop monitoring for restarting while we shutdown. self.stop_restarter() if now: self._kill_kernel() else: self.request_shutdown(restart=restart) # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. self.finish_shutdown() self.cleanup(connection_file=not restart) def restart_kernel(self, now=False, newports=False, **kw): """Restarts a kernel with the arguments that were used to launch it. Parameters ---------- now : bool, optional If True, the kernel is forcefully restarted *immediately*, without having a chance to do any cleanup action. Otherwise the kernel is given 1s to clean up before a forceful restart is issued. In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. newports : bool, optional If the old kernel was launched with random ports, this flag decides whether the same ports and connection file will be used again. If False, the same ports and connection file are used. This is the default. If True, new random port numbers are chosen and a new connection file is written. It is still possible that the newly chosen random port numbers happen to be the same as the old ones. `**kw` : optional Any options specified here will overwrite those used to launch the kernel. """ if self._launch_args is None: raise RuntimeError("Cannot restart the kernel. " "No previous call to 'start_kernel'.") else: # Stop currently running kernel. self.shutdown_kernel(now=now, restart=True) if newports: self.cleanup_random_ports() # Start new kernel. self._launch_args.update(kw) self.start_kernel(**self._launch_args) @property def has_kernel(self): """Has a kernel been started that we are managing.""" return self.kernel is not None def _kill_kernel(self): """Kill the running kernel. This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: if hasattr(signal, 'SIGKILL'): self.signal_kernel(signal.SIGKILL) else: self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. if sys.platform == 'win32': if e.winerror != 5: raise # On Unix, we may get an ESRCH error if the process has already # terminated. Ignore it. else: from errno import ESRCH if e.errno != ESRCH: raise # Block until the kernel terminates. self.kernel.wait() self.kernel = None else: raise RuntimeError("Cannot kill kernel. No kernel is running!") def interrupt_kernel(self): """Interrupts the kernel by sending it a signal. Unlike ``signal_kernel``, this operation is well supported on all platforms. """ if self.has_kernel: interrupt_mode = self.kernel_spec.interrupt_mode if interrupt_mode == 'signal': if sys.platform == 'win32': from .win_interrupt import send_interrupt send_interrupt(self.kernel.win32_interrupt_event) else: self.signal_kernel(signal.SIGINT) elif interrupt_mode == 'message': msg = self.session.msg("interrupt_request", content={}) self._connect_control_socket() self.session.send(self._control_socket, msg) else: raise RuntimeError("Cannot interrupt kernel. No kernel is running!") def signal_kernel(self, signum): """Sends a signal to the process group of the kernel (this usually includes the kernel and any subprocesses spawned by the kernel). Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. """ if self.has_kernel: if hasattr(os, "getpgid") and hasattr(os, "killpg"): try: pgid = os.getpgid(self.kernel.pid) os.killpg(pgid, signum) return except OSError: pass self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") def is_alive(self): """Is the kernel process still running?""" if self.has_kernel: if self.kernel.poll() is None: return True else: return False else: # we don't have a kernel return False KernelManagerABC.register(KernelManager) def start_new_kernel(startup_timeout=60, kernel_name='python', **kwargs): """Start a new kernel, and return its Manager and Client""" km = KernelManager(kernel_name=kernel_name) km.start_kernel(**kwargs) kc = km.client() kc.start_channels() try: kc.wait_for_ready(timeout=startup_timeout) except RuntimeError: kc.stop_channels() km.shutdown_kernel() raise return km, kc @contextmanager def run_kernel(**kwargs): """Context manager to create a kernel in a subprocess. The kernel is shut down when the context exits. Returns ------- kernel_client: connected KernelClient instance """ km, kc = start_new_kernel(**kwargs) try: yield kc finally: kc.stop_channels() km.shutdown_kernel(now=True) ================================================ FILE: lib/client/jupyter_client/managerabc.py ================================================ """Abstract base class for kernel managers.""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import abc from ipython_genutils.py3compat import with_metaclass class KernelManagerABC(with_metaclass(abc.ABCMeta, object)): """KernelManager ABC. The docstrings for this class can be found in the base implementation: `jupyter_client.kernelmanager.KernelManager` """ @abc.abstractproperty def kernel(self): pass #-------------------------------------------------------------------------- # Kernel management #-------------------------------------------------------------------------- @abc.abstractmethod def start_kernel(self, **kw): pass @abc.abstractmethod def shutdown_kernel(self, now=False, restart=False): pass @abc.abstractmethod def restart_kernel(self, now=False, **kw): pass @abc.abstractproperty def has_kernel(self): pass @abc.abstractmethod def interrupt_kernel(self): pass @abc.abstractmethod def signal_kernel(self, signum): pass @abc.abstractmethod def is_alive(self): pass ================================================ FILE: lib/client/jupyter_client/multikernelmanager.py ================================================ """A kernel manager for multiple kernels""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import absolute_import import os import uuid import zmq from traitlets.config.configurable import LoggingConfigurable from ipython_genutils.importstring import import_item from traitlets import ( Instance, Dict, List, Unicode, Any, DottedObjectName ) from ipython_genutils.py3compat import unicode_type from .kernelspec import NATIVE_KERNEL_NAME, KernelSpecManager class DuplicateKernelError(Exception): pass def kernel_method(f): """decorator for proxying MKM.method(kernel_id) to individual KMs by ID""" def wrapped(self, kernel_id, *args, **kwargs): # get the kernel km = self.get_kernel(kernel_id) method = getattr(km, f.__name__) # call the kernel's method r = method(*args, **kwargs) # last thing, call anything defined in the actual class method # such as logging messages f(self, kernel_id, *args, **kwargs) # return the method result return r return wrapped class MultiKernelManager(LoggingConfigurable): """A class for managing multiple kernels.""" default_kernel_name = Unicode(NATIVE_KERNEL_NAME, config=True, help="The name of the default kernel to start" ) kernel_spec_manager = Instance(KernelSpecManager, allow_none=True) kernel_manager_class = DottedObjectName( "jupyter_client.ioloop.IOLoopKernelManager", config=True, help="""The kernel manager class. This is configurable to allow subclassing of the KernelManager for customized behavior. """ ) def _kernel_manager_class_changed(self, name, old, new): self.kernel_manager_factory = import_item(new) kernel_manager_factory = Any(help="this is kernel_manager_class after import") def _kernel_manager_factory_default(self): return import_item(self.kernel_manager_class) context = Instance('zmq.Context') def _context_default(self): return zmq.Context.instance() connection_dir = Unicode('') _kernels = Dict() def list_kernel_ids(self): """Return a list of the kernel ids of the active kernels.""" # Create a copy so we can iterate over kernels in operations # that delete keys. return list(self._kernels.keys()) def __len__(self): """Return the number of running kernels.""" return len(self.list_kernel_ids()) def __contains__(self, kernel_id): return kernel_id in self._kernels def start_kernel(self, kernel_name=None, **kwargs): """Start a new kernel. The caller can pick a kernel_id by passing one in as a keyword arg, otherwise one will be picked using a uuid. The kernel ID for the newly started kernel is returned. """ kernel_id = kwargs.pop('kernel_id', unicode_type(uuid.uuid4())) if kernel_id in self: raise DuplicateKernelError('Kernel already exists: %s' % kernel_id) if kernel_name is None: kernel_name = self.default_kernel_name # kernel_manager_factory is the constructor for the KernelManager # subclass we are using. It can be configured as any Configurable, # including things like its transport and ip. constructor_kwargs = {} if self.kernel_spec_manager: constructor_kwargs['kernel_spec_manager'] = self.kernel_spec_manager km = self.kernel_manager_factory(connection_file=os.path.join( self.connection_dir, "kernel-%s.json" % kernel_id), parent=self, log=self.log, kernel_name=kernel_name, **constructor_kwargs ) km.start_kernel(**kwargs) self._kernels[kernel_id] = km return kernel_id @kernel_method def shutdown_kernel(self, kernel_id, now=False, restart=False): """Shutdown a kernel by its kernel uuid. Parameters ========== kernel_id : uuid The id of the kernel to shutdown. now : bool Should the kernel be shutdown forcibly using a signal. restart : bool Will the kernel be restarted? """ self.log.info("Kernel shutdown: %s" % kernel_id) self.remove_kernel(kernel_id) @kernel_method def request_shutdown(self, kernel_id, restart=False): """Ask a kernel to shut down by its kernel uuid""" @kernel_method def finish_shutdown(self, kernel_id, waittime=None, pollinterval=0.1): """Wait for a kernel to finish shutting down, and kill it if it doesn't """ self.log.info("Kernel shutdown: %s" % kernel_id) @kernel_method def cleanup(self, kernel_id, connection_file=True): """Clean up a kernel's resources""" def remove_kernel(self, kernel_id): """remove a kernel from our mapping. Mainly so that a kernel can be removed if it is already dead, without having to call shutdown_kernel. The kernel object is returned. """ return self._kernels.pop(kernel_id) def shutdown_all(self, now=False): """Shutdown all kernels.""" kids = self.list_kernel_ids() for kid in kids: self.request_shutdown(kid) for kid in kids: self.finish_shutdown(kid) self.cleanup(kid) self.remove_kernel(kid) @kernel_method def interrupt_kernel(self, kernel_id): """Interrupt (SIGINT) the kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ self.log.info("Kernel interrupted: %s" % kernel_id) @kernel_method def signal_kernel(self, kernel_id, signum): """Sends a signal to the kernel by its uuid. Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. Parameters ========== kernel_id : uuid The id of the kernel to signal. """ self.log.info("Signaled Kernel %s with %s" % (kernel_id, signum)) @kernel_method def restart_kernel(self, kernel_id, now=False): """Restart a kernel by its uuid, keeping the same ports. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ self.log.info("Kernel restarted: %s" % kernel_id) @kernel_method def is_alive(self, kernel_id): """Is the kernel alive. This calls KernelManager.is_alive() which calls Popen.poll on the actual kernel subprocess. Parameters ========== kernel_id : uuid The id of the kernel. """ def _check_kernel_id(self, kernel_id): """check that a kernel id is valid""" if kernel_id not in self: raise KeyError("Kernel with id not found: %s" % kernel_id) def get_kernel(self, kernel_id): """Get the single KernelManager object for a kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel. """ self._check_kernel_id(kernel_id) return self._kernels[kernel_id] @kernel_method def add_restart_callback(self, kernel_id, callback, event='restart'): """add a callback for the KernelRestarter""" @kernel_method def remove_restart_callback(self, kernel_id, callback, event='restart'): """remove a callback for the KernelRestarter""" @kernel_method def get_connection_info(self, kernel_id): """Return a dictionary of connection data for a kernel. Parameters ========== kernel_id : uuid The id of the kernel. Returns ======= connection_dict : dict A dict of the information needed to connect to a kernel. This includes the ip address and the integer port numbers of the different channels (stdin_port, iopub_port, shell_port, hb_port). """ @kernel_method def connect_iopub(self, kernel_id, identity=None): """Return a zmq Socket connected to the iopub channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_shell(self, kernel_id, identity=None): """Return a zmq Socket connected to the shell channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_stdin(self, kernel_id, identity=None): """Return a zmq Socket connected to the stdin channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_hb(self, kernel_id, identity=None): """Return a zmq Socket connected to the hb channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ ================================================ FILE: lib/client/jupyter_client/restarter.py ================================================ """A basic kernel monitor with autorestarting. This watches a kernel's state using KernelManager.is_alive and auto restarts the kernel if it dies. It is an incomplete base class, and must be subclassed. """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from traitlets.config.configurable import LoggingConfigurable from traitlets import ( Instance, Float, Dict, Bool, Integer, ) class KernelRestarter(LoggingConfigurable): """Monitor and autorestart a kernel.""" kernel_manager = Instance('jupyter_client.KernelManager') debug = Bool(False, config=True, help="""Whether to include every poll event in debugging output. Has to be set explicitly, because there will be *a lot* of output. """ ) time_to_dead = Float(3.0, config=True, help="""Kernel heartbeat interval in seconds.""" ) restart_limit = Integer(5, config=True, help="""The number of consecutive autorestarts before the kernel is presumed dead.""" ) random_ports_until_alive = Bool(True, config=True, help="""Whether to choose new random ports when restarting before the kernel is alive.""" ) _restarting = Bool(False) _restart_count = Integer(0) _initial_startup = Bool(True) callbacks = Dict() def _callbacks_default(self): return dict(restart=[], dead=[]) def start(self): """Start the polling of the kernel.""" raise NotImplementedError("Must be implemented in a subclass") def stop(self): """Stop the kernel polling.""" raise NotImplementedError("Must be implemented in a subclass") def add_callback(self, f, event='restart'): """register a callback to fire on a particular event Possible values for event: 'restart' (default): kernel has died, and will be restarted. 'dead': restart has failed, kernel will be left dead. """ self.callbacks[event].append(f) def remove_callback(self, f, event='restart'): """unregister a callback to fire on a particular event Possible values for event: 'restart' (default): kernel has died, and will be restarted. 'dead': restart has failed, kernel will be left dead. """ try: self.callbacks[event].remove(f) except ValueError: pass def _fire_callbacks(self, event): """fire our callbacks for a particular event""" for callback in self.callbacks[event]: try: callback() except Exception as e: self.log.error("KernelRestarter: %s callback %r failed", event, callback, exc_info=True) def poll(self): if self.debug: self.log.debug('Polling kernel...') if not self.kernel_manager.is_alive(): if self._restarting: self._restart_count += 1 else: self._restart_count = 1 if self._restart_count >= self.restart_limit: self.log.warning("KernelRestarter: restart failed") self._fire_callbacks('dead') self._restarting = False self._restart_count = 0 self.stop() else: newports = self.random_ports_until_alive and self._initial_startup self.log.info('KernelRestarter: restarting kernel (%i/%i), %s random ports', self._restart_count, self.restart_limit, 'new' if newports else 'keep' ) self._fire_callbacks('restart') self.kernel_manager.restart_kernel(now=True, newports=newports) self._restarting = True else: if self._initial_startup: self._initial_startup = False if self._restarting: self.log.debug("KernelRestarter: restart apparently succeeded") self._restarting = False ================================================ FILE: lib/client/jupyter_client/runapp.py ================================================ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import print_function import logging import signal import time import sys from traitlets.config import catch_config_error from traitlets import ( Instance, Dict, Unicode, Bool, List, CUnicode, Any, Float ) from jupyter_core.application import ( JupyterApp, base_flags, base_aliases ) from . import __version__ from .consoleapp import JupyterConsoleApp, app_aliases, app_flags try: import queue except ImportError: import Queue as queue OUTPUT_TIMEOUT = 10 # copy flags from mixin: flags = dict(base_flags) # start with mixin frontend flags: frontend_flags = dict(app_flags) # update full dict with frontend flags: flags.update(frontend_flags) # copy flags from mixin aliases = dict(base_aliases) # start with mixin frontend flags frontend_aliases = dict(app_aliases) # load updated frontend flags into full dict aliases.update(frontend_aliases) # get flags&aliases into sets, and remove a couple that # shouldn't be scrubbed from backend flags: frontend_aliases = set(frontend_aliases.keys()) frontend_flags = set(frontend_flags.keys()) class RunApp(JupyterApp, JupyterConsoleApp): version = __version__ name = "jupyter run" description = """Run Jupyter kernel code.""" flags = Dict(flags) aliases = Dict(aliases) frontend_aliases = Any(frontend_aliases) frontend_flags = Any(frontend_flags) kernel_timeout = Float(60, config=True, help="""Timeout for giving up on a kernel (in seconds). On first connect and restart, the console tests whether the kernel is running and responsive by sending kernel_info_requests. This sets the timeout in seconds for how long the kernel can take before being presumed dead. """ ) def parse_command_line(self, argv=None): super(RunApp, self).parse_command_line(argv) self.build_kernel_argv(self.extra_args) self.filenames_to_run = self.extra_args[:] @catch_config_error def initialize(self, argv=None): self.log.debug("jupyter run: initialize...") super(RunApp, self).initialize(argv) JupyterConsoleApp.initialize(self) signal.signal(signal.SIGINT, self.handle_sigint) self.init_kernel_info() def handle_sigint(self, *args): if self.kernel_manager: self.kernel_manager.interrupt_kernel() else: print("", file=sys.stderr) error("Cannot interrupt kernels we didn't start.\n") def init_kernel_info(self): """Wait for a kernel to be ready, and store kernel info""" timeout = self.kernel_timeout tic = time.time() self.kernel_client.hb_channel.unpause() msg_id = self.kernel_client.kernel_info() while True: try: reply = self.kernel_client.get_shell_msg(timeout=1) except queue.Empty: if (time.time() - tic) > timeout: raise RuntimeError("Kernel didn't respond to kernel_info_request") else: if reply['parent_header'].get('msg_id') == msg_id: self.kernel_info = reply['content'] return def start(self): self.log.debug("jupyter run: starting...") super(RunApp, self).start() if self.filenames_to_run: for filename in self.filenames_to_run: self.log.debug("jupyter run: executing `%s`" % filename) with open(filename) as fp: code = fp.read() reply = self.kernel_client.execute_interactive(code, timeout=OUTPUT_TIMEOUT) return_code = 0 if reply['content']['status'] == 'ok' else 1 if return_code: raise Exception("jupyter-run error running '%s'" % filename) else: code = sys.stdin.read() reply = self.kernel_client.execute_interactive(code, timeout=OUTPUT_TIMEOUT) return_code = 0 if reply['content']['status'] == 'ok' else 1 if return_code: raise Exception("jupyter-run error running 'stdin'") main = launch_new_instance = RunApp.launch_instance if __name__ == '__main__': main() ================================================ FILE: lib/client/jupyter_client/session.py ================================================ """Session object for building, serializing, sending, and receiving messages. The Session object supports serialization, HMAC signatures, and metadata on messages. Also defined here are utilities for working with Sessions: * A SessionFactory to be used as a base class for configurables that work with Sessions. * A Message object for convenience that allows attribute-access to the msg dict. """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from binascii import b2a_hex import hashlib import hmac import logging import os import pprint import random import warnings from datetime import datetime try: import cPickle pickle = cPickle except: cPickle = None import pickle try: # py3 PICKLE_PROTOCOL = pickle.DEFAULT_PROTOCOL except AttributeError: PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL try: # We are using compare_digest to limit the surface of timing attacks from hmac import compare_digest except ImportError: # Python < 2.7.7: When digests don't match no feedback is provided, # limiting the surface of attack def compare_digest(a,b): return a == b try: from datetime import timezone utc = timezone.utc except ImportError: # Python 2 from dateutil.tz import tzutc utc = tzutc() import zmq from zmq.utils import jsonapi from zmq.eventloop.ioloop import IOLoop from zmq.eventloop.zmqstream import ZMQStream from traitlets.config.configurable import Configurable, LoggingConfigurable from ipython_genutils.importstring import import_item from jupyter_client.jsonutil import extract_dates, squash_dates, date_default from ipython_genutils.py3compat import (str_to_bytes, str_to_unicode, unicode_type, iteritems) from traitlets import (CBytes, Unicode, Bool, Any, Instance, Set, DottedObjectName, CUnicode, Dict, Integer, TraitError, ) from jupyter_client import protocol_version from jupyter_client.adapter import adapt from traitlets.log import get_logger #----------------------------------------------------------------------------- # utility functions #----------------------------------------------------------------------------- def squash_unicode(obj): """coerce unicode back to bytestrings.""" if isinstance(obj,dict): for key in obj.keys(): obj[key] = squash_unicode(obj[key]) if isinstance(key, unicode_type): obj[squash_unicode(key)] = obj.pop(key) elif isinstance(obj, list): for i,v in enumerate(obj): obj[i] = squash_unicode(v) elif isinstance(obj, unicode_type): obj = obj.encode('utf8') return obj #----------------------------------------------------------------------------- # globals and defaults #----------------------------------------------------------------------------- # default values for the thresholds: MAX_ITEMS = 64 MAX_BYTES = 1024 # ISO8601-ify datetime objects # allow unicode # disallow nan, because it's not actually valid JSON json_packer = lambda obj: jsonapi.dumps(obj, default=date_default, ensure_ascii=False, allow_nan=False, ) json_unpacker = lambda s: jsonapi.loads(s) pickle_packer = lambda o: pickle.dumps(squash_dates(o), PICKLE_PROTOCOL) pickle_unpacker = pickle.loads default_packer = json_packer default_unpacker = json_unpacker DELIM = b"" # singleton dummy tracker, which will always report as done DONE = zmq.MessageTracker() #----------------------------------------------------------------------------- # Mixin tools for apps that use Sessions #----------------------------------------------------------------------------- def new_id(): """Generate a new random id. Avoids problematic runtime import in stdlib uuid on Python 2. Returns ------- id string (16 random bytes as hex-encoded text, chunks separated by '-') """ buf = os.urandom(16) return u'-'.join(b2a_hex(x).decode('ascii') for x in ( buf[:4], buf[4:] )) def new_id_bytes(): """Return new_id as ascii bytes""" return new_id().encode('ascii') session_aliases = dict( ident = 'Session.session', user = 'Session.username', keyfile = 'Session.keyfile', ) session_flags = { 'secure' : ({'Session' : { 'key' : new_id_bytes(), 'keyfile' : '' }}, """Use HMAC digests for authentication of messages. Setting this flag will generate a new UUID to use as the HMAC key. """), 'no-secure' : ({'Session' : { 'key' : b'', 'keyfile' : '' }}, """Don't authenticate messages."""), } def default_secure(cfg): """Set the default behavior for a config environment to be secure. If Session.key/keyfile have not been set, set Session.key to a new random UUID. """ warnings.warn("default_secure is deprecated", DeprecationWarning) if 'Session' in cfg: if 'key' in cfg.Session or 'keyfile' in cfg.Session: return # key/keyfile not specified, generate new UUID: cfg.Session.key = new_id_bytes() def utcnow(): """Return timezone-aware UTC timestamp""" return datetime.utcnow().replace(tzinfo=utc) #----------------------------------------------------------------------------- # Classes #----------------------------------------------------------------------------- class SessionFactory(LoggingConfigurable): """The Base class for configurables that have a Session, Context, logger, and IOLoop. """ logname = Unicode('') def _logname_changed(self, name, old, new): self.log = logging.getLogger(new) # not configurable: context = Instance('zmq.Context') def _context_default(self): return zmq.Context.instance() session = Instance('jupyter_client.session.Session', allow_none=True) loop = Instance('tornado.ioloop.IOLoop') def _loop_default(self): return IOLoop.current() def __init__(self, **kwargs): super(SessionFactory, self).__init__(**kwargs) if self.session is None: # construct the session self.session = Session(**kwargs) class Message(object): """A simple message object that maps dict keys to attributes. A Message can be created from a dict and a dict from a Message instance simply by calling dict(msg_obj).""" def __init__(self, msg_dict): dct = self.__dict__ for k, v in iteritems(dict(msg_dict)): if isinstance(v, dict): v = Message(v) dct[k] = v # Having this iterator lets dict(msg_obj) work out of the box. def __iter__(self): return iter(iteritems(self.__dict__)) def __repr__(self): return repr(self.__dict__) def __str__(self): return pprint.pformat(self.__dict__) def __contains__(self, k): return k in self.__dict__ def __getitem__(self, k): return self.__dict__[k] def msg_header(msg_id, msg_type, username, session): """Create a new message header""" date = utcnow() version = protocol_version return locals() def extract_header(msg_or_header): """Given a message or header, return the header.""" if not msg_or_header: return {} try: # See if msg_or_header is the entire message. h = msg_or_header['header'] except KeyError: try: # See if msg_or_header is just the header h = msg_or_header['msg_id'] except KeyError: raise else: h = msg_or_header if not isinstance(h, dict): h = dict(h) return h class Session(Configurable): """Object for handling serialization and sending of messages. The Session object handles building messages and sending them with ZMQ sockets or ZMQStream objects. Objects can communicate with each other over the network via Session objects, and only need to work with the dict-based IPython message spec. The Session will handle serialization/deserialization, security, and metadata. Sessions support configurable serialization via packer/unpacker traits, and signing with HMAC digests via the key/keyfile traits. Parameters ---------- debug : bool whether to trigger extra debugging statements packer/unpacker : str : 'json', 'pickle' or import_string importstrings for methods to serialize message parts. If just 'json' or 'pickle', predefined JSON and pickle packers will be used. Otherwise, the entire importstring must be used. The functions must accept at least valid JSON input, and output *bytes*. For example, to use msgpack: packer = 'msgpack.packb', unpacker='msgpack.unpackb' pack/unpack : callables You can also set the pack/unpack callables for serialization directly. session : bytes the ID of this Session object. The default is to generate a new UUID. username : unicode username added to message headers. The default is to ask the OS. key : bytes The key used to initialize an HMAC signature. If unset, messages will not be signed or checked. keyfile : filepath The file containing a key. If this is set, `key` will be initialized to the contents of the file. """ debug = Bool(False, config=True, help="""Debug output in the Session""") check_pid = Bool(True, config=True, help="""Whether to check PID to protect against calls after fork. This check can be disabled if fork-safety is handled elsewhere. """) packer = DottedObjectName('json',config=True, help="""The name of the packer for serializing messages. Should be one of 'json', 'pickle', or an import name for a custom callable serializer.""") def _packer_changed(self, name, old, new): if new.lower() == 'json': self.pack = json_packer self.unpack = json_unpacker self.unpacker = new elif new.lower() == 'pickle': self.pack = pickle_packer self.unpack = pickle_unpacker self.unpacker = new else: self.pack = import_item(str(new)) unpacker = DottedObjectName('json', config=True, help="""The name of the unpacker for unserializing messages. Only used with custom functions for `packer`.""") def _unpacker_changed(self, name, old, new): if new.lower() == 'json': self.pack = json_packer self.unpack = json_unpacker self.packer = new elif new.lower() == 'pickle': self.pack = pickle_packer self.unpack = pickle_unpacker self.packer = new else: self.unpack = import_item(str(new)) session = CUnicode(u'', config=True, help="""The UUID identifying this session.""") def _session_default(self): u = new_id() self.bsession = u.encode('ascii') return u def _session_changed(self, name, old, new): self.bsession = self.session.encode('ascii') # bsession is the session as bytes bsession = CBytes(b'') username = Unicode(str_to_unicode(os.environ.get('USER', 'username')), help="""Username for the Session. Default is your system username.""", config=True) metadata = Dict({}, config=True, help="""Metadata dictionary, which serves as the default top-level metadata dict for each message.""") # if 0, no adapting to do. adapt_version = Integer(0) # message signature related traits: key = CBytes(config=True, help="""execution key, for signing messages.""") def _key_default(self): return new_id_bytes() def _key_changed(self): self._new_auth() signature_scheme = Unicode('hmac-sha256', config=True, help="""The digest scheme used to construct the message signatures. Must have the form 'hmac-HASH'.""") def _signature_scheme_changed(self, name, old, new): if not new.startswith('hmac-'): raise TraitError("signature_scheme must start with 'hmac-', got %r" % new) hash_name = new.split('-', 1)[1] try: self.digest_mod = getattr(hashlib, hash_name) except AttributeError: raise TraitError("hashlib has no such attribute: %s" % hash_name) self._new_auth() digest_mod = Any() def _digest_mod_default(self): return hashlib.sha256 auth = Instance(hmac.HMAC, allow_none=True) def _new_auth(self): if self.key: self.auth = hmac.HMAC(self.key, digestmod=self.digest_mod) else: self.auth = None digest_history = Set() digest_history_size = Integer(2**16, config=True, help="""The maximum number of digests to remember. The digest history will be culled when it exceeds this value. """ ) keyfile = Unicode('', config=True, help="""path to file containing execution key.""") def _keyfile_changed(self, name, old, new): with open(new, 'rb') as f: self.key = f.read().strip() # for protecting against sends from forks pid = Integer() # serialization traits: pack = Any(default_packer) # the actual packer function def _pack_changed(self, name, old, new): if not callable(new): raise TypeError("packer must be callable, not %s"%type(new)) unpack = Any(default_unpacker) # the actual packer function def _unpack_changed(self, name, old, new): # unpacker is not checked - it is assumed to be if not callable(new): raise TypeError("unpacker must be callable, not %s"%type(new)) # thresholds: copy_threshold = Integer(2**16, config=True, help="Threshold (in bytes) beyond which a buffer should be sent without copying.") buffer_threshold = Integer(MAX_BYTES, config=True, help="Threshold (in bytes) beyond which an object's buffer should be extracted to avoid pickling.") item_threshold = Integer(MAX_ITEMS, config=True, help="""The maximum number of items for a container to be introspected for custom serialization. Containers larger than this are pickled outright. """ ) def __init__(self, **kwargs): """create a Session object Parameters ---------- debug : bool whether to trigger extra debugging statements packer/unpacker : str : 'json', 'pickle' or import_string importstrings for methods to serialize message parts. If just 'json' or 'pickle', predefined JSON and pickle packers will be used. Otherwise, the entire importstring must be used. The functions must accept at least valid JSON input, and output *bytes*. For example, to use msgpack: packer = 'msgpack.packb', unpacker='msgpack.unpackb' pack/unpack : callables You can also set the pack/unpack callables for serialization directly. session : unicode (must be ascii) the ID of this Session object. The default is to generate a new UUID. bsession : bytes The session as bytes username : unicode username added to message headers. The default is to ask the OS. key : bytes The key used to initialize an HMAC signature. If unset, messages will not be signed or checked. signature_scheme : str The message digest scheme. Currently must be of the form 'hmac-HASH', where 'HASH' is a hashing function available in Python's hashlib. The default is 'hmac-sha256'. This is ignored if 'key' is empty. keyfile : filepath The file containing a key. If this is set, `key` will be initialized to the contents of the file. """ super(Session, self).__init__(**kwargs) self._check_packers() self.none = self.pack({}) # ensure self._session_default() if necessary, so bsession is defined: self.session self.pid = os.getpid() self._new_auth() if not self.key: get_logger().warning("Message signing is disabled. This is insecure and not recommended!") def clone(self): """Create a copy of this Session Useful when connecting multiple times to a given kernel. This prevents a shared digest_history warning about duplicate digests due to multiple connections to IOPub in the same process. .. versionadded:: 5.1 """ # make a copy new_session = type(self)() for name in self.traits(): setattr(new_session, name, getattr(self, name)) # fork digest_history new_session.digest_history = set() new_session.digest_history.update(self.digest_history) return new_session @property def msg_id(self): """always return new uuid""" return new_id() def _check_packers(self): """check packers for datetime support.""" pack = self.pack unpack = self.unpack # check simple serialization msg = dict(a=[1,'hi']) try: packed = pack(msg) except Exception as e: msg = "packer '{packer}' could not serialize a simple message: {e}{jsonmsg}" if self.packer == 'json': jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod else: jsonmsg = "" raise ValueError( msg.format(packer=self.packer, e=e, jsonmsg=jsonmsg) ) # ensure packed message is bytes if not isinstance(packed, bytes): raise ValueError("message packed to %r, but bytes are required"%type(packed)) # check that unpack is pack's inverse try: unpacked = unpack(packed) assert unpacked == msg except Exception as e: msg = "unpacker '{unpacker}' could not handle output from packer '{packer}': {e}{jsonmsg}" if self.packer == 'json': jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod else: jsonmsg = "" raise ValueError( msg.format(packer=self.packer, unpacker=self.unpacker, e=e, jsonmsg=jsonmsg) ) # check datetime support msg = dict(t=utcnow()) try: unpacked = unpack(pack(msg)) if isinstance(unpacked['t'], datetime): raise ValueError("Shouldn't deserialize to datetime") except Exception: self.pack = lambda o: pack(squash_dates(o)) self.unpack = lambda s: unpack(s) def msg_header(self, msg_type): return msg_header(self.msg_id, msg_type, self.username, self.session) def msg(self, msg_type, content=None, parent=None, header=None, metadata=None): """Return the nested message dict. This format is different from what is sent over the wire. The serialize/deserialize methods converts this nested message dict to the wire format, which is a list of message parts. """ msg = {} header = self.msg_header(msg_type) if header is None else header msg['header'] = header msg['msg_id'] = header['msg_id'] msg['msg_type'] = header['msg_type'] msg['parent_header'] = {} if parent is None else extract_header(parent) msg['content'] = {} if content is None else content msg['metadata'] = self.metadata.copy() if metadata is not None: msg['metadata'].update(metadata) return msg def sign(self, msg_list): """Sign a message with HMAC digest. If no auth, return b''. Parameters ---------- msg_list : list The [p_header,p_parent,p_content] part of the message list. """ if self.auth is None: return b'' h = self.auth.copy() for m in msg_list: h.update(m) return str_to_bytes(h.hexdigest()) def serialize(self, msg, ident=None): """Serialize the message components to bytes. This is roughly the inverse of deserialize. The serialize/deserialize methods work with full message lists, whereas pack/unpack work with the individual message parts in the message list. Parameters ---------- msg : dict or Message The next message dict as returned by the self.msg method. Returns ------- msg_list : list The list of bytes objects to be sent with the format:: [ident1, ident2, ..., DELIM, HMAC, p_header, p_parent, p_metadata, p_content, buffer1, buffer2, ...] In this list, the ``p_*`` entities are the packed or serialized versions, so if JSON is used, these are utf8 encoded JSON strings. """ content = msg.get('content', {}) if content is None: content = self.none elif isinstance(content, dict): content = self.pack(content) elif isinstance(content, bytes): # content is already packed, as in a relayed message pass elif isinstance(content, unicode_type): # should be bytes, but JSON often spits out unicode content = content.encode('utf8') else: raise TypeError("Content incorrect type: %s"%type(content)) real_message = [self.pack(msg['header']), self.pack(msg['parent_header']), self.pack(msg['metadata']), content, ] to_send = [] if isinstance(ident, list): # accept list of idents to_send.extend(ident) elif ident is not None: to_send.append(ident) to_send.append(DELIM) signature = self.sign(real_message) to_send.append(signature) to_send.extend(real_message) return to_send def send(self, stream, msg_or_type, content=None, parent=None, ident=None, buffers=None, track=False, header=None, metadata=None): """Build and send a message via stream or socket. The message format used by this function internally is as follows: [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content, buffer1,buffer2,...] The serialize/deserialize methods convert the nested message dict into this format. Parameters ---------- stream : zmq.Socket or ZMQStream The socket-like object used to send the data. msg_or_type : str or Message/dict Normally, msg_or_type will be a msg_type unless a message is being sent more than once. If a header is supplied, this can be set to None and the msg_type will be pulled from the header. content : dict or None The content of the message (ignored if msg_or_type is a message). header : dict or None The header dict for the message (ignored if msg_to_type is a message). parent : Message or dict or None The parent or parent header describing the parent of this message (ignored if msg_or_type is a message). ident : bytes or list of bytes The zmq.IDENTITY routing path. metadata : dict or None The metadata describing the message buffers : list or None The already-serialized buffers to be appended to the message. track : bool Whether to track. Only for use with Sockets, because ZMQStream objects cannot track messages. Returns ------- msg : dict The constructed message. """ if not isinstance(stream, zmq.Socket): # ZMQStreams and dummy sockets do not support tracking. track = False if isinstance(msg_or_type, (Message, dict)): # We got a Message or message dict, not a msg_type so don't # build a new Message. msg = msg_or_type buffers = buffers or msg.get('buffers', []) else: msg = self.msg(msg_or_type, content=content, parent=parent, header=header, metadata=metadata) if self.check_pid and not os.getpid() == self.pid: get_logger().warning("WARNING: attempted to send message from fork\n%s", msg ) return buffers = [] if buffers is None else buffers for idx, buf in enumerate(buffers): if isinstance(buf, memoryview): view = buf else: try: # check to see if buf supports the buffer protocol. view = memoryview(buf) except TypeError: raise TypeError("Buffer objects must support the buffer protocol.") # memoryview.contiguous is new in 3.3, # just skip the check on Python 2 if hasattr(view, 'contiguous') and not view.contiguous: # zmq requires memoryviews to be contiguous raise ValueError("Buffer %i (%r) is not contiguous" % (idx, buf)) if self.adapt_version: msg = adapt(msg, self.adapt_version) to_send = self.serialize(msg, ident) to_send.extend(buffers) longest = max([ len(s) for s in to_send ]) copy = (longest < self.copy_threshold) if buffers and track and not copy: # only really track when we are doing zero-copy buffers tracker = stream.send_multipart(to_send, copy=False, track=True) else: # use dummy tracker, which will be done immediately tracker = DONE stream.send_multipart(to_send, copy=copy) if self.debug: pprint.pprint(msg) pprint.pprint(to_send) pprint.pprint(buffers) msg['tracker'] = tracker return msg def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None): """Send a raw message via ident path. This method is used to send a already serialized message. Parameters ---------- stream : ZMQStream or Socket The ZMQ stream or socket to use for sending the message. msg_list : list The serialized list of messages to send. This only includes the [p_header,p_parent,p_metadata,p_content,buffer1,buffer2,...] portion of the message. ident : ident or list A single ident or a list of idents to use in sending. """ to_send = [] if isinstance(ident, bytes): ident = [ident] if ident is not None: to_send.extend(ident) to_send.append(DELIM) to_send.append(self.sign(msg_list)) to_send.extend(msg_list) stream.send_multipart(to_send, flags, copy=copy) def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True): """Receive and unpack a message. Parameters ---------- socket : ZMQStream or Socket The socket or stream to use in receiving. Returns ------- [idents], msg [idents] is a list of idents and msg is a nested message dict of same format as self.msg returns. """ if isinstance(socket, ZMQStream): socket = socket.socket try: msg_list = socket.recv_multipart(mode, copy=copy) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: # We can convert EAGAIN to None as we know in this case # recv_multipart won't return None. return None,None else: raise # split multipart message into identity list and message dict # invalid large messages can cause very expensive string comparisons idents, msg_list = self.feed_identities(msg_list, copy) try: return idents, self.deserialize(msg_list, content=content, copy=copy) except Exception as e: # TODO: handle it raise e def feed_identities(self, msg_list, copy=True): """Split the identities from the rest of the message. Feed until DELIM is reached, then return the prefix as idents and remainder as msg_list. This is easily broken by setting an IDENT to DELIM, but that would be silly. Parameters ---------- msg_list : a list of Message or bytes objects The message to be split. copy : bool flag determining whether the arguments are bytes or Messages Returns ------- (idents, msg_list) : two lists idents will always be a list of bytes, each of which is a ZMQ identity. msg_list will be a list of bytes or zmq.Messages of the form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and should be unpackable/unserializable via self.deserialize at this point. """ if copy: idx = msg_list.index(DELIM) return msg_list[:idx], msg_list[idx+1:] else: failed = True for idx,m in enumerate(msg_list): if m.bytes == DELIM: failed = False break if failed: raise ValueError("DELIM not in msg_list") idents, msg_list = msg_list[:idx], msg_list[idx+1:] return [m.bytes for m in idents], msg_list def _add_digest(self, signature): """add a digest to history to protect against replay attacks""" if self.digest_history_size == 0: # no history, never add digests return self.digest_history.add(signature) if len(self.digest_history) > self.digest_history_size: # threshold reached, cull 10% self._cull_digest_history() def _cull_digest_history(self): """cull the digest history Removes a randomly selected 10% of the digest history """ current = len(self.digest_history) n_to_cull = max(int(current // 10), current - self.digest_history_size) if n_to_cull >= current: self.digest_history = set() return to_cull = random.sample(self.digest_history, n_to_cull) self.digest_history.difference_update(to_cull) def deserialize(self, msg_list, content=True, copy=True): """Unserialize a msg_list to a nested message dict. This is roughly the inverse of serialize. The serialize/deserialize methods work with full message lists, whereas pack/unpack work with the individual message parts in the message list. Parameters ---------- msg_list : list of bytes or Message objects The list of message parts of the form [HMAC,p_header,p_parent, p_metadata,p_content,buffer1,buffer2,...]. content : bool (True) Whether to unpack the content dict (True), or leave it packed (False). copy : bool (True) Whether msg_list contains bytes (True) or the non-copying Message objects in each place (False). Returns ------- msg : dict The nested message dict with top-level keys [header, parent_header, content, buffers]. The buffers are returned as memoryviews. """ minlen = 5 message = {} if not copy: # pyzmq didn't copy the first parts of the message, so we'll do it for i in range(minlen): msg_list[i] = msg_list[i].bytes if self.auth is not None: signature = msg_list[0] if not signature: raise ValueError("Unsigned Message") if signature in self.digest_history: raise ValueError("Duplicate Signature: %r" % signature) if content: # Only store signature if we are unpacking content, don't store if just peeking. self._add_digest(signature) check = self.sign(msg_list[1:5]) if not compare_digest(signature, check): raise ValueError("Invalid Signature: %r" % signature) if not len(msg_list) >= minlen: raise TypeError("malformed message, must have at least %i elements"%minlen) header = self.unpack(msg_list[1]) message['header'] = extract_dates(header) message['msg_id'] = header['msg_id'] message['msg_type'] = header['msg_type'] message['parent_header'] = extract_dates(self.unpack(msg_list[2])) message['metadata'] = self.unpack(msg_list[3]) if content: message['content'] = self.unpack(msg_list[4]) else: message['content'] = msg_list[4] buffers = [memoryview(b) for b in msg_list[5:]] if buffers and buffers[0].shape is None: # force copy to workaround pyzmq #646 buffers = [memoryview(b.bytes) for b in msg_list[5:]] message['buffers'] = buffers if self.debug: pprint.pprint(message) # adapt to the current version return adapt(message) def unserialize(self, *args, **kwargs): warnings.warn( "Session.unserialize is deprecated. Use Session.deserialize.", DeprecationWarning, ) return self.deserialize(*args, **kwargs) def test_msg2obj(): am = dict(x=1) ao = Message(am) assert ao.x == am['x'] am['y'] = dict(z=1) ao = Message(am) assert ao.y.z == am['y']['z'] k1, k2 = 'y', 'z' assert ao[k1][k2] == am[k1][k2] am2 = dict(ao) assert am['x'] == am2['x'] assert am['y']['z'] == am2['y']['z'] ================================================ FILE: lib/client/jupyter_client/threaded.py ================================================ """ Defines a KernelClient that provides thread-safe sockets with async callbacks on message replies. """ from __future__ import absolute_import import atexit import errno import sys from threading import Thread, Event import time # import ZMQError in top-level namespace, to avoid ugly attribute-error messages # during garbage collection of threads at exit: from zmq import ZMQError from zmq.eventloop import ioloop, zmqstream # Local imports from traitlets import Type, Instance from jupyter_client.channels import HBChannel from jupyter_client import KernelClient class ThreadedZMQSocketChannel(object): """A ZMQ socket invoking a callback in the ioloop""" session = None socket = None ioloop = None stream = None _inspect = None def __init__(self, socket, session, loop): """Create a channel. Parameters ---------- socket : :class:`zmq.Socket` The ZMQ socket to use. session : :class:`session.Session` The session to use. loop A pyzmq ioloop to connect the socket to using a ZMQStream """ super(ThreadedZMQSocketChannel, self).__init__() self.socket = socket self.session = session self.ioloop = loop evt = Event() def setup_stream(): self.stream = zmqstream.ZMQStream(self.socket, self.ioloop) self.stream.on_recv(self._handle_recv) evt.set() self.ioloop.add_callback(setup_stream) evt.wait() _is_alive = False def is_alive(self): return self._is_alive def start(self): self._is_alive = True def stop(self): self._is_alive = False def close(self): if self.socket is not None: try: self.socket.close(linger=0) except Exception: pass self.socket = None def send(self, msg): """Queue a message to be sent from the IOLoop's thread. Parameters ---------- msg : message to send This is threadsafe, as it uses IOLoop.add_callback to give the loop's thread control of the action. """ def thread_send(): self.session.send(self.stream, msg) self.ioloop.add_callback(thread_send) def _handle_recv(self, msg): """Callback for stream.on_recv. Unpacks message, and calls handlers with it. """ ident,smsg = self.session.feed_identities(msg) msg = self.session.deserialize(smsg) # let client inspect messages if self._inspect: self._inspect(msg) self.call_handlers(msg) def call_handlers(self, msg): """This method is called in the ioloop thread when a message arrives. Subclasses should override this method to handle incoming messages. It is important to remember that this method is called in the thread so that some logic must be done to ensure that the application level handlers are called in the application thread. """ pass def process_events(self): """Subclasses should override this with a method processing any pending GUI events. """ pass def flush(self, timeout=1.0): """Immediately processes all pending messages on this channel. This is only used for the IOPub channel. Callers should use this method to ensure that :meth:`call_handlers` has been called for all messages that have been received on the 0MQ SUB socket of this channel. This method is thread safe. Parameters ---------- timeout : float, optional The maximum amount of time to spend flushing, in seconds. The default is one second. """ # We do the IOLoop callback process twice to ensure that the IOLoop # gets to perform at least one full poll. stop_time = time.time() + timeout for i in range(2): self._flushed = False self.ioloop.add_callback(self._flush) while not self._flushed and time.time() < stop_time: time.sleep(0.01) def _flush(self): """Callback for :method:`self.flush`.""" self.stream.flush() self._flushed = True class IOLoopThread(Thread): """Run a pyzmq ioloop in a thread to send and receive messages """ _exiting = False ioloop = None def __init__(self): super(IOLoopThread, self).__init__() self.daemon = True @staticmethod @atexit.register def _notice_exit(): # Class definitions can be torn down during interpreter shutdown. # We only need to set _exiting flag if this hasn't happened. if IOLoopThread is not None: IOLoopThread._exiting = True def start(self): """Start the IOLoop thread Don't return until self.ioloop is defined, which is created in the thread """ self._start_event = Event() Thread.start(self) self._start_event.wait() def run(self): """Run my loop, ignoring EINTR events in the poller""" if 'asyncio' in sys.modules: # tornado may be using asyncio, # ensure an eventloop exists for this thread import asyncio asyncio.set_event_loop(asyncio.new_event_loop()) self.ioloop = ioloop.IOLoop() # signal that self.ioloop is defined self._start_event.set() while True: try: self.ioloop.start() except ZMQError as e: if e.errno == errno.EINTR: continue else: raise except Exception: if self._exiting: break else: raise else: break def stop(self): """Stop the channel's event loop and join its thread. This calls :meth:`~threading.Thread.join` and returns when the thread terminates. :class:`RuntimeError` will be raised if :meth:`~threading.Thread.start` is called again. """ if self.ioloop is not None: self.ioloop.add_callback(self.ioloop.stop) self.join() self.close() self.ioloop = None def close(self): if self.ioloop is not None: try: self.ioloop.close(all_fds=True) except Exception: pass class ThreadedKernelClient(KernelClient): """ A KernelClient that provides thread-safe sockets with async callbacks on message replies. """ @property def ioloop(self): return self.ioloop_thread.ioloop ioloop_thread = Instance(IOLoopThread, allow_none=True) def start_channels(self, shell=True, iopub=True, stdin=True, hb=True): self.ioloop_thread = IOLoopThread() self.ioloop_thread.start() if shell: self.shell_channel._inspect = self._check_kernel_info_reply super(ThreadedKernelClient, self).start_channels(shell, iopub, stdin, hb) def _check_kernel_info_reply(self, msg): """This is run in the ioloop thread when the kernel info reply is received """ if msg['msg_type'] == 'kernel_info_reply': self._handle_kernel_info_reply(msg) self.shell_channel._inspect = None def stop_channels(self): super(ThreadedKernelClient, self).stop_channels() if self.ioloop_thread.is_alive(): self.ioloop_thread.stop() iopub_channel_class = Type(ThreadedZMQSocketChannel) shell_channel_class = Type(ThreadedZMQSocketChannel) stdin_channel_class = Type(ThreadedZMQSocketChannel) hb_channel_class = Type(HBChannel) ================================================ FILE: lib/client/jupyter_client/win_interrupt.py ================================================ """Use a Windows event to interrupt a child process like SIGINT. The child needs to explicitly listen for this - see ipykernel.parentpoller.ParentPollerWindows for a Python implementation. """ import ctypes def create_interrupt_event(): """Create an interrupt event handle. The parent process should call this to create the interrupt event that is passed to the child process. It should store this handle and use it with ``send_interrupt`` to interrupt the child process. """ # Create a security attributes struct that permits inheritance of the # handle by new processes. # FIXME: We can clean up this mess by requiring pywin32 for IPython. class SECURITY_ATTRIBUTES(ctypes.Structure): _fields_ = [ ("nLength", ctypes.c_int), ("lpSecurityDescriptor", ctypes.c_void_p), ("bInheritHandle", ctypes.c_int) ] sa = SECURITY_ATTRIBUTES() sa_p = ctypes.pointer(sa) sa.nLength = ctypes.sizeof(SECURITY_ATTRIBUTES) sa.lpSecurityDescriptor = 0 sa.bInheritHandle = 1 return ctypes.windll.kernel32.CreateEventA( sa_p, # lpEventAttributes False, # bManualReset False, # bInitialState '') # lpName def send_interrupt(interrupt_handle): """ Sends an interrupt event using the specified handle. """ ctypes.windll.kernel32.SetEvent(interrupt_handle) ================================================ FILE: lib/client/jupyter_core/__init__.py ================================================ from .version import version_info, __version__ ================================================ FILE: lib/client/jupyter_core/__main__.py ================================================ """Launch the root jupyter command""" from .command import main main() ================================================ FILE: lib/client/jupyter_core/application.py ================================================ # encoding: utf-8 """ A base Application class for Jupyter applications. All Jupyter applications should inherit from this. """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import print_function from copy import deepcopy import logging import os import sys try: # py3 from shutil import which except ImportError: from .utils.shutil_which import which try: raw_input except NameError: # py3 raw_input = input from traitlets.config.application import Application, catch_config_error from traitlets.config.loader import ConfigFileNotFound from traitlets import Unicode, Bool, List, observe from .utils import ensure_dir_exists from ipython_genutils import py3compat from .paths import ( jupyter_config_dir, jupyter_data_dir, jupyter_runtime_dir, jupyter_path, jupyter_config_path, ) # aliases and flags base_aliases = { 'log-level' : 'Application.log_level', 'config' : 'JupyterApp.config_file', } base_flags = { 'debug': ({'Application' : {'log_level' : logging.DEBUG}}, "set log level to logging.DEBUG (maximize logging output)"), 'generate-config': ({'JupyterApp': {'generate_config': True}}, "generate default config file"), 'y': ({'JupyterApp': {'answer_yes': True}}, "Answer yes to any questions instead of prompting."), } class NoStart(Exception): """Exception to raise when an application shouldn't start""" class JupyterApp(Application): """Base class for Jupyter applications""" name = 'jupyter' # override in subclasses description = "A Jupyter Application" aliases = base_aliases flags = base_flags def _log_level_default(self): return logging.INFO jupyter_path = List(Unicode()) def _jupyter_path_default(self): return jupyter_path() config_dir = Unicode() def _config_dir_default(self): return jupyter_config_dir() @property def config_file_paths(self): path = jupyter_config_path() if self.config_dir not in path: path.insert(0, self.config_dir) path.insert(0, py3compat.getcwd()) return path data_dir = Unicode() def _data_dir_default(self): d = jupyter_data_dir() ensure_dir_exists(d, mode=0o700) return d runtime_dir = Unicode() def _runtime_dir_default(self): rd = jupyter_runtime_dir() ensure_dir_exists(rd, mode=0o700) return rd @observe('runtime_dir') def _runtime_dir_changed(self, change): ensure_dir_exists(change['new'], mode=0o700) generate_config = Bool(False, config=True, help="""Generate default config file.""" ) config_file_name = Unicode(config=True, help="Specify a config file to load." ) def _config_file_name_default(self): if not self.name: return '' return self.name.replace('-','_') + u'_config' config_file = Unicode(config=True, help="""Full path of a config file.""", ) answer_yes = Bool(False, config=True, help="""Answer yes to any prompts.""" ) def write_default_config(self): """Write our default config to a .py config file""" if self.config_file: config_file = self.config_file else: config_file = os.path.join(self.config_dir, self.config_file_name + '.py') if os.path.exists(config_file) and not self.answer_yes: answer = '' def ask(): prompt = "Overwrite %s with default config? [y/N]" % config_file try: return raw_input(prompt).lower() or 'n' except KeyboardInterrupt: print('') # empty line return 'n' answer = ask() while not answer.startswith(('y', 'n')): print("Please answer 'yes' or 'no'") answer = ask() if answer.startswith('n'): return config_text = self.generate_config_file() if isinstance(config_text, bytes): config_text = config_text.decode('utf8') print("Writing default config to: %s" % config_file) ensure_dir_exists(os.path.abspath(os.path.dirname(config_file)), 0o700) with open(config_file, mode='w') as f: f.write(config_text) def migrate_config(self): """Migrate config/data from IPython 3""" if os.path.exists(os.path.join(self.config_dir, 'migrated')): # already migrated return from .migrate import get_ipython_dir, migrate # No IPython dir, nothing to migrate if not os.path.exists(get_ipython_dir()): return migrate() def load_config_file(self, suppress_errors=True): """Load the config file. By default, errors in loading config are handled, and a warning printed on screen. For testing, the suppress_errors option is set to False, so errors will make tests fail. """ self.log.debug("Searching %s for config files", self.config_file_paths) base_config = 'jupyter_config' try: super(JupyterApp, self).load_config_file( base_config, path=self.config_file_paths, ) except ConfigFileNotFound: # ignore errors loading parent self.log.debug("Config file %s not found", base_config) pass if self.config_file: path, config_file_name = os.path.split(self.config_file) else: path = self.config_file_paths config_file_name = self.config_file_name if not config_file_name or (config_file_name == base_config): return try: super(JupyterApp, self).load_config_file( config_file_name, path=path ) except ConfigFileNotFound: self.log.debug("Config file not found, skipping: %s", config_file_name) except Exception: # Reraise errors for testing purposes, or if set in # self.raise_config_file_errors if (not suppress_errors) or self.raise_config_file_errors: raise self.log.warning("Error loading config file: %s" % config_file_name, exc_info=True) # subcommand-related def _find_subcommand(self, name): name = '{}-{}'.format(self.name, name) return which(name) @property def _dispatching(self): """Return whether we are dispatching to another command or running ourselves. """ return bool(self.generate_config or self.subapp or self.subcommand) subcommand = Unicode() @catch_config_error def initialize(self, argv=None): # don't hook up crash handler before parsing command-line if argv is None: argv = sys.argv[1:] if argv: subc = self._find_subcommand(argv[0]) if subc: self.argv = argv self.subcommand = subc return self.parse_command_line(argv) cl_config = deepcopy(self.config) if self._dispatching: return self.migrate_config() self.load_config_file() # enforce cl-opts override configfile opts: self.update_config(cl_config) def start(self): """Start the whole thing""" if self.subcommand: os.execv(self.subcommand, [self.subcommand] + self.argv[1:]) raise NoStart() if self.subapp: self.subapp.start() raise NoStart() if self.generate_config: self.write_default_config() raise NoStart() @classmethod def launch_instance(cls, argv=None, **kwargs): """Launch an instance of a Jupyter Application""" try: return super(JupyterApp, cls).launch_instance(argv=argv, **kwargs) except NoStart: return if __name__ == '__main__': JupyterApp.launch_instance() ================================================ FILE: lib/client/jupyter_core/command.py ================================================ """The root `jupyter` command. This does nothing other than dispatch to subcommands or output path info. """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import print_function import argparse import errno import json import os import sys import sysconfig from subprocess import Popen try: # py3 from shutil import which except ImportError: from .utils.shutil_which import which from . import paths from .version import __version__ class JupyterParser(argparse.ArgumentParser): @property def epilog(self): """Add subcommands to epilog on request Avoids searching PATH for subcommands unless help output is requested. """ return 'Available subcommands: %s' % ' '.join(list_subcommands()) @epilog.setter def epilog(self, x): """Ignore epilog set in Parser.__init__""" pass def jupyter_parser(): parser = JupyterParser( description="Jupyter: Interactive Computing", ) group = parser.add_mutually_exclusive_group(required=True) # don't use argparse's version action because it prints to stderr on py2 group.add_argument('--version', action='store_true', help="show the jupyter command's version and exit") group.add_argument('subcommand', type=str, nargs='?', help='the subcommand to launch') group.add_argument('--config-dir', action='store_true', help="show Jupyter config dir") group.add_argument('--data-dir', action='store_true', help="show Jupyter data dir") group.add_argument('--runtime-dir', action='store_true', help="show Jupyter runtime dir") group.add_argument('--paths', action='store_true', help="show all Jupyter paths. Add --json for machine-readable format.") parser.add_argument('--json', action='store_true', help="output paths as machine-readable json") return parser def list_subcommands(): """List all jupyter subcommands searches PATH for `jupyter-name` Returns a list of jupyter's subcommand names, without the `jupyter-` prefix. Nested children (e.g. jupyter-sub-subsub) are not included. """ subcommand_tuples = set() # construct a set of `('foo', 'bar') from `jupyter-foo-bar` for d in _path_with_self(): try: names = os.listdir(d) except OSError: continue for name in names: if name.startswith('jupyter-'): if sys.platform.startswith('win'): # remove file-extension on Windows name = os.path.splitext(name)[0] subcommand_tuples.add(tuple(name.split('-')[1:])) # build a set of subcommand strings, excluding subcommands whose parents are defined subcommands = set() # Only include `jupyter-foo-bar` if `jupyter-foo` is not already present for sub_tup in subcommand_tuples: if not any(sub_tup[:i] in subcommand_tuples for i in range(1, len(sub_tup))): subcommands.add('-'.join(sub_tup)) return sorted(subcommands) def _execvp(cmd, argv): """execvp, except on Windows where it uses Popen Python provides execvp on Windows, but its behavior is problematic (Python bug#9148). """ if sys.platform.startswith('win'): # PATH is ignored when shell=False, # so rely on shutil.which try: from shutil import which except ImportError: from .utils.shutil_which import which cmd_path = which(cmd) if cmd_path is None: raise OSError('%r not found' % cmd, errno.ENOENT) p = Popen([cmd_path] + argv[1:]) # Don't raise KeyboardInterrupt in the parent process. # Set this after spawning, to avoid subprocess inheriting handler. import signal signal.signal(signal.SIGINT, signal.SIG_IGN) p.wait() sys.exit(p.returncode) else: os.execvp(cmd, argv) def _jupyter_abspath(subcommand): """This method get the abspath of a specified jupyter-subcommand with no changes on ENV. """ # get env PATH with self search_path = os.pathsep.join(_path_with_self()) # get the abs path for the jupyter- jupyter_subcommand = 'jupyter-{}'.format(subcommand) abs_path = which(jupyter_subcommand, path=search_path) if abs_path is None: raise Exception( 'Jupyter command `{}` not found.'.format(jupyter_subcommand) ) if not os.access(abs_path, os.X_OK): raise Exception( 'Jupyter command `{}` is not executable.'.format(jupyter_subcommand) ) return abs_path def _path_with_self(): """Put `jupyter`'s dir at the front of PATH Ensures that /path/to/jupyter subcommand will do /path/to/jupyter-subcommand even if /other/jupyter-subcommand is ahead of it on PATH """ path_list = (os.environ.get('PATH') or os.defpath).split(os.pathsep) # Insert the "scripts" directory for this Python installation # This allows the "jupyter" command to be relocated, while still # finding subcommands that have been installed in the default # location. # We put the scripts directory at the *end* of PATH, so that # if the user explicitly overrides a subcommand, that override # still takes effect. try: bindir = sysconfig.get_path('scripts') except KeyError: # The Python environment does not specify a "scripts" location pass else: path_list.append(bindir) scripts = [sys.argv[0]] if os.path.islink(scripts[0]): # include realpath, if `jupyter` is a symlink scripts.append(os.path.realpath(scripts[0])) for script in scripts: bindir = os.path.dirname(script) if ( os.path.isdir(bindir) and os.access(script, os.X_OK) # only if it's a script ): # ensure executable's dir is on PATH # avoids missing subcommands when jupyter is run via absolute path path_list.insert(0, bindir) return path_list def main(): if len(sys.argv) > 1 and not sys.argv[1].startswith('-'): # Don't parse if a subcommand is given # Avoids argparse gobbling up args passed to subcommand, such as `-h`. subcommand = sys.argv[1] else: parser = jupyter_parser() args, opts = parser.parse_known_args() subcommand = args.subcommand if args.version: print('{:<17}:'.format('jupyter core'), __version__) for package, name in [ ('notebook', 'jupyter-notebook'), ('qtconsole', 'qtconsole'), ('IPython', 'ipython'), ('ipykernel', 'ipykernel'), ('jupyter_client', 'jupyter client'), ('jupyterlab', 'jupyter lab'), ('nbconvert', 'nbconvert'), ('ipywidgets', 'ipywidgets'), ('nbformat', 'nbformat'), ('traitlets', 'traitlets'), ]: version = None try: mod = __import__(package) version = mod.__version__ except ImportError: version = 'not installed' print('{:<17}:'.format(name), version) return if args.json and not args.paths: sys.exit("--json is only used with --paths") if args.config_dir: print(paths.jupyter_config_dir()) return if args.data_dir: print(paths.jupyter_data_dir()) return if args.runtime_dir: print(paths.jupyter_runtime_dir()) return if args.paths: data = {} data['runtime'] = [paths.jupyter_runtime_dir()] data['config'] = paths.jupyter_config_path() data['data'] = paths.jupyter_path() if args.json: print(json.dumps(data)) else: for name in sorted(data): path = data[name] print('%s:' % name) for p in path: print(' ' + p) return if not subcommand: parser.print_usage(file=sys.stderr) sys.exit("subcommand is required") command = _jupyter_abspath(subcommand) try: _execvp(command, sys.argv[1:]) except OSError as e: sys.exit("Error executing Jupyter command %r: %s" % (subcommand, e)) if __name__ == '__main__': main() ================================================ FILE: lib/client/jupyter_core/migrate.py ================================================ from __future__ import unicode_literals """Migrating IPython < 4.0 to Jupyter This *copies* configuration and resources to their new locations in Jupyter Migrations: - .ipython/ - nbextensions -> JUPYTER_DATA_DIR/nbextensions - kernels -> JUPYTER_DATA_DIR/kernels - .ipython/profile_default/ - static/custom -> .jupyter/custom - nbconfig -> .jupyter/nbconfig - security/ - notebook_secret, notebook_cookie_secret, nbsignatures.db -> JUPYTER_DATA_DIR - ipython_{notebook,nbconvert,qtconsole}_config.py -> .jupyter/jupyter_{name}_config.py """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import os import re import shutil from datetime import datetime from traitlets.config import PyFileConfigLoader, JSONFileConfigLoader from traitlets.log import get_logger from .utils import ensure_dir_exists from .paths import jupyter_config_dir, jupyter_data_dir from .application import JupyterApp pjoin = os.path.join migrations = { pjoin('{ipython_dir}', 'nbextensions'): pjoin('{jupyter_data}', 'nbextensions'), pjoin('{ipython_dir}', 'kernels'): pjoin('{jupyter_data}', 'kernels'), pjoin('{profile}', 'nbconfig'): pjoin('{jupyter_config}', 'nbconfig'), } custom_src_t = pjoin('{profile}', 'static', 'custom') custom_dst_t = pjoin('{jupyter_config}', 'custom') for security_file in ('notebook_secret', 'notebook_cookie_secret', 'nbsignatures.db'): src = pjoin('{profile}', 'security', security_file) dst = pjoin('{jupyter_data}', security_file) migrations[src] = dst config_migrations = ['notebook', 'nbconvert', 'qtconsole'] regex = re.compile config_substitutions = { regex(r'\bIPythonQtConsoleApp\b'): 'JupyterQtConsoleApp', regex(r'\bIPythonWidget\b'): 'JupyterWidget', regex(r'\bRichIPythonWidget\b'): 'RichJupyterWidget', regex(r'\bIPython\.html\b'): 'notebook', regex(r'\bIPython\.nbconvert\b'): 'nbconvert', } def get_ipython_dir(): """Return the IPython directory location. Not imported from IPython because the IPython implementation ensures that a writable directory exists, creating a temporary directory if not. We don't want to trigger that when checking if migration should happen. We only need to support the IPython < 4 behavior for migration, so importing for forward-compatibility and edge cases is not important. """ return os.environ.get('IPYTHONDIR', os.path.expanduser('~/.ipython')) def migrate_dir(src, dst): """Migrate a directory from src to dst""" log = get_logger() if not os.listdir(src): log.debug("No files in %s" % src) return False if os.path.exists(dst): if os.listdir(dst): # already exists, non-empty log.debug("%s already exists" % dst) return False else: os.rmdir(dst) log.info("Copying %s -> %s" % (src, dst)) ensure_dir_exists(os.path.dirname(dst)) shutil.copytree(src, dst, symlinks=True) return True def migrate_file(src, dst, substitutions=None): """Migrate a single file from src to dst substitutions is an optional dict of {regex: replacement} for performing replacements on the file. """ log = get_logger() if os.path.exists(dst): # already exists log.debug("%s already exists" % dst) return False log.info("Copying %s -> %s" % (src, dst)) ensure_dir_exists(os.path.dirname(dst)) shutil.copy(src, dst) if substitutions: with open(dst) as f: text = f.read() for pat, replacement in substitutions.items(): text = pat.sub(replacement, text) with open(dst, 'w') as f: f.write(text) return True def migrate_one(src, dst): """Migrate one item dispatches to migrate_dir/_file """ log = get_logger() if os.path.isfile(src): return migrate_file(src, dst) elif os.path.isdir(src): return migrate_dir(src, dst) else: log.debug("Nothing to migrate for %s" % src) return False def migrate_static_custom(src, dst): """Migrate non-empty custom.js,css from src to dst src, dst are 'custom' directories containing custom.{js,css} """ log = get_logger() migrated = False custom_js = pjoin(src, 'custom.js') custom_css = pjoin(src, 'custom.css') # check if custom_js is empty: custom_js_empty = True if os.path.isfile(custom_js): with open(custom_js) as f: js = f.read().strip() for line in js.splitlines(): if not ( line.isspace() or line.strip().startswith(('/*', '*', '//')) ): custom_js_empty = False break # check if custom_css is empty: custom_css_empty = True if os.path.isfile(custom_css): with open(custom_css) as f: css = f.read().strip() custom_css_empty = css.startswith('/*') and css.endswith('*/') if custom_js_empty: log.debug("Ignoring empty %s" % custom_js) if custom_css_empty: log.debug("Ignoring empty %s" % custom_css) if custom_js_empty and custom_css_empty: # nothing to migrate return False ensure_dir_exists(dst) if not custom_js_empty or not custom_css_empty: ensure_dir_exists(dst) if not custom_js_empty: if migrate_file(custom_js, pjoin(dst, 'custom.js')): migrated = True if not custom_css_empty: if migrate_file(custom_css, pjoin(dst, 'custom.css')): migrated = True return migrated def migrate_config(name, env): """Migrate a config file Includes substitutions for updated configurable names. """ log = get_logger() src_base = pjoin('{profile}', 'ipython_{name}_config').format(name=name, **env) dst_base = pjoin('{jupyter_config}', 'jupyter_{name}_config').format(name=name, **env) loaders = { '.py': PyFileConfigLoader, '.json': JSONFileConfigLoader, } migrated = [] for ext in ('.py', '.json'): src = src_base + ext dst = dst_base + ext if os.path.exists(src): cfg = loaders[ext](src).load_config() if cfg: if migrate_file(src, dst, substitutions=config_substitutions): migrated.append(src) else: # don't migrate empty config files log.debug("Not migrating empty config file: %s" % src) return migrated def migrate(): """Migrate IPython configuration to Jupyter""" env = { 'jupyter_data': jupyter_data_dir(), 'jupyter_config': jupyter_config_dir(), 'ipython_dir': get_ipython_dir(), 'profile': os.path.join(get_ipython_dir(), 'profile_default'), } migrated = False for src_t, dst_t in migrations.items(): src = src_t.format(**env) dst = dst_t.format(**env) if os.path.exists(src): if migrate_one(src, dst): migrated = True for name in config_migrations: if migrate_config(name, env): migrated = True custom_src = custom_src_t.format(**env) custom_dst = custom_dst_t.format(**env) if os.path.exists(custom_src): if migrate_static_custom(custom_src, custom_dst): migrated = True # write a marker to avoid re-running migration checks ensure_dir_exists(env['jupyter_config']) with open(os.path.join(env['jupyter_config'], 'migrated'), 'w') as f: f.write(datetime.utcnow().isoformat()) return migrated class JupyterMigrate(JupyterApp): name = 'jupyter-migrate' description = """ Migrate configuration and data from .ipython prior to 4.0 to Jupyter locations. This migrates: - config files in the default profile - kernels in ~/.ipython/kernels - notebook javascript extensions in ~/.ipython/extensions - custom.js/css to .jupyter/custom to their new Jupyter locations. All files are copied, not moved. If the destinations already exist, nothing will be done. """ def start(self): if not migrate(): self.log.info("Found nothing to migrate.") main = JupyterMigrate.launch_instance if __name__ == '__main__': main() ================================================ FILE: lib/client/jupyter_core/paths.py ================================================ # encoding: utf-8 """Path utility functions.""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. # Derived from IPython.utils.path, which is # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. import os import sys import stat import errno import tempfile from ipython_genutils import py3compat from contextlib import contextmanager from ipython_genutils import py3compat pjoin = os.path.join # UF_HIDDEN is a stat flag not defined in the stat module. # It is used by BSD to indicate hidden files. UF_HIDDEN = getattr(stat, 'UF_HIDDEN', 32768) def get_home_dir(): """Get the real path of the home directory""" homedir = os.path.expanduser('~') # Next line will make things work even when /home/ is a symlink to # /usr/home as it is on FreeBSD, for example homedir = os.path.realpath(homedir) homedir = py3compat.str_to_unicode(homedir, encoding=sys.getfilesystemencoding()) return homedir _dtemps = {} def _mkdtemp_once(name): """Make or reuse a temporary directory. If this is called with the same name in the same process, it will return the same directory. """ try: return _dtemps[name] except KeyError: d = _dtemps[name] = tempfile.mkdtemp(prefix=name + '-') return d def jupyter_config_dir(): """Get the Jupyter config directory for this platform and user. Returns JUPYTER_CONFIG_DIR if defined, else ~/.jupyter """ env = os.environ home_dir = get_home_dir() if env.get('JUPYTER_NO_CONFIG'): return _mkdtemp_once('jupyter-clean-cfg') if env.get('JUPYTER_CONFIG_DIR'): return env['JUPYTER_CONFIG_DIR'] return pjoin(home_dir, '.jupyter') def jupyter_data_dir(): """Get the config directory for Jupyter data files. These are non-transient, non-configuration files. Returns JUPYTER_DATA_DIR if defined, else a platform-appropriate path. """ env = os.environ if env.get('JUPYTER_DATA_DIR'): return env['JUPYTER_DATA_DIR'] home = get_home_dir() if sys.platform == 'darwin': return os.path.join(home, 'Library', 'Jupyter') elif os.name == 'nt': appdata = os.environ.get('APPDATA', None) if appdata: return pjoin(appdata, 'jupyter') else: return pjoin(jupyter_config_dir(), 'data') else: # Linux, non-OS X Unix, AIX, etc. xdg = env.get("XDG_DATA_HOME", None) if not xdg: xdg = pjoin(home, '.local', 'share') return pjoin(xdg, 'jupyter') def jupyter_runtime_dir(): """Return the runtime dir for transient jupyter files. Returns JUPYTER_RUNTIME_DIR if defined. The default is now (data_dir)/runtime on all platforms; we no longer use XDG_RUNTIME_DIR after various problems. """ env = os.environ if env.get('JUPYTER_RUNTIME_DIR'): return env['JUPYTER_RUNTIME_DIR'] return pjoin(jupyter_data_dir(), 'runtime') if os.name == 'nt': programdata = os.environ.get('PROGRAMDATA', None) if programdata: SYSTEM_JUPYTER_PATH = [pjoin(programdata, 'jupyter')] else: # PROGRAMDATA is not defined by default on XP. SYSTEM_JUPYTER_PATH = [os.path.join(sys.prefix, 'share', 'jupyter')] else: SYSTEM_JUPYTER_PATH = [ "/usr/local/share/jupyter", "/usr/share/jupyter", ] ENV_JUPYTER_PATH = [os.path.join(sys.prefix, 'share', 'jupyter')] def jupyter_path(*subdirs): """Return a list of directories to search for data files JUPYTER_PATH environment variable has highest priority. If ``*subdirs`` are given, that subdirectory will be added to each element. Examples: >>> jupyter_path() ['~/.local/jupyter', '/usr/local/share/jupyter'] >>> jupyter_path('kernels') ['~/.local/jupyter/kernels', '/usr/local/share/jupyter/kernels'] """ paths = [] # highest priority is env if os.environ.get('JUPYTER_PATH'): paths.extend( p.rstrip(os.sep) for p in os.environ['JUPYTER_PATH'].split(os.pathsep) ) # then user dir paths.append(jupyter_data_dir()) # then sys.prefix for p in ENV_JUPYTER_PATH: if p not in SYSTEM_JUPYTER_PATH: paths.append(p) # finally, system paths.extend(SYSTEM_JUPYTER_PATH) # add subdir, if requested if subdirs: paths = [ pjoin(p, *subdirs) for p in paths ] return paths if os.name == 'nt': programdata = os.environ.get('PROGRAMDATA', None) if programdata: SYSTEM_CONFIG_PATH = [os.path.join(programdata, 'jupyter')] else: # PROGRAMDATA is not defined by default on XP. SYSTEM_CONFIG_PATH = [] else: SYSTEM_CONFIG_PATH = [ "/usr/local/etc/jupyter", "/etc/jupyter", ] ENV_CONFIG_PATH = [os.path.join(sys.prefix, 'etc', 'jupyter')] def jupyter_config_path(): """Return the search path for Jupyter config files as a list.""" paths = [jupyter_config_dir()] if os.environ.get('JUPYTER_NO_CONFIG'): return paths # highest priority is env if os.environ.get('JUPYTER_CONFIG_PATH'): paths.extend( p.rstrip(os.sep) for p in os.environ['JUPYTER_CONFIG_PATH'].split(os.pathsep) ) # then sys.prefix for p in ENV_CONFIG_PATH: if p not in SYSTEM_CONFIG_PATH: paths.append(p) paths.extend(SYSTEM_CONFIG_PATH) return paths def exists(path): """Replacement for `os.path.exists` which works for host mapped volumes on Windows containers """ try: os.lstat(path) except OSError: return False return True def is_file_hidden_win(abs_path, stat_res=None): """Is a file hidden? This only checks the file itself; it should be called in combination with checking the directory containing the file. Use is_hidden() instead to check the file and its parent directories. Parameters ---------- abs_path : unicode The absolute path to check. stat_res : os.stat_result, optional Ignored on Windows, exists for compatibility with POSIX version of the function. """ if os.path.basename(abs_path).startswith('.'): return True win32_FILE_ATTRIBUTE_HIDDEN = 0x02 import ctypes try: attrs = ctypes.windll.kernel32.GetFileAttributesW( py3compat.cast_unicode(abs_path) ) except AttributeError: pass else: if attrs > 0 and attrs & win32_FILE_ATTRIBUTE_HIDDEN: return True return False def is_file_hidden_posix(abs_path, stat_res=None): """Is a file hidden? This only checks the file itself; it should be called in combination with checking the directory containing the file. Use is_hidden() instead to check the file and its parent directories. Parameters ---------- abs_path : unicode The absolute path to check. stat_res : os.stat_result, optional The result of calling stat() on abs_path. If not passed, this function will call stat() internally. """ if os.path.basename(abs_path).startswith('.'): return True if stat_res is None or stat.S_ISLNK(stat_res.st_mode): try: stat_res = os.stat(abs_path) except OSError as e: if e.errno == errno.ENOENT: return False raise # check that dirs can be listed if stat.S_ISDIR(stat_res.st_mode): # use x-access, not actual listing, in case of slow/large listings if not os.access(abs_path, os.X_OK | os.R_OK): return True # check UF_HIDDEN if getattr(stat_res, 'st_flags', 0) & UF_HIDDEN: return True return False if sys.platform == 'win32': is_file_hidden = is_file_hidden_win else: is_file_hidden = is_file_hidden_posix def is_hidden(abs_path, abs_root=''): """Is a file hidden or contained in a hidden directory? This will start with the rightmost path element and work backwards to the given root to see if a path is hidden or in a hidden directory. Hidden is determined by either name starting with '.' or the UF_HIDDEN flag as reported by stat. If abs_path is the same directory as abs_root, it will be visible even if that is a hidden folder. This only checks the visibility of files and directories *within* abs_root. Parameters ---------- abs_path : unicode The absolute path to check for hidden directories. abs_root : unicode The absolute path of the root directory in which hidden directories should be checked for. """ if os.path.normpath(abs_path) == os.path.normpath(abs_root): return False if is_file_hidden(abs_path): return True if not abs_root: abs_root = abs_path.split(os.sep, 1)[0] + os.sep inside_root = abs_path[len(abs_root):] if any(part.startswith('.') for part in inside_root.split(os.sep)): return True # check UF_HIDDEN on any location up to root. # is_file_hidden() already checked the file, so start from its parent dir path = os.path.dirname(abs_path) while path and path.startswith(abs_root) and path != abs_root: if not exists(path): path = os.path.dirname(path) continue try: # may fail on Windows junctions st = os.lstat(path) except OSError: return True if getattr(st, 'st_flags', 0) & UF_HIDDEN: return True path = os.path.dirname(path) return False def win32_restrict_file_to_user(fname): """Secure a windows file to read-only access for the user. Follows guidance from win32 library creator: http://timgolden.me.uk/python/win32_how_do_i/add-security-to-a-file.html This method should be executed against an already generated file which has no secrets written to it yet. Parameters ---------- fname : unicode The path to the file to secure """ import win32api import win32security import ntsecuritycon as con # everyone, _domain, _type = win32security.LookupAccountName("", "Everyone") admins = win32security.CreateWellKnownSid(win32security.WinBuiltinAdministratorsSid) user, _domain, _type = win32security.LookupAccountName("", win32api.GetUserNameEx(win32api.NameSamCompatible)) sd = win32security.GetFileSecurity(fname, win32security.DACL_SECURITY_INFORMATION) dacl = win32security.ACL() # dacl.AddAccessAllowedAce(win32security.ACL_REVISION, con.FILE_ALL_ACCESS, everyone) dacl.AddAccessAllowedAce(win32security.ACL_REVISION, con.FILE_GENERIC_READ | con.FILE_GENERIC_WRITE, user) dacl.AddAccessAllowedAce(win32security.ACL_REVISION, con.FILE_ALL_ACCESS, admins) sd.SetSecurityDescriptorDacl(1, dacl, 0) win32security.SetFileSecurity(fname, win32security.DACL_SECURITY_INFORMATION, sd) def get_file_mode(fname): """Retrieves the file mode corresponding to fname in a filesystem-tolerant manner. Parameters ---------- fname : unicode The path to the file to get mode from """ # Some filesystems (e.g., CIFS) auto-enable the execute bit on files. As a result, we # should tolerate the execute bit on the file's owner when validating permissions - thus # the missing one's bit on the third octet. return stat.S_IMODE(os.stat(fname).st_mode) & 0o7677 # Use 4 octets since S_IMODE does the same @contextmanager def secure_write(fname, binary=False): """Opens a file in the most restricted pattern available for writing content. This limits the file mode to `0o0600` and yields the resulting opened filed handle. Parameters ---------- fname : unicode The path to the file to write binary: boolean Indicates that the file is binary """ mode = 'wb' if binary else 'w' open_flag = os.O_CREAT | os.O_WRONLY | os.O_TRUNC try: os.remove(fname) except (IOError, OSError): # Skip any issues with the file not existing pass if os.name == 'nt': # Python on windows does not respect the group and public bits for chmod, so we need # to take additional steps to secure the contents. # Touch file pre-emptively to avoid editing permissions in open files in Windows fd = os.open(fname, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o0600) os.close(fd) open_flag = os.O_WRONLY | os.O_TRUNC win32_restrict_file_to_user(fname) with os.fdopen(os.open(fname, open_flag, 0o0600), mode) as f: if os.name != 'nt': # Enforce that the file got the requested permissions before writing file_mode = get_file_mode(fname) if 0o0600 != file_mode: raise RuntimeError("Permissions assignment failed for secure file: '{file}'." "Got '{permissions}' instead of '0o0600'" .format(file=fname, permissions=oct(file_mode))) yield f ================================================ FILE: lib/client/jupyter_core/troubleshoot.py ================================================ #!/usr/bin/env python """ display environment information that isfrequently used to troubleshoot installations of Jupyter or IPython """ # import argparse import os import platform import subprocess import sys # def get_args(): # """ # TODO: output in JSON or xml? maybe? # """ # pass def subs(cmd): """ get data from commands that we need to run outside of python """ try: stdout = subprocess.check_output(cmd) return stdout.decode('utf-8', 'replace').strip() except (OSError, subprocess.CalledProcessError): return None def get_data(): """ returns a dict of various user environment data """ env = {} env['path'] = os.environ.get('PATH') env['sys_path'] = sys.path env['sys_exe'] = sys.executable env['sys_version'] = sys.version env['platform'] = platform.platform() # FIXME: which on Windows? if sys.platform == 'win32': env['where'] = subs(['where', 'jupyter']) env['which'] = None else: env['which'] = subs(['which', '-a', 'jupyter']) env['where'] = None env['pip'] = subs([sys.executable, '-m', 'pip', 'list']) env['conda'] = subs(['conda', 'list']) return env def main(): """ print out useful info """ #pylint: disable=superfluous-parens # args = get_args() environment_data = get_data() print('$PATH:') for directory in environment_data['path'].split(os.pathsep): print('\t' + directory) print('\n' + 'sys.path:') for directory in environment_data['sys_path']: print('\t' + directory) print('\n' + 'sys.executable:') print('\t' + environment_data['sys_exe']) print('\n' + 'sys.version:') if '\n' in environment_data['sys_version']: for data in environment_data['sys_version'].split('\n'): print('\t' + data) else: print('\t' + environment_data['sys_version']) print('\n' + 'platform.platform():') print('\t' + environment_data['platform']) if environment_data['which']: print('\n' + 'which -a jupyter:') for line in environment_data['which'].split('\n'): print('\t' + line) if environment_data['where']: print('\n' + 'where jupyter:') for line in environment_data['where'].split('\n'): print('\t' + line) if environment_data['pip']: print('\n' + 'pip list:') for package in environment_data['pip'].split('\n'): print('\t' + package) if environment_data['conda']: print('\n' + 'conda list:') for package in environment_data['conda'].split('\n'): print('\t' + package) if __name__ == '__main__': main() ================================================ FILE: lib/client/jupyter_core/utils/__init__.py ================================================ import errno import os def ensure_dir_exists(path, mode=0o777): """ensure that a directory exists If it doesn't exist, try to create it, protecting against a race condition if another process is doing the same. The default permissions are determined by the current umask. """ try: os.makedirs(path, mode=mode) except OSError as e: if e.errno != errno.EEXIST: raise if not os.path.isdir(path): raise IOError("%r exists but is not a directory" % path) ================================================ FILE: lib/client/jupyter_core/utils/shutil_which.py ================================================ # Verbatim copy of shutil.which from Python 3.4.3 # License: PSF # Only used on Python < 3 import os, sys def which(cmd, mode=os.F_OK | os.X_OK, path=None): """Given a command, mode, and a PATH string, return the path which conforms to the given mode on the PATH, or None if there is no such file. `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result of os.environ.get("PATH"), or can be overridden with a custom search path. """ # Check that a given file can be accessed with the correct mode. # Additionally check that `file` is not a directory, as on Windows # directories pass the os.access check. def _access_check(fn, mode): return (os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn)) # If we're given a path with a directory part, look it up directly rather # than referring to PATH directories. This includes checking relative to the # current directory, e.g. ./script if os.path.dirname(cmd): if _access_check(cmd, mode): return cmd return None if path is None: path = os.environ.get("PATH", os.defpath) if not path: return None path = path.split(os.pathsep) if sys.platform == "win32": # The current directory takes precedence on Windows. if not os.curdir in path: path.insert(0, os.curdir) # PATHEXT is necessary to check on Windows. pathext = os.environ.get("PATHEXT", "").split(os.pathsep) # See if the given file matches any of the expected path extensions. # This will allow us to short circuit when given "python.exe". # If it does match, only test that one, otherwise we have to try # others. if any(cmd.lower().endswith(ext.lower()) for ext in pathext): files = [cmd] else: files = [cmd + ext for ext in pathext] else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. files = [cmd] seen = set() for dir in path: normdir = os.path.normcase(dir) if not normdir in seen: seen.add(normdir) for thefile in files: name = os.path.join(dir, thefile) if _access_check(name, mode): return name return None ================================================ FILE: lib/client/jupyter_core/version.py ================================================ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. version_info = (4, 6, 1) __version__ = '.'.join(map(str, version_info)) ================================================ FILE: lib/client/traitlets/__init__.py ================================================ from .traitlets import * from .utils.importstring import import_item from ._version import version_info, __version__ ================================================ FILE: lib/client/traitlets/_version.py ================================================ version_info = (4, 3, 3) __version__ = '.'.join(map(str, version_info)) ================================================ FILE: lib/client/traitlets/config/__init__.py ================================================ # encoding: utf-8 # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. from .application import * from .configurable import * from .loader import Config ================================================ FILE: lib/client/traitlets/config/application.py ================================================ # encoding: utf-8 """A base class for a configurable application.""" # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import print_function from copy import deepcopy import json import logging import os import re import sys from collections import defaultdict, OrderedDict from decorator import decorator from traitlets.config.configurable import Configurable, SingletonConfigurable from traitlets.config.loader import ( KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound, JSONFileConfigLoader ) from traitlets.traitlets import ( Bool, Unicode, List, Enum, Dict, Instance, TraitError, observe, observe_compat, default, ) from ipython_genutils.importstring import import_item from ipython_genutils.text import indent, wrap_paragraphs, dedent from ipython_genutils import py3compat import six #----------------------------------------------------------------------------- # Descriptions for the various sections #----------------------------------------------------------------------------- # merge flags&aliases into options option_description = """ Arguments that take values are actually convenience aliases to full Configurables, whose aliases are listed on the help line. For more information on full configurables, see '--help-all'. """.strip() # trim newlines of front and back keyvalue_description = """ Parameters are set from command-line arguments of the form: `--Class.trait=value`. This line is evaluated in Python, so simple expressions are allowed, e.g.:: `--C.a='range(3)'` For setting C.a=[0,1,2]. """.strip() # trim newlines of front and back # sys.argv can be missing, for example when python is embedded. See the docs # for details: http://docs.python.org/2/c-api/intro.html#embedding-python if not hasattr(sys, "argv"): sys.argv = [""] subcommand_description = """ Subcommands are launched as `{app} cmd [args]`. For information on using subcommand 'cmd', do: `{app} cmd -h`. """ # get running program name #----------------------------------------------------------------------------- # Application class #----------------------------------------------------------------------------- _envvar = os.environ.get('TRAITLETS_APPLICATION_RAISE_CONFIG_FILE_ERROR','') if _envvar.lower() in {'1','true'}: TRAITLETS_APPLICATION_RAISE_CONFIG_FILE_ERROR = True elif _envvar.lower() in {'0','false',''} : TRAITLETS_APPLICATION_RAISE_CONFIG_FILE_ERROR = False else: raise ValueError("Unsupported value for environment variable: 'TRAITLETS_APPLICATION_RAISE_CONFIG_FILE_ERROR' is set to '%s' which is none of {'0', '1', 'false', 'true', ''}."% _envvar ) @decorator def catch_config_error(method, app, *args, **kwargs): """Method decorator for catching invalid config (Trait/ArgumentErrors) during init. On a TraitError (generally caused by bad config), this will print the trait's message, and exit the app. For use on init methods, to prevent invoking excepthook on invalid input. """ try: return method(app, *args, **kwargs) except (TraitError, ArgumentError) as e: app.print_help() app.log.fatal("Bad config encountered during initialization:") app.log.fatal(str(e)) app.log.debug("Config at the time: %s", app.config) app.exit(1) class ApplicationError(Exception): pass class LevelFormatter(logging.Formatter): """Formatter with additional `highlevel` record This field is empty if log level is less than highlevel_limit, otherwise it is formatted with self.highlevel_format. Useful for adding 'WARNING' to warning messages, without adding 'INFO' to info, etc. """ highlevel_limit = logging.WARN highlevel_format = " %(levelname)s |" def format(self, record): if record.levelno >= self.highlevel_limit: record.highlevel = self.highlevel_format % record.__dict__ else: record.highlevel = "" return super(LevelFormatter, self).format(record) class Application(SingletonConfigurable): """A singleton application with full configuration support.""" # The name of the application, will usually match the name of the command # line application name = Unicode(u'application') # The description of the application that is printed at the beginning # of the help. description = Unicode(u'This is an application.') # default section descriptions option_description = Unicode(option_description) keyvalue_description = Unicode(keyvalue_description) subcommand_description = Unicode(subcommand_description) python_config_loader_class = PyFileConfigLoader json_config_loader_class = JSONFileConfigLoader # The usage and example string that goes at the end of the help string. examples = Unicode() # A sequence of Configurable subclasses whose config=True attributes will # be exposed at the command line. classes = [] def _classes_inc_parents(self): """Iterate through configurable classes, including configurable parents Children should always be after parents, and each class should only be yielded once. """ seen = set() for c in self.classes: # We want to sort parents before children, so we reverse the MRO for parent in reversed(c.mro()): if issubclass(parent, Configurable) and (parent not in seen): seen.add(parent) yield parent # The version string of this application. version = Unicode(u'0.0') # the argv used to initialize the application argv = List() # Whether failing to load config files should prevent startup raise_config_file_errors = Bool(TRAITLETS_APPLICATION_RAISE_CONFIG_FILE_ERROR) # The log level for the application log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'), default_value=logging.WARN, help="Set the log level by value or name.").tag(config=True) @observe('log_level') @observe_compat def _log_level_changed(self, change): """Adjust the log level when log_level is set.""" new = change.new if isinstance(new, six.string_types): new = getattr(logging, new) self.log_level = new self.log.setLevel(new) _log_formatter_cls = LevelFormatter log_datefmt = Unicode("%Y-%m-%d %H:%M:%S", help="The date format used by logging formatters for %(asctime)s" ).tag(config=True) log_format = Unicode("[%(name)s]%(highlevel)s %(message)s", help="The Logging format template", ).tag(config=True) @observe('log_datefmt', 'log_format') @observe_compat def _log_format_changed(self, change): """Change the log formatter when log_format is set.""" _log_handler = self.log.handlers[0] _log_formatter = self._log_formatter_cls(fmt=self.log_format, datefmt=self.log_datefmt) _log_handler.setFormatter(_log_formatter) @default('log') def _log_default(self): """Start logging for this application. The default is to log to stderr using a StreamHandler, if no default handler already exists. The log level starts at logging.WARN, but this can be adjusted by setting the ``log_level`` attribute. """ log = logging.getLogger(self.__class__.__name__) log.setLevel(self.log_level) log.propagate = False _log = log # copied from Logger.hasHandlers() (new in Python 3.2) while _log: if _log.handlers: return log if not _log.propagate: break else: _log = _log.parent if sys.executable and sys.executable.endswith('pythonw.exe'): # this should really go to a file, but file-logging is only # hooked up in parallel applications _log_handler = logging.StreamHandler(open(os.devnull, 'w')) else: _log_handler = logging.StreamHandler() _log_formatter = self._log_formatter_cls(fmt=self.log_format, datefmt=self.log_datefmt) _log_handler.setFormatter(_log_formatter) log.addHandler(_log_handler) return log # the alias map for configurables aliases = Dict({'log-level' : 'Application.log_level'}) # flags for loading Configurables or store_const style flags # flags are loaded from this dict by '--key' flags # this must be a dict of two-tuples, the first element being the Config/dict # and the second being the help string for the flag flags = Dict() @observe('flags') @observe_compat def _flags_changed(self, change): """ensure flags dict is valid""" new = change.new for key, value in new.items(): assert len(value) == 2, "Bad flag: %r:%s" % (key, value) assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s" % (key, value) assert isinstance(value[1], six.string_types), "Bad flag: %r:%s" % (key, value) # subcommands for launching other applications # if this is not empty, this will be a parent Application # this must be a dict of two-tuples, # the first element being the application class/import string # and the second being the help string for the subcommand subcommands = Dict() # parse_command_line will initialize a subapp, if requested subapp = Instance('traitlets.config.application.Application', allow_none=True) # extra command-line arguments that don't set config values extra_args = List(Unicode()) cli_config = Instance(Config, (), {}, help="""The subset of our configuration that came from the command-line We re-load this configuration after loading config files, to ensure that it maintains highest priority. """ ) _loaded_config_files = List() def __init__(self, **kwargs): SingletonConfigurable.__init__(self, **kwargs) # Ensure my class is in self.classes, so my attributes appear in command line # options and config files. cls = self.__class__ if cls not in self.classes: if self.classes is cls.classes: # class attr, assign instead of insert cls.classes = [cls] + self.classes else: self.classes.insert(0, self.__class__) @observe('config') @observe_compat def _config_changed(self, change): super(Application, self)._config_changed(change) self.log.debug('Config changed:') self.log.debug(repr(change.new)) @catch_config_error def initialize(self, argv=None): """Do the basic steps to configure me. Override in subclasses. """ self.parse_command_line(argv) def start(self): """Start the app mainloop. Override in subclasses. """ if self.subapp is not None: return self.subapp.start() def print_alias_help(self): """Print the alias part of the help.""" if not self.aliases: return lines = [] classdict = {} for cls in self.classes: # include all parents (up to, but excluding Configurable) in available names for c in cls.mro()[:-3]: classdict[c.__name__] = c for alias, longname in self.aliases.items(): classname, traitname = longname.split('.',1) cls = classdict[classname] trait = cls.class_traits(config=True)[traitname] help = cls.class_get_trait_help(trait).splitlines() # reformat first line help[0] = help[0].replace(longname, alias) + ' (%s)'%longname if len(alias) == 1: help[0] = help[0].replace('--%s='%alias, '-%s '%alias) lines.extend(help) # lines.append('') print(os.linesep.join(lines)) def print_flag_help(self): """Print the flag part of the help.""" if not self.flags: return lines = [] for m, (cfg,help) in self.flags.items(): prefix = '--' if len(m) > 1 else '-' lines.append(prefix+m) lines.append(indent(dedent(help.strip()))) # lines.append('') print(os.linesep.join(lines)) def print_options(self): if not self.flags and not self.aliases: return lines = ['Options'] lines.append('-'*len(lines[0])) lines.append('') for p in wrap_paragraphs(self.option_description): lines.append(p) lines.append('') print(os.linesep.join(lines)) self.print_flag_help() self.print_alias_help() print() def print_subcommands(self): """Print the subcommand part of the help.""" if not self.subcommands: return lines = ["Subcommands"] lines.append('-'*len(lines[0])) lines.append('') for p in wrap_paragraphs(self.subcommand_description.format( app=self.name)): lines.append(p) lines.append('') for subc, (cls, help) in self.subcommands.items(): lines.append(subc) if help: lines.append(indent(dedent(help.strip()))) lines.append('') print(os.linesep.join(lines)) def print_help(self, classes=False): """Print the help for each Configurable class in self.classes. If classes=False (the default), only flags and aliases are printed. """ self.print_description() self.print_subcommands() self.print_options() if classes: help_classes = self.classes if help_classes: print("Class parameters") print("----------------") print() for p in wrap_paragraphs(self.keyvalue_description): print(p) print() for cls in help_classes: cls.class_print_help() print() else: print("To see all available configurables, use `--help-all`") print() self.print_examples() def document_config_options(self): """Generate rST format documentation for the config options this application Returns a multiline string. """ return '\n'.join(c.class_config_rst_doc() for c in self._classes_inc_parents()) def print_description(self): """Print the application description.""" for p in wrap_paragraphs(self.description): print(p) print() def print_examples(self): """Print usage and examples. This usage string goes at the end of the command line help string and should contain examples of the application's usage. """ if self.examples: print("Examples") print("--------") print() print(indent(dedent(self.examples.strip()))) print() def print_version(self): """Print the version string.""" print(self.version) @catch_config_error def initialize_subcommand(self, subc, argv=None): """Initialize a subcommand with argv.""" subapp,help = self.subcommands.get(subc) if isinstance(subapp, six.string_types): subapp = import_item(subapp) # clear existing instances self.__class__.clear_instance() # instantiate self.subapp = subapp.instance(parent=self) # and initialize subapp self.subapp.initialize(argv) def flatten_flags(self): """flatten flags and aliases, so cl-args override as expected. This prevents issues such as an alias pointing to InteractiveShell, but a config file setting the same trait in TerminalInteraciveShell getting inappropriate priority over the command-line arg. Only aliases with exactly one descendent in the class list will be promoted. """ # build a tree of classes in our list that inherit from a particular # it will be a dict by parent classname of classes in our list # that are descendents mro_tree = defaultdict(list) for cls in self.classes: clsname = cls.__name__ for parent in cls.mro()[1:-3]: # exclude cls itself and Configurable,HasTraits,object mro_tree[parent.__name__].append(clsname) # flatten aliases, which have the form: # { 'alias' : 'Class.trait' } aliases = {} for alias, cls_trait in self.aliases.items(): cls,trait = cls_trait.split('.',1) children = mro_tree[cls] if len(children) == 1: # exactly one descendent, promote alias cls = children[0] aliases[alias] = '.'.join([cls,trait]) # flatten flags, which are of the form: # { 'key' : ({'Cls' : {'trait' : value}}, 'help')} flags = {} for key, (flagdict, help) in self.flags.items(): newflag = {} for cls, subdict in flagdict.items(): children = mro_tree[cls] # exactly one descendent, promote flag section if len(children) == 1: cls = children[0] newflag[cls] = subdict flags[key] = (newflag, help) return flags, aliases @catch_config_error def parse_command_line(self, argv=None): """Parse the command line arguments.""" argv = sys.argv[1:] if argv is None else argv self.argv = [ py3compat.cast_unicode(arg) for arg in argv ] if argv and argv[0] == 'help': # turn `ipython help notebook` into `ipython notebook -h` argv = argv[1:] + ['-h'] if self.subcommands and len(argv) > 0: # we have subcommands, and one may have been specified subc, subargv = argv[0], argv[1:] if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands: # it's a subcommand, and *not* a flag or class parameter return self.initialize_subcommand(subc, subargv) # Arguments after a '--' argument are for the script IPython may be # about to run, not IPython iteslf. For arguments parsed here (help and # version), we want to only search the arguments up to the first # occurrence of '--', which we're calling interpreted_argv. try: interpreted_argv = argv[:argv.index('--')] except ValueError: interpreted_argv = argv if any(x in interpreted_argv for x in ('-h', '--help-all', '--help')): self.print_help('--help-all' in interpreted_argv) self.exit(0) if '--version' in interpreted_argv or '-V' in interpreted_argv: self.print_version() self.exit(0) # flatten flags&aliases, so cl-args get appropriate priority: flags,aliases = self.flatten_flags() loader = KVArgParseConfigLoader(argv=argv, aliases=aliases, flags=flags, log=self.log) self.cli_config = deepcopy(loader.load_config()) self.update_config(self.cli_config) # store unparsed args in extra_args self.extra_args = loader.extra_args @classmethod def _load_config_files(cls, basefilename, path=None, log=None, raise_config_file_errors=False): """Load config files (py,json) by filename and path. yield each config object in turn. """ if not isinstance(path, list): path = [path] for path in path[::-1]: # path list is in descending priority order, so load files backwards: pyloader = cls.python_config_loader_class(basefilename+'.py', path=path, log=log) if log: log.debug("Looking for %s in %s", basefilename, path or os.getcwd()) jsonloader = cls.json_config_loader_class(basefilename+'.json', path=path, log=log) loaded = [] filenames = [] for loader in [pyloader, jsonloader]: config = None try: config = loader.load_config() except ConfigFileNotFound: pass except Exception: # try to get the full filename, but it will be empty in the # unlikely event that the error raised before filefind finished filename = loader.full_filename or basefilename # problem while running the file if raise_config_file_errors: raise if log: log.error("Exception while loading config file %s", filename, exc_info=True) else: if log: log.debug("Loaded config file: %s", loader.full_filename) if config: for filename, earlier_config in zip(filenames, loaded): collisions = earlier_config.collisions(config) if collisions and log: log.warning("Collisions detected in {0} and {1} config files." " {1} has higher priority: {2}".format( filename, loader.full_filename, json.dumps(collisions, indent=2), )) yield (config, loader.full_filename) loaded.append(config) filenames.append(loader.full_filename) @property def loaded_config_files(self): """Currently loaded configuration files""" return self._loaded_config_files[:] @catch_config_error def load_config_file(self, filename, path=None): """Load config files by filename and path.""" filename, ext = os.path.splitext(filename) new_config = Config() for (config, filename) in self._load_config_files(filename, path=path, log=self.log, raise_config_file_errors=self.raise_config_file_errors, ): new_config.merge(config) if filename not in self._loaded_config_files: # only add to list of loaded files if not previously loaded self._loaded_config_files.append(filename) # add self.cli_config to preserve CLI config priority new_config.merge(self.cli_config) self.update_config(new_config) def _classes_in_config_sample(self): """ Yields only classes with own traits, and their subclasses. Thus, produced sample config-file will contain all classes on which a trait-value may be overridden: - either on the class owning the trait, - or on its subclasses, even if those subclasses do not define any traits themselves. """ cls_to_config = OrderedDict( (cls, bool(cls.class_own_traits(config=True))) for cls in self._classes_inc_parents()) def is_any_parent_included(cls): return any(b in cls_to_config and cls_to_config[b] for b in cls.__bases__) ## Mark "empty" classes for inclusion if their parents own-traits, # and loop until no more classes gets marked. # while True: to_incl_orig = cls_to_config.copy() cls_to_config = OrderedDict( (cls, inc_yes or is_any_parent_included(cls)) for cls, inc_yes in cls_to_config.items()) if cls_to_config == to_incl_orig: break for cl, inc_yes in cls_to_config.items(): if inc_yes: yield cl def generate_config_file(self): """generate default config file from Configurables""" lines = ["# Configuration file for %s." % self.name] lines.append('') for cls in self._classes_in_config_sample(): lines.append(cls.class_config_section()) return '\n'.join(lines) def exit(self, exit_status=0): self.log.debug("Exiting application: %s" % self.name) sys.exit(exit_status) @classmethod def launch_instance(cls, argv=None, **kwargs): """Launch a global instance of this Application If a global instance already exists, this reinitializes and starts it """ app = cls.instance(**kwargs) app.initialize(argv) app.start() #----------------------------------------------------------------------------- # utility functions, for convenience #----------------------------------------------------------------------------- def boolean_flag(name, configurable, set_help='', unset_help=''): """Helper for building basic --trait, --no-trait flags. Parameters ---------- name : str The name of the flag. configurable : str The 'Class.trait' string of the trait to be set/unset with the flag set_help : unicode help string for --name flag unset_help : unicode help string for --no-name flag Returns ------- cfg : dict A dict with two keys: 'name', and 'no-name', for setting and unsetting the trait, respectively. """ # default helpstrings set_help = set_help or "set %s=True"%configurable unset_help = unset_help or "set %s=False"%configurable cls,trait = configurable.split('.') setter = {cls : {trait : True}} unsetter = {cls : {trait : False}} return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)} def get_config(): """Get the config object for the global Application instance, if there is one otherwise return an empty config object """ if Application.initialized(): return Application.instance().config else: return Config() ================================================ FILE: lib/client/traitlets/config/configurable.py ================================================ # encoding: utf-8 """A base class for objects that are configurable.""" # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import print_function, absolute_import from copy import deepcopy import warnings from .loader import Config, LazyConfigValue, _is_section_key from traitlets.traitlets import HasTraits, Instance, observe, observe_compat, default from ipython_genutils.text import indent, dedent, wrap_paragraphs #----------------------------------------------------------------------------- # Helper classes for Configurables #----------------------------------------------------------------------------- class ConfigurableError(Exception): pass class MultipleInstanceError(ConfigurableError): pass #----------------------------------------------------------------------------- # Configurable implementation #----------------------------------------------------------------------------- class Configurable(HasTraits): config = Instance(Config, (), {}) parent = Instance('traitlets.config.configurable.Configurable', allow_none=True) def __init__(self, **kwargs): """Create a configurable given a config config. Parameters ---------- config : Config If this is empty, default values are used. If config is a :class:`Config` instance, it will be used to configure the instance. parent : Configurable instance, optional The parent Configurable instance of this object. Notes ----- Subclasses of Configurable must call the :meth:`__init__` method of :class:`Configurable` *before* doing anything else and using :func:`super`:: class MyConfigurable(Configurable): def __init__(self, config=None): super(MyConfigurable, self).__init__(config=config) # Then any other code you need to finish initialization. This ensures that instances will be configured properly. """ parent = kwargs.pop('parent', None) if parent is not None: # config is implied from parent if kwargs.get('config', None) is None: kwargs['config'] = parent.config self.parent = parent config = kwargs.pop('config', None) # load kwarg traits, other than config super(Configurable, self).__init__(**kwargs) # load config if config is not None: # We used to deepcopy, but for now we are trying to just save # by reference. This *could* have side effects as all components # will share config. In fact, I did find such a side effect in # _config_changed below. If a config attribute value was a mutable type # all instances of a component were getting the same copy, effectively # making that a class attribute. # self.config = deepcopy(config) self.config = config else: # allow _config_default to return something self._load_config(self.config) # Ensure explicit kwargs are applied after loading config. # This is usually redundant, but ensures config doesn't override # explicitly assigned values. for key, value in kwargs.items(): setattr(self, key, value) #------------------------------------------------------------------------- # Static trait notifiations #------------------------------------------------------------------------- @classmethod def section_names(cls): """return section names as a list""" return [c.__name__ for c in reversed(cls.__mro__) if issubclass(c, Configurable) and issubclass(cls, c) ] def _find_my_config(self, cfg): """extract my config from a global Config object will construct a Config object of only the config values that apply to me based on my mro(), as well as those of my parent(s) if they exist. If I am Bar and my parent is Foo, and their parent is Tim, this will return merge following config sections, in this order:: [Bar, Foo.bar, Tim.Foo.Bar] With the last item being the highest priority. """ cfgs = [cfg] if self.parent: cfgs.append(self.parent._find_my_config(cfg)) my_config = Config() for c in cfgs: for sname in self.section_names(): # Don't do a blind getattr as that would cause the config to # dynamically create the section with name Class.__name__. if c._has_section(sname): my_config.merge(c[sname]) return my_config def _load_config(self, cfg, section_names=None, traits=None): """load traits from a Config object""" if traits is None: traits = self.traits(config=True) if section_names is None: section_names = self.section_names() my_config = self._find_my_config(cfg) # hold trait notifications until after all config has been loaded with self.hold_trait_notifications(): for name, config_value in my_config.items(): if name in traits: if isinstance(config_value, LazyConfigValue): # ConfigValue is a wrapper for using append / update on containers # without having to copy the initial value initial = getattr(self, name) config_value = config_value.get_value(initial) # We have to do a deepcopy here if we don't deepcopy the entire # config object. If we don't, a mutable config_value will be # shared by all instances, effectively making it a class attribute. setattr(self, name, deepcopy(config_value)) elif not _is_section_key(name) and not isinstance(config_value, Config): from difflib import get_close_matches if isinstance(self, LoggingConfigurable): warn = self.log.warning else: warn = lambda msg: warnings.warn(msg, stacklevel=9) matches = get_close_matches(name, traits) msg = u"Config option `{option}` not recognized by `{klass}`.".format( option=name, klass=self.__class__.__name__) if len(matches) == 1: msg += u" Did you mean `{matches}`?".format(matches=matches[0]) elif len(matches) >= 1: msg +=" Did you mean one of: `{matches}`?".format(matches=', '.join(sorted(matches))) warn(msg) @observe('config') @observe_compat def _config_changed(self, change): """Update all the class traits having ``config=True`` in metadata. For any class trait with a ``config`` metadata attribute that is ``True``, we update the trait with the value of the corresponding config entry. """ # Get all traits with a config metadata entry that is True traits = self.traits(config=True) # We auto-load config section for this class as well as any parent # classes that are Configurable subclasses. This starts with Configurable # and works down the mro loading the config for each section. section_names = self.section_names() self._load_config(change.new, traits=traits, section_names=section_names) def update_config(self, config): """Update config and load the new values""" # traitlets prior to 4.2 created a copy of self.config in order to trigger change events. # Some projects (IPython < 5) relied upon one side effect of this, # that self.config prior to update_config was not modified in-place. # For backward-compatibility, we must ensure that self.config # is a new object and not modified in-place, # but config consumers should not rely on this behavior. self.config = deepcopy(self.config) # load config self._load_config(config) # merge it into self.config self.config.merge(config) # TODO: trigger change event if/when dict-update change events take place # DO NOT trigger full trait-change @classmethod def class_get_help(cls, inst=None): """Get the help string for this class in ReST format. If `inst` is given, it's current trait values will be used in place of class defaults. """ assert inst is None or isinstance(inst, cls) final_help = [] final_help.append(u'%s options' % cls.__name__) final_help.append(len(final_help[0])*u'-') for k, v in sorted(cls.class_traits(config=True).items()): help = cls.class_get_trait_help(v, inst) final_help.append(help) return '\n'.join(final_help) @classmethod def class_get_trait_help(cls, trait, inst=None): """Get the help string for a single trait. If `inst` is given, it's current trait values will be used in place of the class default. """ assert inst is None or isinstance(inst, cls) lines = [] header = "--%s.%s=<%s>" % (cls.__name__, trait.name, trait.__class__.__name__) lines.append(header) if inst is not None: lines.append(indent('Current: %r' % getattr(inst, trait.name), 4)) else: try: dvr = trait.default_value_repr() except Exception: dvr = None # ignore defaults we can't construct if dvr is not None: if len(dvr) > 64: dvr = dvr[:61]+'...' lines.append(indent('Default: %s' % dvr, 4)) if 'Enum' in trait.__class__.__name__: # include Enum choices lines.append(indent('Choices: %r' % (trait.values,))) help = trait.help if help != '': help = '\n'.join(wrap_paragraphs(help, 76)) lines.append(indent(help, 4)) return '\n'.join(lines) @classmethod def class_print_help(cls, inst=None): """Get the help string for a single trait and print it.""" print(cls.class_get_help(inst)) @classmethod def class_config_section(cls): """Get the config class config section""" def c(s): """return a commented, wrapped block.""" s = '\n\n'.join(wrap_paragraphs(s, 78)) return '## ' + s.replace('\n', '\n# ') # section header breaker = '#' + '-'*78 parent_classes = ','.join(p.__name__ for p in cls.__bases__) s = "# %s(%s) configuration" % (cls.__name__, parent_classes) lines = [breaker, s, breaker, ''] # get the description trait desc = cls.class_traits().get('description') if desc: desc = desc.default_value if not desc: # no description from trait, use __doc__ desc = getattr(cls, '__doc__', '') if desc: lines.append(c(desc)) lines.append('') for name, trait in sorted(cls.class_own_traits(config=True).items()): lines.append(c(trait.help)) lines.append('#c.%s.%s = %s' % (cls.__name__, name, trait.default_value_repr())) lines.append('') return '\n'.join(lines) @classmethod def class_config_rst_doc(cls): """Generate rST documentation for this class' config options. Excludes traits defined on parent classes. """ lines = [] classname = cls.__name__ for k, trait in sorted(cls.class_own_traits(config=True).items()): ttype = trait.__class__.__name__ termline = classname + '.' + trait.name # Choices or type if 'Enum' in ttype: # include Enum choices termline += ' : ' + '|'.join(repr(x) for x in trait.values) else: termline += ' : ' + ttype lines.append(termline) # Default value try: dvr = trait.default_value_repr() except Exception: dvr = None # ignore defaults we can't construct if dvr is not None: if len(dvr) > 64: dvr = dvr[:61]+'...' # Double up backslashes, so they get to the rendered docs dvr = dvr.replace('\\n', '\\\\n') lines.append(' Default: ``%s``' % dvr) lines.append('') help = trait.help or 'No description' lines.append(indent(dedent(help), 4)) # Blank line lines.append('') return '\n'.join(lines) class LoggingConfigurable(Configurable): """A parent class for Configurables that log. Subclasses have a log trait, and the default behavior is to get the logger from the currently running Application. """ log = Instance('logging.Logger') @default('log') def _log_default(self): from traitlets import log return log.get_logger() class SingletonConfigurable(LoggingConfigurable): """A configurable that only allows one instance. This class is for classes that should only have one instance of itself or *any* subclass. To create and retrieve such a class use the :meth:`SingletonConfigurable.instance` method. """ _instance = None @classmethod def _walk_mro(cls): """Walk the cls.mro() for parent classes that are also singletons For use in instance() """ for subclass in cls.mro(): if issubclass(cls, subclass) and \ issubclass(subclass, SingletonConfigurable) and \ subclass != SingletonConfigurable: yield subclass @classmethod def clear_instance(cls): """unset _instance for this class and singleton parents. """ if not cls.initialized(): return for subclass in cls._walk_mro(): if isinstance(subclass._instance, cls): # only clear instances that are instances # of the calling class subclass._instance = None @classmethod def instance(cls, *args, **kwargs): """Returns a global instance of this class. This method create a new instance if none have previously been created and returns a previously created instance is one already exists. The arguments and keyword arguments passed to this method are passed on to the :meth:`__init__` method of the class upon instantiation. Examples -------- Create a singleton class using instance, and retrieve it:: >>> from traitlets.config.configurable import SingletonConfigurable >>> class Foo(SingletonConfigurable): pass >>> foo = Foo.instance() >>> foo == Foo.instance() True Create a subclass that is retrived using the base class instance:: >>> class Bar(SingletonConfigurable): pass >>> class Bam(Bar): pass >>> bam = Bam.instance() >>> bam == Bar.instance() True """ # Create and save the instance if cls._instance is None: inst = cls(*args, **kwargs) # Now make sure that the instance will also be returned by # parent classes' _instance attribute. for subclass in cls._walk_mro(): subclass._instance = inst if isinstance(cls._instance, cls): return cls._instance else: raise MultipleInstanceError( 'Multiple incompatible subclass instances of ' '%s are being created.' % cls.__name__ ) @classmethod def initialized(cls): """Has an instance been created?""" return hasattr(cls, "_instance") and cls._instance is not None ================================================ FILE: lib/client/traitlets/config/loader.py ================================================ # encoding: utf-8 """A simple configuration system.""" # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. import argparse import copy import logging import os import re import sys import json from ast import literal_eval from ipython_genutils.path import filefind from ipython_genutils import py3compat from ipython_genutils.encoding import DEFAULT_ENCODING from six import text_type from traitlets.traitlets import HasTraits, List, Any #----------------------------------------------------------------------------- # Exceptions #----------------------------------------------------------------------------- class ConfigError(Exception): pass class ConfigLoaderError(ConfigError): pass class ConfigFileNotFound(ConfigError): pass class ArgumentError(ConfigLoaderError): pass #----------------------------------------------------------------------------- # Argparse fix #----------------------------------------------------------------------------- # Unfortunately argparse by default prints help messages to stderr instead of # stdout. This makes it annoying to capture long help screens at the command # line, since one must know how to pipe stderr, which many users don't know how # to do. So we override the print_help method with one that defaults to # stdout and use our class instead. class ArgumentParser(argparse.ArgumentParser): """Simple argparse subclass that prints help to stdout by default.""" def print_help(self, file=None): if file is None: file = sys.stdout return super(ArgumentParser, self).print_help(file) print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__ #----------------------------------------------------------------------------- # Config class for holding config information #----------------------------------------------------------------------------- class LazyConfigValue(HasTraits): """Proxy object for exposing methods on configurable containers Exposes: - append, extend, insert on lists - update on dicts - update, add on sets """ _value = None # list methods _extend = List() _prepend = List() def append(self, obj): self._extend.append(obj) def extend(self, other): self._extend.extend(other) def prepend(self, other): """like list.extend, but for the front""" self._prepend[:0] = other _inserts = List() def insert(self, index, other): if not isinstance(index, int): raise TypeError("An integer is required") self._inserts.append((index, other)) # dict methods # update is used for both dict and set _update = Any() def update(self, other): if self._update is None: if isinstance(other, dict): self._update = {} else: self._update = set() self._update.update(other) # set methods def add(self, obj): self.update({obj}) def get_value(self, initial): """construct the value from the initial one after applying any insert / extend / update changes """ if self._value is not None: return self._value value = copy.deepcopy(initial) if isinstance(value, list): for idx, obj in self._inserts: value.insert(idx, obj) value[:0] = self._prepend value.extend(self._extend) elif isinstance(value, dict): if self._update: value.update(self._update) elif isinstance(value, set): if self._update: value.update(self._update) self._value = value return value def to_dict(self): """return JSONable dict form of my data Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples. """ d = {} if self._update: d['update'] = self._update if self._extend: d['extend'] = self._extend if self._prepend: d['prepend'] = self._prepend elif self._inserts: d['inserts'] = self._inserts return d def _is_section_key(key): """Is a Config key a section name (does it start with a capital)?""" if key and key[0].upper()==key[0] and not key.startswith('_'): return True else: return False class Config(dict): """An attribute based dict that can do smart merges.""" def __init__(self, *args, **kwds): dict.__init__(self, *args, **kwds) self._ensure_subconfig() def _ensure_subconfig(self): """ensure that sub-dicts that should be Config objects are casts dicts that are under section keys to Config objects, which is necessary for constructing Config objects from dict literals. """ for key in self: obj = self[key] if _is_section_key(key) \ and isinstance(obj, dict) \ and not isinstance(obj, Config): setattr(self, key, Config(obj)) def _merge(self, other): """deprecated alias, use Config.merge()""" self.merge(other) def merge(self, other): """merge another config object into this one""" to_update = {} for k, v in other.items(): if k not in self: to_update[k] = v else: # I have this key if isinstance(v, Config) and isinstance(self[k], Config): # Recursively merge common sub Configs self[k].merge(v) else: # Plain updates for non-Configs to_update[k] = v self.update(to_update) def collisions(self, other): """Check for collisions between two config objects. Returns a dict of the form {"Class": {"trait": "collision message"}}`, indicating which values have been ignored. An empty dict indicates no collisions. """ collisions = {} for section in self: if section not in other: continue mine = self[section] theirs = other[section] for key in mine: if key in theirs and mine[key] != theirs[key]: collisions.setdefault(section, {}) collisions[section][key] = "%r ignored, using %r" % (mine[key], theirs[key]) return collisions def __contains__(self, key): # allow nested contains of the form `"Section.key" in config` if '.' in key: first, remainder = key.split('.', 1) if first not in self: return False return remainder in self[first] return super(Config, self).__contains__(key) # .has_key is deprecated for dictionaries. has_key = __contains__ def _has_section(self, key): return _is_section_key(key) and key in self def copy(self): return type(self)(dict.copy(self)) def __copy__(self): return self.copy() def __deepcopy__(self, memo): new_config = type(self)() for key, value in self.items(): if isinstance(value, (Config, LazyConfigValue)): # deep copy config objects value = copy.deepcopy(value, memo) elif type(value) in {dict, list, set, tuple}: # shallow copy plain container traits value = copy.copy(value) new_config[key] = value return new_config def __getitem__(self, key): try: return dict.__getitem__(self, key) except KeyError: if _is_section_key(key): c = Config() dict.__setitem__(self, key, c) return c elif not key.startswith('_'): # undefined, create lazy value, used for container methods v = LazyConfigValue() dict.__setitem__(self, key, v) return v else: raise KeyError def __setitem__(self, key, value): if _is_section_key(key): if not isinstance(value, Config): raise ValueError('values whose keys begin with an uppercase ' 'char must be Config instances: %r, %r' % (key, value)) dict.__setitem__(self, key, value) def __getattr__(self, key): if key.startswith('__'): return dict.__getattr__(self, key) try: return self.__getitem__(key) except KeyError as e: raise AttributeError(e) def __setattr__(self, key, value): if key.startswith('__'): return dict.__setattr__(self, key, value) try: self.__setitem__(key, value) except KeyError as e: raise AttributeError(e) def __delattr__(self, key): if key.startswith('__'): return dict.__delattr__(self, key) try: dict.__delitem__(self, key) except KeyError as e: raise AttributeError(e) #----------------------------------------------------------------------------- # Config loading classes #----------------------------------------------------------------------------- class ConfigLoader(object): """A object for loading configurations from just about anywhere. The resulting configuration is packaged as a :class:`Config`. Notes ----- A :class:`ConfigLoader` does one thing: load a config from a source (file, command line arguments) and returns the data as a :class:`Config` object. There are lots of things that :class:`ConfigLoader` does not do. It does not implement complex logic for finding config files. It does not handle default values or merge multiple configs. These things need to be handled elsewhere. """ def _log_default(self): from traitlets.log import get_logger return get_logger() def __init__(self, log=None): """A base class for config loaders. log : instance of :class:`logging.Logger` to use. By default loger of :meth:`traitlets.config.application.Application.instance()` will be used Examples -------- >>> cl = ConfigLoader() >>> config = cl.load_config() >>> config {} """ self.clear() if log is None: self.log = self._log_default() self.log.debug('Using default logger') else: self.log = log def clear(self): self.config = Config() def load_config(self): """Load a config from somewhere, return a :class:`Config` instance. Usually, this will cause self.config to be set and then returned. However, in most cases, :meth:`ConfigLoader.clear` should be called to erase any previous state. """ self.clear() return self.config class FileConfigLoader(ConfigLoader): """A base class for file based configurations. As we add more file based config loaders, the common logic should go here. """ def __init__(self, filename, path=None, **kw): """Build a config loader for a filename and path. Parameters ---------- filename : str The file name of the config file. path : str, list, tuple The path to search for the config file on, or a sequence of paths to try in order. """ super(FileConfigLoader, self).__init__(**kw) self.filename = filename self.path = path self.full_filename = '' def _find_file(self): """Try to find the file by searching the paths.""" self.full_filename = filefind(self.filename, self.path) class JSONFileConfigLoader(FileConfigLoader): """A JSON file loader for config Can also act as a context manager that rewrite the configuration file to disk on exit. Example:: with JSONFileConfigLoader('myapp.json','/home/jupyter/configurations/') as c: c.MyNewConfigurable.new_value = 'Updated' """ def load_config(self): """Load the config from a file and return it as a Config object.""" self.clear() try: self._find_file() except IOError as e: raise ConfigFileNotFound(str(e)) dct = self._read_file_as_dict() self.config = self._convert_to_config(dct) return self.config def _read_file_as_dict(self): with open(self.full_filename) as f: return json.load(f) def _convert_to_config(self, dictionary): if 'version' in dictionary: version = dictionary.pop('version') else: version = 1 if version == 1: return Config(dictionary) else: raise ValueError('Unknown version of JSON config file: {version}'.format(version=version)) def __enter__(self): self.load_config() return self.config def __exit__(self, exc_type, exc_value, traceback): """ Exit the context manager but do not handle any errors. In case of any error, we do not want to write the potentially broken configuration to disk. """ self.config.version = 1 json_config = json.dumps(self.config, indent=2) with open(self.full_filename, 'w') as f: f.write(json_config) class PyFileConfigLoader(FileConfigLoader): """A config loader for pure python files. This is responsible for locating a Python config file by filename and path, then executing it to construct a Config object. """ def load_config(self): """Load the config from a file and return it as a Config object.""" self.clear() try: self._find_file() except IOError as e: raise ConfigFileNotFound(str(e)) self._read_file_as_dict() return self.config def load_subconfig(self, fname, path=None): """Injected into config file namespace as load_subconfig""" if path is None: path = self.path loader = self.__class__(fname, path) try: sub_config = loader.load_config() except ConfigFileNotFound: # Pass silently if the sub config is not there, # treat it as an empty config file. pass else: self.config.merge(sub_config) def _read_file_as_dict(self): """Load the config file into self.config, with recursive loading.""" def get_config(): """Unnecessary now, but a deprecation warning is more trouble than it's worth.""" return self.config namespace = dict( c=self.config, load_subconfig=self.load_subconfig, get_config=get_config, __file__=self.full_filename, ) fs_encoding = sys.getfilesystemencoding() or 'ascii' conf_filename = self.full_filename.encode(fs_encoding) py3compat.execfile(conf_filename, namespace) class CommandLineConfigLoader(ConfigLoader): """A config loader for command line arguments. As we add more command line based loaders, the common logic should go here. """ def _exec_config_str(self, lhs, rhs): """execute self.config. = * expands ~ with expanduser * tries to assign with literal_eval, otherwise assigns with just the string, allowing `--C.a=foobar` and `--C.a="foobar"` to be equivalent. *Not* equivalent are `--C.a=4` and `--C.a='4'`. """ rhs = os.path.expanduser(rhs) try: # Try to see if regular Python syntax will work. This # won't handle strings as the quote marks are removed # by the system shell. value = literal_eval(rhs) except (NameError, SyntaxError, ValueError): # This case happens if the rhs is a string. value = rhs exec(u'self.config.%s = value' % lhs) def _load_flag(self, cfg): """update self.config from a flag, which can be a dict or Config""" if isinstance(cfg, (dict, Config)): # don't clobber whole config sections, update # each section from config: for sec,c in cfg.items(): self.config[sec].update(c) else: raise TypeError("Invalid flag: %r" % cfg) # raw --identifier=value pattern # but *also* accept '-' as wordsep, for aliases # accepts: --foo=a # --Class.trait=value # --alias-name=value # rejects: -foo=value # --foo # --Class.trait kv_pattern = re.compile(r'\-\-[A-Za-z][\w\-]*(\.[\w\-]+)*\=.*') # just flags, no assignments, with two *or one* leading '-' # accepts: --foo # -foo-bar-again # rejects: --anything=anything # --two.word flag_pattern = re.compile(r'\-\-?\w[\-\w]*$') class KeyValueConfigLoader(CommandLineConfigLoader): """A config loader that loads key value pairs from the command line. This allows command line options to be gives in the following form:: ipython --profile="foo" --InteractiveShell.autocall=False """ def __init__(self, argv=None, aliases=None, flags=None, **kw): """Create a key value pair config loader. Parameters ---------- argv : list A list that has the form of sys.argv[1:] which has unicode elements of the form u"key=value". If this is None (default), then sys.argv[1:] will be used. aliases : dict A dict of aliases for configurable traits. Keys are the short aliases, Values are the resolved trait. Of the form: `{'alias' : 'Configurable.trait'}` flags : dict A dict of flags, keyed by str name. Vaues can be Config objects, dicts, or "key=value" strings. If Config or dict, when the flag is triggered, The flag is loaded as `self.config.update(m)`. Returns ------- config : Config The resulting Config object. Examples -------- >>> from traitlets.config.loader import KeyValueConfigLoader >>> cl = KeyValueConfigLoader() >>> d = cl.load_config(["--A.name='brian'","--B.number=0"]) >>> sorted(d.items()) [('A', {'name': 'brian'}), ('B', {'number': 0})] """ super(KeyValueConfigLoader, self).__init__(**kw) if argv is None: argv = sys.argv[1:] self.argv = argv self.aliases = aliases or {} self.flags = flags or {} def clear(self): super(KeyValueConfigLoader, self).clear() self.extra_args = [] def _decode_argv(self, argv, enc=None): """decode argv if bytes, using stdin.encoding, falling back on default enc""" uargv = [] if enc is None: enc = DEFAULT_ENCODING for arg in argv: if not isinstance(arg, text_type): # only decode if not already decoded arg = arg.decode(enc) uargv.append(arg) return uargv def load_config(self, argv=None, aliases=None, flags=None): """Parse the configuration and generate the Config object. After loading, any arguments that are not key-value or flags will be stored in self.extra_args - a list of unparsed command-line arguments. This is used for arguments such as input files or subcommands. Parameters ---------- argv : list, optional A list that has the form of sys.argv[1:] which has unicode elements of the form u"key=value". If this is None (default), then self.argv will be used. aliases : dict A dict of aliases for configurable traits. Keys are the short aliases, Values are the resolved trait. Of the form: `{'alias' : 'Configurable.trait'}` flags : dict A dict of flags, keyed by str name. Values can be Config objects or dicts. When the flag is triggered, The config is loaded as `self.config.update(cfg)`. """ self.clear() if argv is None: argv = self.argv if aliases is None: aliases = self.aliases if flags is None: flags = self.flags # ensure argv is a list of unicode strings: uargv = self._decode_argv(argv) for idx,raw in enumerate(uargv): # strip leading '-' item = raw.lstrip('-') if raw == '--': # don't parse arguments after '--' # this is useful for relaying arguments to scripts, e.g. # ipython -i foo.py --matplotlib=qt -- args after '--' go-to-foo.py self.extra_args.extend(uargv[idx+1:]) break if kv_pattern.match(raw): lhs,rhs = item.split('=',1) # Substitute longnames for aliases. if lhs in aliases: lhs = aliases[lhs] if '.' not in lhs: # probably a mistyped alias, but not technically illegal self.log.warning("Unrecognized alias: '%s', it will probably have no effect.", raw) try: self._exec_config_str(lhs, rhs) except Exception: raise ArgumentError("Invalid argument: '%s'" % raw) elif flag_pattern.match(raw): if item in flags: cfg,_ = flags[item] self._load_flag(cfg) else: raise ArgumentError("Unrecognized flag: '%s'"%raw) elif raw.startswith('-'): kv = '--'+item if kv_pattern.match(kv): raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv)) else: raise ArgumentError("Invalid argument: '%s'"%raw) else: # keep all args that aren't valid in a list, # in case our parent knows what to do with them. self.extra_args.append(item) return self.config class ArgParseConfigLoader(CommandLineConfigLoader): """A loader that uses the argparse module to load from the command line.""" def __init__(self, argv=None, aliases=None, flags=None, log=None, *parser_args, **parser_kw): """Create a config loader for use with argparse. Parameters ---------- argv : optional, list If given, used to read command-line arguments from, otherwise sys.argv[1:] is used. parser_args : tuple A tuple of positional arguments that will be passed to the constructor of :class:`argparse.ArgumentParser`. parser_kw : dict A tuple of keyword arguments that will be passed to the constructor of :class:`argparse.ArgumentParser`. Returns ------- config : Config The resulting Config object. """ super(CommandLineConfigLoader, self).__init__(log=log) self.clear() if argv is None: argv = sys.argv[1:] self.argv = argv self.aliases = aliases or {} self.flags = flags or {} self.parser_args = parser_args self.version = parser_kw.pop("version", None) kwargs = dict(argument_default=argparse.SUPPRESS) kwargs.update(parser_kw) self.parser_kw = kwargs def load_config(self, argv=None, aliases=None, flags=None): """Parse command line arguments and return as a Config object. Parameters ---------- args : optional, list If given, a list with the structure of sys.argv[1:] to parse arguments from. If not given, the instance's self.argv attribute (given at construction time) is used.""" self.clear() if argv is None: argv = self.argv if aliases is None: aliases = self.aliases if flags is None: flags = self.flags self._create_parser(aliases, flags) self._parse_args(argv) self._convert_to_config() return self.config def get_extra_args(self): if hasattr(self, 'extra_args'): return self.extra_args else: return [] def _create_parser(self, aliases=None, flags=None): self.parser = ArgumentParser(*self.parser_args, **self.parser_kw) self._add_arguments(aliases, flags) def _add_arguments(self, aliases=None, flags=None): raise NotImplementedError("subclasses must implement _add_arguments") def _parse_args(self, args): """self.parser->self.parsed_data""" # decode sys.argv to support unicode command-line options enc = DEFAULT_ENCODING uargs = [py3compat.cast_unicode(a, enc) for a in args] self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs) def _convert_to_config(self): """self.parsed_data->self.config""" for k, v in vars(self.parsed_data).items(): exec("self.config.%s = v"%k, locals(), globals()) class KVArgParseConfigLoader(ArgParseConfigLoader): """A config loader that loads aliases and flags with argparse, but will use KVLoader for the rest. This allows better parsing of common args, such as `ipython -c 'print 5'`, but still gets arbitrary config with `ipython --InteractiveShell.use_readline=False`""" def _add_arguments(self, aliases=None, flags=None): self.alias_flags = {} # print aliases, flags if aliases is None: aliases = self.aliases if flags is None: flags = self.flags paa = self.parser.add_argument for key,value in aliases.items(): if key in flags: # flags nargs = '?' else: nargs = None if len(key) == 1: paa('-'+key, '--'+key, type=text_type, dest=value, nargs=nargs) else: paa('--'+key, type=text_type, dest=value, nargs=nargs) for key, (value, help) in flags.items(): if key in self.aliases: # self.alias_flags[self.aliases[key]] = value continue if len(key) == 1: paa('-'+key, '--'+key, action='append_const', dest='_flags', const=value) else: paa('--'+key, action='append_const', dest='_flags', const=value) def _convert_to_config(self): """self.parsed_data->self.config, parse unrecognized extra args via KVLoader.""" # remove subconfigs list from namespace before transforming the Namespace if '_flags' in self.parsed_data: subcs = self.parsed_data._flags del self.parsed_data._flags else: subcs = [] for k, v in vars(self.parsed_data).items(): if v is None: # it was a flag that shares the name of an alias subcs.append(self.alias_flags[k]) else: # eval the KV assignment self._exec_config_str(k, v) for subc in subcs: self._load_flag(subc) if self.extra_args: sub_parser = KeyValueConfigLoader(log=self.log) sub_parser.load_config(self.extra_args) self.config.merge(sub_parser.config) self.extra_args = sub_parser.extra_args def load_pyconfig_files(config_files, path): """Load multiple Python config files, merging each of them in turn. Parameters ========== config_files : list of str List of config files names to load and merge into the config. path : unicode The full path to the location of the config files. """ config = Config() for cf in config_files: loader = PyFileConfigLoader(cf, path=path) try: next_config = loader.load_config() except ConfigFileNotFound: pass except: raise else: config.merge(next_config) return config ================================================ FILE: lib/client/traitlets/config/manager.py ================================================ """Manager to read and modify config data in JSON files. """ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. import errno import io import json import os from six import PY3 from traitlets.config import LoggingConfigurable from traitlets.traitlets import Unicode def recursive_update(target, new): """Recursively update one dictionary using another. None values will delete their keys. """ for k, v in new.items(): if isinstance(v, dict): if k not in target: target[k] = {} recursive_update(target[k], v) if not target[k]: # Prune empty subdicts del target[k] elif v is None: target.pop(k, None) else: target[k] = v class BaseJSONConfigManager(LoggingConfigurable): """General JSON config manager Deals with persisting/storing config in a json file """ config_dir = Unicode('.') def ensure_config_dir_exists(self): try: os.makedirs(self.config_dir, 0o755) except OSError as e: if e.errno != errno.EEXIST: raise def file_name(self, section_name): return os.path.join(self.config_dir, section_name+'.json') def get(self, section_name): """Retrieve the config data for the specified section. Returns the data as a dictionary, or an empty dictionary if the file doesn't exist. """ filename = self.file_name(section_name) if os.path.isfile(filename): with io.open(filename, encoding='utf-8') as f: return json.load(f) else: return {} def set(self, section_name, data): """Store the given config data. """ filename = self.file_name(section_name) self.ensure_config_dir_exists() if PY3: f = io.open(filename, 'w', encoding='utf-8') else: f = open(filename, 'wb') with f: json.dump(data, f, indent=2) def update(self, section_name, new_data): """Modify the config section by recursively updating it with new_data. Returns the modified config data as a dictionary. """ data = self.get(section_name) recursive_update(data, new_data) self.set(section_name, data) return data ================================================ FILE: lib/client/traitlets/log.py ================================================ """Grab the global logger instance.""" # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. import logging _logger = None def get_logger(): """Grab the global logger instance. If a global Application is instantiated, grab its logger. Otherwise, grab the root logger. """ global _logger if _logger is None: from .config import Application if Application.initialized(): _logger = Application.instance().log else: _logger = logging.getLogger('traitlets') # Add a NullHandler to silence warnings about not being # initialized, per best practice for libraries. _logger.addHandler(logging.NullHandler()) return _logger ================================================ FILE: lib/client/traitlets/traitlets.py ================================================ # encoding: utf-8 """ A lightweight Traits like module. This is designed to provide a lightweight, simple, pure Python version of many of the capabilities of enthought.traits. This includes: * Validation * Type specification with defaults * Static and dynamic notification * Basic predefined types * An API that is similar to enthought.traits We don't support: * Delegation * Automatic GUI generation * A full set of trait types. Most importantly, we don't provide container traits (list, dict, tuple) that can trigger notifications if their contents change. * API compatibility with enthought.traits There are also some important difference in our design: * enthought.traits does not validate default values. We do. We choose to create this module because we need these capabilities, but we need them to be pure Python so they work in all Python implementations, including Jython and IronPython. Inheritance diagram: .. inheritance-diagram:: traitlets.traitlets :parts: 3 """ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. # # Adapted from enthought.traits, Copyright (c) Enthought, Inc., # also under the terms of the Modified BSD License. import contextlib import inspect import os import re import sys import types import enum try: from types import ClassType, InstanceType ClassTypes = (ClassType, type) except: ClassTypes = (type,) from warnings import warn, warn_explicit import six from .utils.getargspec import getargspec from .utils.importstring import import_item from .utils.sentinel import Sentinel from .utils.bunch import Bunch SequenceTypes = (list, tuple, set, frozenset) #----------------------------------------------------------------------------- # Basic classes #----------------------------------------------------------------------------- Undefined = Sentinel('Undefined', 'traitlets', ''' Used in Traitlets to specify that no defaults are set in kwargs ''' ) All = Sentinel('All', 'traitlets', ''' Used in Traitlets to listen to all types of notification or to notifications from all trait attributes. ''' ) # Deprecated alias NoDefaultSpecified = Undefined class TraitError(Exception): pass #----------------------------------------------------------------------------- # Utilities #----------------------------------------------------------------------------- from ipython_genutils.py3compat import cast_unicode_py2 _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$") def isidentifier(s): if six.PY2: return bool(_name_re.match(s)) else: return s.isidentifier() _deprecations_shown = set() def _should_warn(key): """Add our own checks for too many deprecation warnings. Limit to once per package. """ env_flag = os.environ.get('TRAITLETS_ALL_DEPRECATIONS') if env_flag and env_flag != '0': return True if key not in _deprecations_shown: _deprecations_shown.add(key) return True else: return False def _deprecated_method(method, cls, method_name, msg): """Show deprecation warning about a magic method definition. Uses warn_explicit to bind warning to method definition instead of triggering code, which isn't relevant. """ warn_msg = "{classname}.{method_name} is deprecated in traitlets 4.1: {msg}".format( classname=cls.__name__, method_name=method_name, msg=msg ) for parent in inspect.getmro(cls): if method_name in parent.__dict__: cls = parent break # limit deprecation messages to once per package package_name = cls.__module__.split('.', 1)[0] key = (package_name, msg) if not _should_warn(key): return try: fname = inspect.getsourcefile(method) or "" lineno = inspect.getsourcelines(method)[1] or 0 except (IOError, TypeError) as e: # Failed to inspect for some reason warn(warn_msg + ('\n(inspection failed) %s' % e), DeprecationWarning) else: warn_explicit(warn_msg, DeprecationWarning, fname, lineno) def class_of(object): """ Returns a string containing the class name of an object with the correct indefinite article ('a' or 'an') preceding it (e.g., 'an Image', 'a PlotValue'). """ if isinstance( object, six.string_types ): return add_article( object ) return add_article( object.__class__.__name__ ) def add_article(name): """ Returns a string containing the correct indefinite article ('a' or 'an') prefixed to the specified string. """ if name[:1].lower() in 'aeiou': return 'an ' + name return 'a ' + name def repr_type(obj): """ Return a string representation of a value and its type for readable error messages. """ the_type = type(obj) if six.PY2 and the_type is InstanceType: # Old-style class. the_type = obj.__class__ msg = '%r %r' % (obj, the_type) return msg def is_trait(t): """ Returns whether the given value is an instance or subclass of TraitType. """ return (isinstance(t, TraitType) or (isinstance(t, type) and issubclass(t, TraitType))) def parse_notifier_name(names): """Convert the name argument to a list of names. Examples -------- >>> parse_notifier_name([]) [All] >>> parse_notifier_name('a') ['a'] >>> parse_notifier_name(['a', 'b']) ['a', 'b'] >>> parse_notifier_name(All) [All] """ if names is All or isinstance(names, six.string_types): return [names] else: if not names or All in names: return [All] for n in names: if not isinstance(n, six.string_types): raise TypeError("names must be strings, not %r" % n) return names class _SimpleTest: def __init__ ( self, value ): self.value = value def __call__ ( self, test ): return test == self.value def __repr__(self): return ">> c = link((src, 'value'), (tgt, 'value')) >>> src.value = 5 # updates other objects as well """ updating = False def __init__(self, source, target): _validate_link(source, target) self.source, self.target = source, target try: setattr(target[0], target[1], getattr(source[0], source[1])) finally: source[0].observe(self._update_target, names=source[1]) target[0].observe(self._update_source, names=target[1]) @contextlib.contextmanager def _busy_updating(self): self.updating = True try: yield finally: self.updating = False def _update_target(self, change): if self.updating: return with self._busy_updating(): setattr(self.target[0], self.target[1], change.new) def _update_source(self, change): if self.updating: return with self._busy_updating(): setattr(self.source[0], self.source[1], change.new) def unlink(self): self.source[0].unobserve(self._update_target, names=self.source[1]) self.target[0].unobserve(self._update_source, names=self.target[1]) self.source, self.target = None, None class directional_link(object): """Link the trait of a source object with traits of target objects. Parameters ---------- source : (object, attribute name) pair target : (object, attribute name) pair transform: callable (optional) Data transformation between source and target. Examples -------- >>> c = directional_link((src, 'value'), (tgt, 'value')) >>> src.value = 5 # updates target objects >>> tgt.value = 6 # does not update source object """ updating = False def __init__(self, source, target, transform=None): self._transform = transform if transform else lambda x: x _validate_link(source, target) self.source, self.target = source, target try: setattr(target[0], target[1], self._transform(getattr(source[0], source[1]))) finally: self.source[0].observe(self._update, names=self.source[1]) @contextlib.contextmanager def _busy_updating(self): self.updating = True try: yield finally: self.updating = False def _update(self, change): if self.updating: return with self._busy_updating(): setattr(self.target[0], self.target[1], self._transform(change.new)) def unlink(self): self.source[0].unobserve(self._update, names=self.source[1]) self.source, self.target = None, None dlink = directional_link #----------------------------------------------------------------------------- # Base Descriptor Class #----------------------------------------------------------------------------- class BaseDescriptor(object): """Base descriptor class Notes ----- This implements Python's descriptor prototol. This class is the base class for all such descriptors. The only magic we use is a custom metaclass for the main :class:`HasTraits` class that does the following: 1. Sets the :attr:`name` attribute of every :class:`BaseDescriptor` instance in the class dict to the name of the attribute. 2. Sets the :attr:`this_class` attribute of every :class:`BaseDescriptor` instance in the class dict to the *class* that declared the trait. This is used by the :class:`This` trait to allow subclasses to accept superclasses for :class:`This` values. """ name = None this_class = None def class_init(self, cls, name): """Part of the initialization which may depend on the underlying HasDescriptors class. It is typically overloaded for specific types. This method is called by :meth:`MetaHasDescriptors.__init__` passing the class (`cls`) and `name` under which the descriptor has been assigned. """ self.this_class = cls self.name = name def instance_init(self, obj): """Part of the initialization which may depend on the underlying HasDescriptors instance. It is typically overloaded for specific types. This method is called by :meth:`HasTraits.__new__` and in the :meth:`BaseDescriptor.instance_init` method of descriptors holding other descriptors. """ pass class TraitType(BaseDescriptor): """A base class for all trait types. """ metadata = {} default_value = Undefined allow_none = False read_only = False info_text = 'any value' def __init__(self, default_value=Undefined, allow_none=False, read_only=None, help=None, config=None, **kwargs): """Declare a traitlet. If *allow_none* is True, None is a valid value in addition to any values that are normally valid. The default is up to the subclass. For most trait types, the default value for ``allow_none`` is False. Extra metadata can be associated with the traitlet using the .tag() convenience method or by using the traitlet instance's .metadata dictionary. """ if default_value is not Undefined: self.default_value = default_value if allow_none: self.allow_none = allow_none if read_only is not None: self.read_only = read_only self.help = help if help is not None else '' if len(kwargs) > 0: stacklevel = 1 f = inspect.currentframe() # count supers to determine stacklevel for warning while f.f_code.co_name == '__init__': stacklevel += 1 f = f.f_back mod = f.f_globals.get('__name__') or '' pkg = mod.split('.', 1)[0] key = tuple(['metadata-tag', pkg] + sorted(kwargs)) if _should_warn(key): warn("metadata %s was set from the constructor. " "With traitlets 4.1, metadata should be set using the .tag() method, " "e.g., Int().tag(key1='value1', key2='value2')" % (kwargs,), DeprecationWarning, stacklevel=stacklevel) if len(self.metadata) > 0: self.metadata = self.metadata.copy() self.metadata.update(kwargs) else: self.metadata = kwargs else: self.metadata = self.metadata.copy() if config is not None: self.metadata['config'] = config # We add help to the metadata during a deprecation period so that # code that looks for the help string there can find it. if help is not None: self.metadata['help'] = help def get_default_value(self): """DEPRECATED: Retrieve the static default value for this trait. Use self.default_value instead """ warn("get_default_value is deprecated in traitlets 4.0: use the .default_value attribute", DeprecationWarning, stacklevel=2) return self.default_value def init_default_value(self, obj): """DEPRECATED: Set the static default value for the trait type. """ warn("init_default_value is deprecated in traitlets 4.0, and may be removed in the future", DeprecationWarning, stacklevel=2) value = self._validate(obj, self.default_value) obj._trait_values[self.name] = value return value def _dynamic_default_callable(self, obj): """Retrieve a callable to calculate the default for this traitlet. This looks for: * default generators registered with the @default descriptor. * obj._{name}_default() on the class with the traitlet, or a subclass that obj belongs to. * trait.make_dynamic_default, which is defined by Instance If neither exist, it returns None """ # Traitlets without a name are not on the instance, e.g. in List or Union if self.name: # Only look for default handlers in classes derived from self.this_class. mro = type(obj).mro() meth_name = '_%s_default' % self.name for cls in mro[:mro.index(self.this_class) + 1]: if hasattr(cls, '_trait_default_generators'): default_handler = cls._trait_default_generators.get(self.name) if default_handler is not None and default_handler.this_class == cls: return types.MethodType(default_handler.func, obj) if meth_name in cls.__dict__: method = getattr(obj, meth_name) return method return getattr(self, 'make_dynamic_default', None) def instance_init(self, obj): # If no dynamic initialiser is present, and the trait implementation or # use provides a static default, transfer that to obj._trait_values. with obj.cross_validation_lock: if (self._dynamic_default_callable(obj) is None) \ and (self.default_value is not Undefined): v = self._validate(obj, self.default_value) if self.name is not None: obj._trait_values[self.name] = v def get(self, obj, cls=None): try: value = obj._trait_values[self.name] except KeyError: # Check for a dynamic initializer. dynamic_default = self._dynamic_default_callable(obj) if dynamic_default is None: raise TraitError("No default value found for %s trait of %r" % (self.name, obj)) value = self._validate(obj, dynamic_default()) obj._trait_values[self.name] = value return value except Exception: # This should never be reached. raise TraitError('Unexpected error in TraitType: ' 'default value not set properly') else: return value def __get__(self, obj, cls=None): """Get the value of the trait by self.name for the instance. Default values are instantiated when :meth:`HasTraits.__new__` is called. Thus by the time this method gets called either the default value or a user defined value (they called :meth:`__set__`) is in the :class:`HasTraits` instance. """ if obj is None: return self else: return self.get(obj, cls) def set(self, obj, value): new_value = self._validate(obj, value) try: old_value = obj._trait_values[self.name] except KeyError: old_value = self.default_value obj._trait_values[self.name] = new_value try: silent = bool(old_value == new_value) except: # if there is an error in comparing, default to notify silent = False if silent is not True: # we explicitly compare silent to True just in case the equality # comparison above returns something other than True/False obj._notify_trait(self.name, old_value, new_value) def __set__(self, obj, value): """Set the value of the trait by self.name for the instance. Values pass through a validation stage where errors are raised when impropper types, or types that cannot be coerced, are encountered. """ if self.read_only: raise TraitError('The "%s" trait is read-only.' % self.name) else: self.set(obj, value) def _validate(self, obj, value): if value is None and self.allow_none: return value if hasattr(self, 'validate'): value = self.validate(obj, value) if obj._cross_validation_lock is False: value = self._cross_validate(obj, value) return value def _cross_validate(self, obj, value): if self.name in obj._trait_validators: proposal = Bunch({'trait': self, 'value': value, 'owner': obj}) value = obj._trait_validators[self.name](obj, proposal) elif hasattr(obj, '_%s_validate' % self.name): meth_name = '_%s_validate' % self.name cross_validate = getattr(obj, meth_name) _deprecated_method(cross_validate, obj.__class__, meth_name, "use @validate decorator instead.") value = cross_validate(value, self) return value def __or__(self, other): if isinstance(other, Union): return Union([self] + other.trait_types) else: return Union([self, other]) def info(self): return self.info_text def error(self, obj, value): if obj is not None: e = "The '%s' trait of %s instance must be %s, but a value of %s was specified." \ % (self.name, class_of(obj), self.info(), repr_type(value)) else: e = "The '%s' trait must be %s, but a value of %r was specified." \ % (self.name, self.info(), repr_type(value)) raise TraitError(e) def get_metadata(self, key, default=None): """DEPRECATED: Get a metadata value. Use .metadata[key] or .metadata.get(key, default) instead. """ if key == 'help': msg = "use the instance .help string directly, like x.help" else: msg = "use the instance .metadata dictionary directly, like x.metadata[key] or x.metadata.get(key, default)" warn("Deprecated in traitlets 4.1, " + msg, DeprecationWarning, stacklevel=2) return self.metadata.get(key, default) def set_metadata(self, key, value): """DEPRECATED: Set a metadata key/value. Use .metadata[key] = value instead. """ if key == 'help': msg = "use the instance .help string directly, like x.help = value" else: msg = "use the instance .metadata dictionary directly, like x.metadata[key] = value" warn("Deprecated in traitlets 4.1, " + msg, DeprecationWarning, stacklevel=2) self.metadata[key] = value def tag(self, **metadata): """Sets metadata and returns self. This allows convenient metadata tagging when initializing the trait, such as: >>> Int(0).tag(config=True, sync=True) """ maybe_constructor_keywords = set(metadata.keys()).intersection({'help','allow_none', 'read_only', 'default_value'}) if maybe_constructor_keywords: warn('The following attributes are set in using `tag`, but seem to be constructor keywords arguments: %s '% maybe_constructor_keywords, UserWarning, stacklevel=2) self.metadata.update(metadata) return self def default_value_repr(self): return repr(self.default_value) #----------------------------------------------------------------------------- # The HasTraits implementation #----------------------------------------------------------------------------- class _CallbackWrapper(object): """An object adapting a on_trait_change callback into an observe callback. The comparison operator __eq__ is implemented to enable removal of wrapped callbacks. """ def __init__(self, cb): self.cb = cb # Bound methods have an additional 'self' argument. offset = -1 if isinstance(self.cb, types.MethodType) else 0 self.nargs = len(getargspec(cb)[0]) + offset if (self.nargs > 4): raise TraitError('a trait changed callback must have 0-4 arguments.') def __eq__(self, other): # The wrapper is equal to the wrapped element if isinstance(other, _CallbackWrapper): return self.cb == other.cb else: return self.cb == other def __call__(self, change): # The wrapper is callable if self.nargs == 0: self.cb() elif self.nargs == 1: self.cb(change.name) elif self.nargs == 2: self.cb(change.name, change.new) elif self.nargs == 3: self.cb(change.name, change.old, change.new) elif self.nargs == 4: self.cb(change.name, change.old, change.new, change.owner) def _callback_wrapper(cb): if isinstance(cb, _CallbackWrapper): return cb else: return _CallbackWrapper(cb) class MetaHasDescriptors(type): """A metaclass for HasDescriptors. This metaclass makes sure that any TraitType class attributes are instantiated and sets their name attribute. """ def __new__(mcls, name, bases, classdict): """Create the HasDescriptors class.""" for k, v in classdict.items(): # ---------------------------------------------------------------- # Support of deprecated behavior allowing for TraitType types # to be used instead of TraitType instances. if inspect.isclass(v) and issubclass(v, TraitType): warn("Traits should be given as instances, not types (for example, `Int()`, not `Int`)." " Passing types is deprecated in traitlets 4.1.", DeprecationWarning, stacklevel=2) classdict[k] = v() # ---------------------------------------------------------------- return super(MetaHasDescriptors, mcls).__new__(mcls, name, bases, classdict) def __init__(cls, name, bases, classdict): """Finish initializing the HasDescriptors class.""" super(MetaHasDescriptors, cls).__init__(name, bases, classdict) cls.setup_class(classdict) def setup_class(cls, classdict): """Setup descriptor instance on the class This sets the :attr:`this_class` and :attr:`name` attributes of each BaseDescriptor in the class dict of the newly created ``cls`` before calling their :attr:`class_init` method. """ for k, v in classdict.items(): if isinstance(v, BaseDescriptor): v.class_init(cls, k) class MetaHasTraits(MetaHasDescriptors): """A metaclass for HasTraits.""" def setup_class(cls, classdict): cls._trait_default_generators = {} super(MetaHasTraits, cls).setup_class(classdict) def observe(*names, **kwargs): """A decorator which can be used to observe Traits on a class. The handler passed to the decorator will be called with one ``change`` dict argument. The change dictionary at least holds a 'type' key and a 'name' key, corresponding respectively to the type of notification and the name of the attribute that triggered the notification. Other keys may be passed depending on the value of 'type'. In the case where type is 'change', we also have the following keys: * ``owner`` : the HasTraits instance * ``old`` : the old value of the modified trait attribute * ``new`` : the new value of the modified trait attribute * ``name`` : the name of the modified trait attribute. Parameters ---------- *names The str names of the Traits to observe on the object. type: str, kwarg-only The type of event to observe (e.g. 'change') """ if not names: raise TypeError("Please specify at least one trait name to observe.") for name in names: if name is not All and not isinstance(name, six.string_types): raise TypeError("trait names to observe must be strings or All, not %r" % name) return ObserveHandler(names, type=kwargs.get('type', 'change')) def observe_compat(func): """Backward-compatibility shim decorator for observers Use with: @observe('name') @observe_compat def _foo_changed(self, change): ... With this, `super()._foo_changed(self, name, old, new)` in subclasses will still work. Allows adoption of new observer API without breaking subclasses that override and super. """ def compatible_observer(self, change_or_name, old=Undefined, new=Undefined): if isinstance(change_or_name, dict): change = change_or_name else: clsname = self.__class__.__name__ warn("A parent of %s._%s_changed has adopted the new (traitlets 4.1) @observe(change) API" % ( clsname, change_or_name), DeprecationWarning) change = Bunch( type='change', old=old, new=new, name=change_or_name, owner=self, ) return func(self, change) return compatible_observer def validate(*names): """A decorator to register cross validator of HasTraits object's state when a Trait is set. The handler passed to the decorator must have one ``proposal`` dict argument. The proposal dictionary must hold the following keys: * ``owner`` : the HasTraits instance * ``value`` : the proposed value for the modified trait attribute * ``trait`` : the TraitType instance associated with the attribute Parameters ---------- names The str names of the Traits to validate. Notes ----- Since the owner has access to the ``HasTraits`` instance via the 'owner' key, the registered cross validator could potentially make changes to attributes of the ``HasTraits`` instance. However, we recommend not to do so. The reason is that the cross-validation of attributes may run in arbitrary order when exiting the ``hold_trait_notifications`` context, and such changes may not commute. """ if not names: raise TypeError("Please specify at least one trait name to validate.") for name in names: if name is not All and not isinstance(name, six.string_types): raise TypeError("trait names to validate must be strings or All, not %r" % name) return ValidateHandler(names) def default(name): """ A decorator which assigns a dynamic default for a Trait on a HasTraits object. Parameters ---------- name The str name of the Trait on the object whose default should be generated. Notes ----- Unlike observers and validators which are properties of the HasTraits instance, default value generators are class-level properties. Besides, default generators are only invoked if they are registered in subclasses of `this_type`. :: class A(HasTraits): bar = Int() @default('bar') def get_bar_default(self): return 11 class B(A): bar = Float() # This trait ignores the default generator defined in # the base class A class C(B): @default('bar') def some_other_default(self): # This default generator should not be return 3.0 # ignored since it is defined in a # class derived from B.a.this_class. """ if not isinstance(name, six.string_types): raise TypeError("Trait name must be a string or All, not %r" % name) return DefaultHandler(name) class EventHandler(BaseDescriptor): def _init_call(self, func): self.func = func return self def __call__(self, *args, **kwargs): """Pass `*args` and `**kwargs` to the handler's function if it exists.""" if hasattr(self, 'func'): return self.func(*args, **kwargs) else: return self._init_call(*args, **kwargs) def __get__(self, inst, cls=None): if inst is None: return self return types.MethodType(self.func, inst) class ObserveHandler(EventHandler): def __init__(self, names, type): self.trait_names = names self.type = type def instance_init(self, inst): inst.observe(self, self.trait_names, type=self.type) class ValidateHandler(EventHandler): def __init__(self, names): self.trait_names = names def instance_init(self, inst): inst._register_validator(self, self.trait_names) class DefaultHandler(EventHandler): def __init__(self, name): self.trait_name = name def class_init(self, cls, name): super(DefaultHandler, self).class_init(cls, name) cls._trait_default_generators[self.trait_name] = self class HasDescriptors(six.with_metaclass(MetaHasDescriptors, object)): """The base class for all classes that have descriptors. """ def __new__(cls, *args, **kwargs): # This is needed because object.__new__ only accepts # the cls argument. new_meth = super(HasDescriptors, cls).__new__ if new_meth is object.__new__: inst = new_meth(cls) else: inst = new_meth(cls, *args, **kwargs) inst.setup_instance(*args, **kwargs) return inst def setup_instance(self, *args, **kwargs): """ This is called **before** self.__init__ is called. """ self._cross_validation_lock = False cls = self.__class__ for key in dir(cls): # Some descriptors raise AttributeError like zope.interface's # __provides__ attributes even though they exist. This causes # AttributeErrors even though they are listed in dir(cls). try: value = getattr(cls, key) except AttributeError: pass else: if isinstance(value, BaseDescriptor): value.instance_init(self) class HasTraits(six.with_metaclass(MetaHasTraits, HasDescriptors)): def setup_instance(self, *args, **kwargs): self._trait_values = {} self._trait_notifiers = {} self._trait_validators = {} super(HasTraits, self).setup_instance(*args, **kwargs) def __init__(self, *args, **kwargs): # Allow trait values to be set using keyword arguments. # We need to use setattr for this to trigger validation and # notifications. super_args = args super_kwargs = {} with self.hold_trait_notifications(): for key, value in kwargs.items(): if self.has_trait(key): setattr(self, key, value) else: # passthrough args that don't set traits to super super_kwargs[key] = value try: super(HasTraits, self).__init__(*super_args, **super_kwargs) except TypeError as e: arg_s_list = [ repr(arg) for arg in super_args ] for k, v in super_kwargs.items(): arg_s_list.append("%s=%r" % (k, v)) arg_s = ', '.join(arg_s_list) warn( "Passing unrecoginized arguments to super({classname}).__init__({arg_s}).\n" "{error}\n" "This is deprecated in traitlets 4.2." "This error will be raised in a future release of traitlets." .format( arg_s=arg_s, classname=self.__class__.__name__, error=e, ), DeprecationWarning, stacklevel=2, ) def __getstate__(self): d = self.__dict__.copy() # event handlers stored on an instance are # expected to be reinstantiated during a # recall of instance_init during __setstate__ d['_trait_notifiers'] = {} d['_trait_validators'] = {} return d def __setstate__(self, state): self.__dict__ = state.copy() # event handlers are reassigned to self cls = self.__class__ for key in dir(cls): # Some descriptors raise AttributeError like zope.interface's # __provides__ attributes even though they exist. This causes # AttributeErrors even though they are listed in dir(cls). try: value = getattr(cls, key) except AttributeError: pass else: if isinstance(value, EventHandler): value.instance_init(self) @property @contextlib.contextmanager def cross_validation_lock(self): """ A contextmanager for running a block with our cross validation lock set to True. At the end of the block, the lock's value is restored to its value prior to entering the block. """ if self._cross_validation_lock: yield return else: try: self._cross_validation_lock = True yield finally: self._cross_validation_lock = False @contextlib.contextmanager def hold_trait_notifications(self): """Context manager for bundling trait change notifications and cross validation. Use this when doing multiple trait assignments (init, config), to avoid race conditions in trait notifiers requesting other trait values. All trait notifications will fire after all values have been assigned. """ if self._cross_validation_lock: yield return else: cache = {} notify_change = self.notify_change def compress(past_changes, change): """Merges the provided change with the last if possible.""" if past_changes is None: return [change] else: if past_changes[-1]['type'] == 'change' and change.type == 'change': past_changes[-1]['new'] = change.new else: # In case of changes other than 'change', append the notification. past_changes.append(change) return past_changes def hold(change): name = change.name cache[name] = compress(cache.get(name), change) try: # Replace notify_change with `hold`, caching and compressing # notifications, disable cross validation and yield. self.notify_change = hold self._cross_validation_lock = True yield # Cross validate final values when context is released. for name in list(cache.keys()): trait = getattr(self.__class__, name) value = trait._cross_validate(self, getattr(self, name)) self.set_trait(name, value) except TraitError as e: # Roll back in case of TraitError during final cross validation. self.notify_change = lambda x: None for name, changes in cache.items(): for change in changes[::-1]: # TODO: Separate in a rollback function per notification type. if change.type == 'change': if change.old is not Undefined: self.set_trait(name, change.old) else: self._trait_values.pop(name) cache = {} raise e finally: self._cross_validation_lock = False # Restore method retrieval from class del self.notify_change # trigger delayed notifications for changes in cache.values(): for change in changes: self.notify_change(change) def _notify_trait(self, name, old_value, new_value): self.notify_change(Bunch( name=name, old=old_value, new=new_value, owner=self, type='change', )) def notify_change(self, change): if not isinstance(change, Bunch): # cast to bunch if given a dict change = Bunch(change) name, type = change.name, change.type callables = [] callables.extend(self._trait_notifiers.get(name, {}).get(type, [])) callables.extend(self._trait_notifiers.get(name, {}).get(All, [])) callables.extend(self._trait_notifiers.get(All, {}).get(type, [])) callables.extend(self._trait_notifiers.get(All, {}).get(All, [])) # Now static ones magic_name = '_%s_changed' % name if hasattr(self, magic_name): class_value = getattr(self.__class__, magic_name) if not isinstance(class_value, ObserveHandler): _deprecated_method(class_value, self.__class__, magic_name, "use @observe and @unobserve instead.") cb = getattr(self, magic_name) # Only append the magic method if it was not manually registered if cb not in callables: callables.append(_callback_wrapper(cb)) # Call them all now # Traits catches and logs errors here. I allow them to raise for c in callables: # Bound methods have an additional 'self' argument. if isinstance(c, _CallbackWrapper): c = c.__call__ elif isinstance(c, EventHandler) and c.name is not None: c = getattr(self, c.name) c(change) def _add_notifiers(self, handler, name, type): if name not in self._trait_notifiers: nlist = [] self._trait_notifiers[name] = {type: nlist} else: if type not in self._trait_notifiers[name]: nlist = [] self._trait_notifiers[name][type] = nlist else: nlist = self._trait_notifiers[name][type] if handler not in nlist: nlist.append(handler) def _remove_notifiers(self, handler, name, type): try: if handler is None: del self._trait_notifiers[name][type] else: self._trait_notifiers[name][type].remove(handler) except KeyError: pass def on_trait_change(self, handler=None, name=None, remove=False): """DEPRECATED: Setup a handler to be called when a trait changes. This is used to setup dynamic notifications of trait changes. Static handlers can be created by creating methods on a HasTraits subclass with the naming convention '_[traitname]_changed'. Thus, to create static handler for the trait 'a', create the method _a_changed(self, name, old, new) (fewer arguments can be used, see below). If `remove` is True and `handler` is not specified, all change handlers for the specified name are uninstalled. Parameters ---------- handler : callable, None A callable that is called when a trait changes. Its signature can be handler(), handler(name), handler(name, new), handler(name, old, new), or handler(name, old, new, self). name : list, str, None If None, the handler will apply to all traits. If a list of str, handler will apply to all names in the list. If a str, the handler will apply just to that name. remove : bool If False (the default), then install the handler. If True then unintall it. """ warn("on_trait_change is deprecated in traitlets 4.1: use observe instead", DeprecationWarning, stacklevel=2) if name is None: name = All if remove: self.unobserve(_callback_wrapper(handler), names=name) else: self.observe(_callback_wrapper(handler), names=name) def observe(self, handler, names=All, type='change'): """Setup a handler to be called when a trait changes. This is used to setup dynamic notifications of trait changes. Parameters ---------- handler : callable A callable that is called when a trait changes. Its signature should be ``handler(change)``, where ``change`` is a dictionary. The change dictionary at least holds a 'type' key. * ``type``: the type of notification. Other keys may be passed depending on the value of 'type'. In the case where type is 'change', we also have the following keys: * ``owner`` : the HasTraits instance * ``old`` : the old value of the modified trait attribute * ``new`` : the new value of the modified trait attribute * ``name`` : the name of the modified trait attribute. names : list, str, All If names is All, the handler will apply to all traits. If a list of str, handler will apply to all names in the list. If a str, the handler will apply just to that name. type : str, All (default: 'change') The type of notification to filter by. If equal to All, then all notifications are passed to the observe handler. """ names = parse_notifier_name(names) for n in names: self._add_notifiers(handler, n, type) def unobserve(self, handler, names=All, type='change'): """Remove a trait change handler. This is used to unregister handlers to trait change notifications. Parameters ---------- handler : callable The callable called when a trait attribute changes. names : list, str, All (default: All) The names of the traits for which the specified handler should be uninstalled. If names is All, the specified handler is uninstalled from the list of notifiers corresponding to all changes. type : str or All (default: 'change') The type of notification to filter by. If All, the specified handler is uninstalled from the list of notifiers corresponding to all types. """ names = parse_notifier_name(names) for n in names: self._remove_notifiers(handler, n, type) def unobserve_all(self, name=All): """Remove trait change handlers of any type for the specified name. If name is not specified, removes all trait notifiers.""" if name is All: self._trait_notifiers = {} else: try: del self._trait_notifiers[name] except KeyError: pass def _register_validator(self, handler, names): """Setup a handler to be called when a trait should be cross validated. This is used to setup dynamic notifications for cross-validation. If a validator is already registered for any of the provided names, a TraitError is raised and no new validator is registered. Parameters ---------- handler : callable A callable that is called when the given trait is cross-validated. Its signature is handler(proposal), where proposal is a Bunch (dictionary with attribute access) with the following attributes/keys: * ``owner`` : the HasTraits instance * ``value`` : the proposed value for the modified trait attribute * ``trait`` : the TraitType instance associated with the attribute names : List of strings The names of the traits that should be cross-validated """ for name in names: magic_name = '_%s_validate' % name if hasattr(self, magic_name): class_value = getattr(self.__class__, magic_name) if not isinstance(class_value, ValidateHandler): _deprecated_method(class_value, self.__class, magic_name, "use @validate decorator instead.") for name in names: self._trait_validators[name] = handler def add_traits(self, **traits): """Dynamically add trait attributes to the HasTraits instance.""" self.__class__ = type(self.__class__.__name__, (self.__class__,), traits) for trait in traits.values(): trait.instance_init(self) def set_trait(self, name, value): """Forcibly sets trait attribute, including read-only attributes.""" cls = self.__class__ if not self.has_trait(name): raise TraitError("Class %s does not have a trait named %s" % (cls.__name__, name)) else: getattr(cls, name).set(self, value) @classmethod def class_trait_names(cls, **metadata): """Get a list of all the names of this class' traits. This method is just like the :meth:`trait_names` method, but is unbound. """ return list(cls.class_traits(**metadata)) @classmethod def class_traits(cls, **metadata): """Get a ``dict`` of all the traits of this class. The dictionary is keyed on the name and the values are the TraitType objects. This method is just like the :meth:`traits` method, but is unbound. The TraitTypes returned don't know anything about the values that the various HasTrait's instances are holding. The metadata kwargs allow functions to be passed in which filter traits based on metadata values. The functions should take a single value as an argument and return a boolean. If any function returns False, then the trait is not included in the output. If a metadata key doesn't exist, None will be passed to the function. """ traits = dict([memb for memb in getmembers(cls) if isinstance(memb[1], TraitType)]) if len(metadata) == 0: return traits result = {} for name, trait in traits.items(): for meta_name, meta_eval in metadata.items(): if type(meta_eval) is not types.FunctionType: meta_eval = _SimpleTest(meta_eval) if not meta_eval(trait.metadata.get(meta_name, None)): break else: result[name] = trait return result @classmethod def class_own_traits(cls, **metadata): """Get a dict of all the traitlets defined on this class, not a parent. Works like `class_traits`, except for excluding traits from parents. """ sup = super(cls, cls) return {n: t for (n, t) in cls.class_traits(**metadata).items() if getattr(sup, n, None) is not t} def has_trait(self, name): """Returns True if the object has a trait with the specified name.""" return isinstance(getattr(self.__class__, name, None), TraitType) def trait_names(self, **metadata): """Get a list of all the names of this class' traits.""" return list(self.traits(**metadata)) def traits(self, **metadata): """Get a ``dict`` of all the traits of this class. The dictionary is keyed on the name and the values are the TraitType objects. The TraitTypes returned don't know anything about the values that the various HasTrait's instances are holding. The metadata kwargs allow functions to be passed in which filter traits based on metadata values. The functions should take a single value as an argument and return a boolean. If any function returns False, then the trait is not included in the output. If a metadata key doesn't exist, None will be passed to the function. """ traits = dict([memb for memb in getmembers(self.__class__) if isinstance(memb[1], TraitType)]) if len(metadata) == 0: return traits result = {} for name, trait in traits.items(): for meta_name, meta_eval in metadata.items(): if type(meta_eval) is not types.FunctionType: meta_eval = _SimpleTest(meta_eval) if not meta_eval(trait.metadata.get(meta_name, None)): break else: result[name] = trait return result def trait_metadata(self, traitname, key, default=None): """Get metadata values for trait by key.""" try: trait = getattr(self.__class__, traitname) except AttributeError: raise TraitError("Class %s does not have a trait named %s" % (self.__class__.__name__, traitname)) metadata_name = '_' + traitname + '_metadata' if hasattr(self, metadata_name) and key in getattr(self, metadata_name): return getattr(self, metadata_name).get(key, default) else: return trait.metadata.get(key, default) @classmethod def class_own_trait_events(cls, name): """Get a dict of all event handlers defined on this class, not a parent. Works like ``event_handlers``, except for excluding traits from parents. """ sup = super(cls, cls) return {n: e for (n, e) in cls.events(name).items() if getattr(sup, n, None) is not e} @classmethod def trait_events(cls, name=None): """Get a ``dict`` of all the event handlers of this class. Parameters ---------- name: str (default: None) The name of a trait of this class. If name is ``None`` then all the event handlers of this class will be returned instead. Returns ------- The event handlers associated with a trait name, or all event handlers. """ events = {} for k, v in getmembers(cls): if isinstance(v, EventHandler): if name is None: events[k] = v elif name in v.trait_names: events[k] = v elif hasattr(v, 'tags'): if cls.trait_names(**v.tags): events[k] = v return events #----------------------------------------------------------------------------- # Actual TraitTypes implementations/subclasses #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- # TraitTypes subclasses for handling classes and instances of classes #----------------------------------------------------------------------------- class ClassBasedTraitType(TraitType): """ A trait with error reporting and string -> type resolution for Type, Instance and This. """ def _resolve_string(self, string): """ Resolve a string supplied for a type into an actual object. """ return import_item(string) def error(self, obj, value): kind = type(value) if six.PY2 and kind is InstanceType: msg = 'class %s' % value.__class__.__name__ else: msg = '%s (i.e. %s)' % ( str( kind )[1:-1], repr( value ) ) if obj is not None: e = "The '%s' trait of %s instance must be %s, but a value of %s was specified." \ % (self.name, class_of(obj), self.info(), msg) else: e = "The '%s' trait must be %s, but a value of %r was specified." \ % (self.name, self.info(), msg) raise TraitError(e) class Type(ClassBasedTraitType): """A trait whose value must be a subclass of a specified class.""" def __init__ (self, default_value=Undefined, klass=None, **kwargs): """Construct a Type trait A Type trait specifies that its values must be subclasses of a particular class. If only ``default_value`` is given, it is used for the ``klass`` as well. If neither are given, both default to ``object``. Parameters ---------- default_value : class, str or None The default value must be a subclass of klass. If an str, the str must be a fully specified class name, like 'foo.bar.Bah'. The string is resolved into real class, when the parent :class:`HasTraits` class is instantiated. klass : class, str [ default object ] Values of this trait must be a subclass of klass. The klass may be specified in a string like: 'foo.bar.MyClass'. The string is resolved into real class, when the parent :class:`HasTraits` class is instantiated. allow_none : bool [ default False ] Indicates whether None is allowed as an assignable value. """ if default_value is Undefined: new_default_value = object if (klass is None) else klass else: new_default_value = default_value if klass is None: if (default_value is None) or (default_value is Undefined): klass = object else: klass = default_value if not (inspect.isclass(klass) or isinstance(klass, six.string_types)): raise TraitError("A Type trait must specify a class.") self.klass = klass super(Type, self).__init__(new_default_value, **kwargs) def validate(self, obj, value): """Validates that the value is a valid object instance.""" if isinstance(value, six.string_types): try: value = self._resolve_string(value) except ImportError: raise TraitError("The '%s' trait of %s instance must be a type, but " "%r could not be imported" % (self.name, obj, value)) try: if issubclass(value, self.klass): return value except: pass self.error(obj, value) def info(self): """ Returns a description of the trait.""" if isinstance(self.klass, six.string_types): klass = self.klass else: klass = self.klass.__module__ + '.' + self.klass.__name__ result = "a subclass of '%s'" % klass if self.allow_none: return result + ' or None' return result def instance_init(self, obj): self._resolve_classes() super(Type, self).instance_init(obj) def _resolve_classes(self): if isinstance(self.klass, six.string_types): self.klass = self._resolve_string(self.klass) if isinstance(self.default_value, six.string_types): self.default_value = self._resolve_string(self.default_value) def default_value_repr(self): value = self.default_value if isinstance(value, six.string_types): return repr(value) else: return repr('{}.{}'.format(value.__module__, value.__name__)) class Instance(ClassBasedTraitType): """A trait whose value must be an instance of a specified class. The value can also be an instance of a subclass of the specified class. Subclasses can declare default classes by overriding the klass attribute """ klass = None def __init__(self, klass=None, args=None, kw=None, **kwargs): """Construct an Instance trait. This trait allows values that are instances of a particular class or its subclasses. Our implementation is quite different from that of enthough.traits as we don't allow instances to be used for klass and we handle the ``args`` and ``kw`` arguments differently. Parameters ---------- klass : class, str The class that forms the basis for the trait. Class names can also be specified as strings, like 'foo.bar.Bar'. args : tuple Positional arguments for generating the default value. kw : dict Keyword arguments for generating the default value. allow_none : bool [ default False ] Indicates whether None is allowed as a value. Notes ----- If both ``args`` and ``kw`` are None, then the default value is None. If ``args`` is a tuple and ``kw`` is a dict, then the default is created as ``klass(*args, **kw)``. If exactly one of ``args`` or ``kw`` is None, the None is replaced by ``()`` or ``{}``, respectively. """ if klass is None: klass = self.klass if (klass is not None) and (inspect.isclass(klass) or isinstance(klass, six.string_types)): self.klass = klass else: raise TraitError('The klass attribute must be a class' ' not: %r' % klass) if (kw is not None) and not isinstance(kw, dict): raise TraitError("The 'kw' argument must be a dict or None.") if (args is not None) and not isinstance(args, tuple): raise TraitError("The 'args' argument must be a tuple or None.") self.default_args = args self.default_kwargs = kw super(Instance, self).__init__(**kwargs) def validate(self, obj, value): if isinstance(value, self.klass): return value else: self.error(obj, value) def info(self): if isinstance(self.klass, six.string_types): klass = self.klass else: klass = self.klass.__name__ result = class_of(klass) if self.allow_none: return result + ' or None' return result def instance_init(self, obj): self._resolve_classes() super(Instance, self).instance_init(obj) def _resolve_classes(self): if isinstance(self.klass, six.string_types): self.klass = self._resolve_string(self.klass) def make_dynamic_default(self): if (self.default_args is None) and (self.default_kwargs is None): return None return self.klass(*(self.default_args or ()), **(self.default_kwargs or {})) def default_value_repr(self): return repr(self.make_dynamic_default()) class ForwardDeclaredMixin(object): """ Mixin for forward-declared versions of Instance and Type. """ def _resolve_string(self, string): """ Find the specified class name by looking for it in the module in which our this_class attribute was defined. """ modname = self.this_class.__module__ return import_item('.'.join([modname, string])) class ForwardDeclaredType(ForwardDeclaredMixin, Type): """ Forward-declared version of Type. """ pass class ForwardDeclaredInstance(ForwardDeclaredMixin, Instance): """ Forward-declared version of Instance. """ pass class This(ClassBasedTraitType): """A trait for instances of the class containing this trait. Because how how and when class bodies are executed, the ``This`` trait can only have a default value of None. This, and because we always validate default values, ``allow_none`` is *always* true. """ info_text = 'an instance of the same type as the receiver or None' def __init__(self, **kwargs): super(This, self).__init__(None, **kwargs) def validate(self, obj, value): # What if value is a superclass of obj.__class__? This is # complicated if it was the superclass that defined the This # trait. if isinstance(value, self.this_class) or (value is None): return value else: self.error(obj, value) class Union(TraitType): """A trait type representing a Union type.""" def __init__(self, trait_types, **kwargs): """Construct a Union trait. This trait allows values that are allowed by at least one of the specified trait types. A Union traitlet cannot have metadata on its own, besides the metadata of the listed types. Parameters ---------- trait_types: sequence The list of trait types of length at least 1. Notes ----- Union([Float(), Bool(), Int()]) attempts to validate the provided values with the validation function of Float, then Bool, and finally Int. """ self.trait_types = trait_types self.info_text = " or ".join([tt.info() for tt in self.trait_types]) super(Union, self).__init__(**kwargs) def class_init(self, cls, name): for trait_type in self.trait_types: trait_type.class_init(cls, None) super(Union, self).class_init(cls, name) def instance_init(self, obj): for trait_type in self.trait_types: trait_type.instance_init(obj) super(Union, self).instance_init(obj) def validate(self, obj, value): with obj.cross_validation_lock: for trait_type in self.trait_types: try: v = trait_type._validate(obj, value) # In the case of an element trait, the name is None if self.name is not None: setattr(obj, '_' + self.name + '_metadata', trait_type.metadata) return v except TraitError: continue self.error(obj, value) def __or__(self, other): if isinstance(other, Union): return Union(self.trait_types + other.trait_types) else: return Union(self.trait_types + [other]) def make_dynamic_default(self): if self.default_value is not Undefined: return self.default_value for trait_type in self.trait_types: if trait_type.default_value is not Undefined: return trait_type.default_value elif hasattr(trait_type, 'make_dynamic_default'): return trait_type.make_dynamic_default() #----------------------------------------------------------------------------- # Basic TraitTypes implementations/subclasses #----------------------------------------------------------------------------- class Any(TraitType): """A trait which allows any value.""" default_value = None info_text = 'any value' def _validate_bounds(trait, obj, value): """ Validate that a number to be applied to a trait is between bounds. If value is not between min_bound and max_bound, this raises a TraitError with an error message appropriate for this trait. """ if trait.min is not None and value < trait.min: raise TraitError( "The value of the '{name}' trait of {klass} instance should " "not be less than {min_bound}, but a value of {value} was " "specified".format( name=trait.name, klass=class_of(obj), value=value, min_bound=trait.min)) if trait.max is not None and value > trait.max: raise TraitError( "The value of the '{name}' trait of {klass} instance should " "not be greater than {max_bound}, but a value of {value} was " "specified".format( name=trait.name, klass=class_of(obj), value=value, max_bound=trait.max)) return value class Int(TraitType): """An int trait.""" default_value = 0 info_text = 'an int' def __init__(self, default_value=Undefined, allow_none=False, **kwargs): self.min = kwargs.pop('min', None) self.max = kwargs.pop('max', None) super(Int, self).__init__(default_value=default_value, allow_none=allow_none, **kwargs) def validate(self, obj, value): if not isinstance(value, int): self.error(obj, value) return _validate_bounds(self, obj, value) class CInt(Int): """A casting version of the int trait.""" def validate(self, obj, value): try: value = int(value) except: self.error(obj, value) return _validate_bounds(self, obj, value) if six.PY2: class Long(TraitType): """A long integer trait.""" default_value = 0 info_text = 'a long' def __init__(self, default_value=Undefined, allow_none=False, **kwargs): self.min = kwargs.pop('min', None) self.max = kwargs.pop('max', None) super(Long, self).__init__( default_value=default_value, allow_none=allow_none, **kwargs) def _validate_long(self, obj, value): if isinstance(value, long): return value if isinstance(value, int): return long(value) self.error(obj, value) def validate(self, obj, value): value = self._validate_long(obj, value) return _validate_bounds(self, obj, value) class CLong(Long): """A casting version of the long integer trait.""" def validate(self, obj, value): try: value = long(value) except: self.error(obj, value) return _validate_bounds(self, obj, value) class Integer(TraitType): """An integer trait. Longs that are unnecessary (<= sys.maxint) are cast to ints.""" default_value = 0 info_text = 'an integer' def __init__(self, default_value=Undefined, allow_none=False, **kwargs): self.min = kwargs.pop('min', None) self.max = kwargs.pop('max', None) super(Integer, self).__init__( default_value=default_value, allow_none=allow_none, **kwargs) def _validate_int(self, obj, value): if isinstance(value, int): return value if isinstance(value, long): # downcast longs that fit in int: # note that int(n > sys.maxint) returns a long, so # we don't need a condition on this cast return int(value) if sys.platform == "cli": from System import Int64 if isinstance(value, Int64): return int(value) self.error(obj, value) def validate(self, obj, value): value = self._validate_int(obj, value) return _validate_bounds(self, obj, value) else: Long, CLong = Int, CInt Integer = Int class Float(TraitType): """A float trait.""" default_value = 0.0 info_text = 'a float' def __init__(self, default_value=Undefined, allow_none=False, **kwargs): self.min = kwargs.pop('min', -float('inf')) self.max = kwargs.pop('max', float('inf')) super(Float, self).__init__(default_value=default_value, allow_none=allow_none, **kwargs) def validate(self, obj, value): if isinstance(value, int): value = float(value) if not isinstance(value, float): self.error(obj, value) return _validate_bounds(self, obj, value) class CFloat(Float): """A casting version of the float trait.""" def validate(self, obj, value): try: value = float(value) except: self.error(obj, value) return _validate_bounds(self, obj, value) class Complex(TraitType): """A trait for complex numbers.""" default_value = 0.0 + 0.0j info_text = 'a complex number' def validate(self, obj, value): if isinstance(value, complex): return value if isinstance(value, (float, int)): return complex(value) self.error(obj, value) class CComplex(Complex): """A casting version of the complex number trait.""" def validate (self, obj, value): try: return complex(value) except: self.error(obj, value) # We should always be explicit about whether we're using bytes or unicode, both # for Python 3 conversion and for reliable unicode behaviour on Python 2. So # we don't have a Str type. class Bytes(TraitType): """A trait for byte strings.""" default_value = b'' info_text = 'a bytes object' def validate(self, obj, value): if isinstance(value, bytes): return value self.error(obj, value) class CBytes(Bytes): """A casting version of the byte string trait.""" def validate(self, obj, value): try: return bytes(value) except: self.error(obj, value) class Unicode(TraitType): """A trait for unicode strings.""" default_value = u'' info_text = 'a unicode string' def validate(self, obj, value): if isinstance(value, six.text_type): return value if isinstance(value, bytes): try: return value.decode('ascii', 'strict') except UnicodeDecodeError: msg = "Could not decode {!r} for unicode trait '{}' of {} instance." raise TraitError(msg.format(value, self.name, class_of(obj))) self.error(obj, value) class CUnicode(Unicode): """A casting version of the unicode trait.""" def validate(self, obj, value): try: return six.text_type(value) except: self.error(obj, value) class ObjectName(TraitType): """A string holding a valid object name in this version of Python. This does not check that the name exists in any scope.""" info_text = "a valid object identifier in Python" if six.PY2: # Python 2: def coerce_str(self, obj, value): "In Python 2, coerce ascii-only unicode to str" if isinstance(value, unicode): try: return str(value) except UnicodeEncodeError: self.error(obj, value) return value else: coerce_str = staticmethod(lambda _,s: s) def validate(self, obj, value): value = self.coerce_str(obj, value) if isinstance(value, six.string_types) and isidentifier(value): return value self.error(obj, value) class DottedObjectName(ObjectName): """A string holding a valid dotted object name in Python, such as A.b3._c""" def validate(self, obj, value): value = self.coerce_str(obj, value) if isinstance(value, six.string_types) and all(isidentifier(a) for a in value.split('.')): return value self.error(obj, value) class Bool(TraitType): """A boolean (True, False) trait.""" default_value = False info_text = 'a boolean' def validate(self, obj, value): if isinstance(value, bool): return value self.error(obj, value) class CBool(Bool): """A casting version of the boolean trait.""" def validate(self, obj, value): try: return bool(value) except: self.error(obj, value) class Enum(TraitType): """An enum whose value must be in a given sequence.""" def __init__(self, values, default_value=Undefined, **kwargs): self.values = values if kwargs.get('allow_none', False) and default_value is Undefined: default_value = None super(Enum, self).__init__(default_value, **kwargs) def validate(self, obj, value): if value in self.values: return value self.error(obj, value) def info(self): """ Returns a description of the trait.""" result = 'any of ' + repr(self.values) if self.allow_none: return result + ' or None' return result class CaselessStrEnum(Enum): """An enum of strings where the case should be ignored.""" def __init__(self, values, default_value=Undefined, **kwargs): values = [cast_unicode_py2(value) for value in values] super(CaselessStrEnum, self).__init__(values, default_value=default_value, **kwargs) def validate(self, obj, value): if isinstance(value, str): value = cast_unicode_py2(value) if not isinstance(value, six.string_types): self.error(obj, value) for v in self.values: if v.lower() == value.lower(): return v self.error(obj, value) class Container(Instance): """An instance of a container (list, set, etc.) To be subclassed by overriding klass. """ klass = None _cast_types = () _valid_defaults = SequenceTypes _trait = None def __init__(self, trait=None, default_value=None, **kwargs): """Create a container trait type from a list, set, or tuple. The default value is created by doing ``List(default_value)``, which creates a copy of the ``default_value``. ``trait`` can be specified, which restricts the type of elements in the container to that TraitType. If only one arg is given and it is not a Trait, it is taken as ``default_value``: ``c = List([1, 2, 3])`` Parameters ---------- trait : TraitType [ optional ] the type for restricting the contents of the Container. If unspecified, types are not checked. default_value : SequenceType [ optional ] The default value for the Trait. Must be list/tuple/set, and will be cast to the container type. allow_none : bool [ default False ] Whether to allow the value to be None **kwargs : any further keys for extensions to the Trait (e.g. config) """ # allow List([values]): if default_value is None and not is_trait(trait): default_value = trait trait = None if default_value is None: args = () elif isinstance(default_value, self._valid_defaults): args = (default_value,) else: raise TypeError('default value of %s was %s' %(self.__class__.__name__, default_value)) if is_trait(trait): if isinstance(trait, type): warn("Traits should be given as instances, not types (for example, `Int()`, not `Int`)." " Passing types is deprecated in traitlets 4.1.", DeprecationWarning, stacklevel=3) self._trait = trait() if isinstance(trait, type) else trait elif trait is not None: raise TypeError("`trait` must be a Trait or None, got %s" % repr_type(trait)) super(Container,self).__init__(klass=self.klass, args=args, **kwargs) def element_error(self, obj, element, validator): e = "Element of the '%s' trait of %s instance must be %s, but a value of %s was specified." \ % (self.name, class_of(obj), validator.info(), repr_type(element)) raise TraitError(e) def validate(self, obj, value): if isinstance(value, self._cast_types): value = self.klass(value) value = super(Container, self).validate(obj, value) if value is None: return value value = self.validate_elements(obj, value) return value def validate_elements(self, obj, value): validated = [] if self._trait is None or isinstance(self._trait, Any): return value for v in value: try: v = self._trait._validate(obj, v) except TraitError: self.element_error(obj, v, self._trait) else: validated.append(v) return self.klass(validated) def class_init(self, cls, name): if isinstance(self._trait, TraitType): self._trait.class_init(cls, None) super(Container, self).class_init(cls, name) def instance_init(self, obj): if isinstance(self._trait, TraitType): self._trait.instance_init(obj) super(Container, self).instance_init(obj) class List(Container): """An instance of a Python list.""" klass = list _cast_types = (tuple,) def __init__(self, trait=None, default_value=None, minlen=0, maxlen=sys.maxsize, **kwargs): """Create a List trait type from a list, set, or tuple. The default value is created by doing ``list(default_value)``, which creates a copy of the ``default_value``. ``trait`` can be specified, which restricts the type of elements in the container to that TraitType. If only one arg is given and it is not a Trait, it is taken as ``default_value``: ``c = List([1, 2, 3])`` Parameters ---------- trait : TraitType [ optional ] the type for restricting the contents of the Container. If unspecified, types are not checked. default_value : SequenceType [ optional ] The default value for the Trait. Must be list/tuple/set, and will be cast to the container type. minlen : Int [ default 0 ] The minimum length of the input list maxlen : Int [ default sys.maxsize ] The maximum length of the input list """ self._minlen = minlen self._maxlen = maxlen super(List, self).__init__(trait=trait, default_value=default_value, **kwargs) def length_error(self, obj, value): e = "The '%s' trait of %s instance must be of length %i <= L <= %i, but a value of %s was specified." \ % (self.name, class_of(obj), self._minlen, self._maxlen, value) raise TraitError(e) def validate_elements(self, obj, value): length = len(value) if length < self._minlen or length > self._maxlen: self.length_error(obj, value) return super(List, self).validate_elements(obj, value) def validate(self, obj, value): value = super(List, self).validate(obj, value) value = self.validate_elements(obj, value) return value class Set(List): """An instance of a Python set.""" klass = set _cast_types = (tuple, list) # Redefine __init__ just to make the docstring more accurate. def __init__(self, trait=None, default_value=None, minlen=0, maxlen=sys.maxsize, **kwargs): """Create a Set trait type from a list, set, or tuple. The default value is created by doing ``set(default_value)``, which creates a copy of the ``default_value``. ``trait`` can be specified, which restricts the type of elements in the container to that TraitType. If only one arg is given and it is not a Trait, it is taken as ``default_value``: ``c = Set({1, 2, 3})`` Parameters ---------- trait : TraitType [ optional ] the type for restricting the contents of the Container. If unspecified, types are not checked. default_value : SequenceType [ optional ] The default value for the Trait. Must be list/tuple/set, and will be cast to the container type. minlen : Int [ default 0 ] The minimum length of the input list maxlen : Int [ default sys.maxsize ] The maximum length of the input list """ super(Set, self).__init__(trait, default_value, minlen, maxlen, **kwargs) class Tuple(Container): """An instance of a Python tuple.""" klass = tuple _cast_types = (list,) def __init__(self, *traits, **kwargs): """Create a tuple from a list, set, or tuple. Create a fixed-type tuple with Traits: ``t = Tuple(Int(), Str(), CStr())`` would be length 3, with Int,Str,CStr for each element. If only one arg is given and it is not a Trait, it is taken as default_value: ``t = Tuple((1, 2, 3))`` Otherwise, ``default_value`` *must* be specified by keyword. Parameters ---------- `*traits` : TraitTypes [ optional ] the types for restricting the contents of the Tuple. If unspecified, types are not checked. If specified, then each positional argument corresponds to an element of the tuple. Tuples defined with traits are of fixed length. default_value : SequenceType [ optional ] The default value for the Tuple. Must be list/tuple/set, and will be cast to a tuple. If ``traits`` are specified, ``default_value`` must conform to the shape and type they specify. """ default_value = kwargs.pop('default_value', Undefined) # allow Tuple((values,)): if len(traits) == 1 and default_value is Undefined and not is_trait(traits[0]): default_value = traits[0] traits = () if default_value is Undefined: args = () elif isinstance(default_value, self._valid_defaults): args = (default_value,) else: raise TypeError('default value of %s was %s' %(self.__class__.__name__, default_value)) self._traits = [] for trait in traits: if isinstance(trait, type): warn("Traits should be given as instances, not types (for example, `Int()`, not `Int`)" " Passing types is deprecated in traitlets 4.1.", DeprecationWarning, stacklevel=2) t = trait() if isinstance(trait, type) else trait self._traits.append(t) if self._traits and default_value is None: # don't allow default to be an empty container if length is specified args = None super(Container,self).__init__(klass=self.klass, args=args, **kwargs) def validate_elements(self, obj, value): if not self._traits: # nothing to validate return value if len(value) != len(self._traits): e = "The '%s' trait of %s instance requires %i elements, but a value of %s was specified." \ % (self.name, class_of(obj), len(self._traits), repr_type(value)) raise TraitError(e) validated = [] for t, v in zip(self._traits, value): try: v = t._validate(obj, v) except TraitError: self.element_error(obj, v, t) else: validated.append(v) return tuple(validated) def class_init(self, cls, name): for trait in self._traits: if isinstance(trait, TraitType): trait.class_init(cls, None) super(Container, self).class_init(cls, name) def instance_init(self, obj): for trait in self._traits: if isinstance(trait, TraitType): trait.instance_init(obj) super(Container, self).instance_init(obj) class Dict(Instance): """An instance of a Python dict.""" _trait = None def __init__(self, trait=None, traits=None, default_value=Undefined, **kwargs): """Create a dict trait type from a Python dict. The default value is created by doing ``dict(default_value)``, which creates a copy of the ``default_value``. Parameters ---------- trait : TraitType [ optional ] The specified trait type to check and use to restrict contents of the Container. If unspecified, trait types are not checked. traits : Dictionary of trait types [ optional ] A Python dictionary containing the types that are valid for restricting the content of the Dict Container for certain keys. default_value : SequenceType [ optional ] The default value for the Dict. Must be dict, tuple, or None, and will be cast to a dict if not None. If `trait` is specified, the `default_value` must conform to the constraints it specifies. """ # Handling positional arguments if default_value is Undefined and trait is not None: if not is_trait(trait): default_value = trait trait = None # Handling default value if default_value is Undefined: default_value = {} if default_value is None: args = None elif isinstance(default_value, dict): args = (default_value,) elif isinstance(default_value, SequenceTypes): args = (default_value,) else: raise TypeError('default value of Dict was %s' % default_value) # Case where a type of TraitType is provided rather than an instance if is_trait(trait): if isinstance(trait, type): warn("Traits should be given as instances, not types (for example, `Int()`, not `Int`)" " Passing types is deprecated in traitlets 4.1.", DeprecationWarning, stacklevel=2) self._trait = trait() if isinstance(trait, type) else trait elif trait is not None: raise TypeError("`trait` must be a Trait or None, got %s" % repr_type(trait)) self._traits = traits super(Dict, self).__init__(klass=dict, args=args, **kwargs) def element_error(self, obj, element, validator): e = "Element of the '%s' trait of %s instance must be %s, but a value of %s was specified." \ % (self.name, class_of(obj), validator.info(), repr_type(element)) raise TraitError(e) def validate(self, obj, value): value = super(Dict, self).validate(obj, value) if value is None: return value value = self.validate_elements(obj, value) return value def validate_elements(self, obj, value): use_dict = bool(self._traits) default_to = (self._trait or Any()) if not use_dict and isinstance(default_to, Any): return value validated = {} for key in value: if use_dict and key in self._traits: validate_with = self._traits[key] else: validate_with = default_to try: v = value[key] if not isinstance(validate_with, Any): v = validate_with._validate(obj, v) except TraitError: self.element_error(obj, v, validate_with) else: validated[key] = v return self.klass(validated) def class_init(self, cls, name): if isinstance(self._trait, TraitType): self._trait.class_init(cls, None) if self._traits is not None: for trait in self._traits.values(): trait.class_init(cls, None) super(Dict, self).class_init(cls, name) def instance_init(self, obj): if isinstance(self._trait, TraitType): self._trait.instance_init(obj) if self._traits is not None: for trait in self._traits.values(): trait.instance_init(obj) super(Dict, self).instance_init(obj) class TCPAddress(TraitType): """A trait for an (ip, port) tuple. This allows for both IPv4 IP addresses as well as hostnames. """ default_value = ('127.0.0.1', 0) info_text = 'an (ip, port) tuple' def validate(self, obj, value): if isinstance(value, tuple): if len(value) == 2: if isinstance(value[0], six.string_types) and isinstance(value[1], int): port = value[1] if port >= 0 and port <= 65535: return value self.error(obj, value) class CRegExp(TraitType): """A casting compiled regular expression trait. Accepts both strings and compiled regular expressions. The resulting attribute will be a compiled regular expression.""" info_text = 'a regular expression' def validate(self, obj, value): try: return re.compile(value) except: self.error(obj, value) class UseEnum(TraitType): """Use a Enum class as model for the data type description. Note that if no default-value is provided, the first enum-value is used as default-value. .. sourcecode:: python # -- SINCE: Python 3.4 (or install backport: pip install enum34) import enum from traitlets import HasTraits, UseEnum class Color(enum.Enum): red = 1 # -- IMPLICIT: default_value blue = 2 green = 3 class MyEntity(HasTraits): color = UseEnum(Color, default_value=Color.blue) entity = MyEntity(color=Color.red) entity.color = Color.green # USE: Enum-value (preferred) entity.color = "green" # USE: name (as string) entity.color = "Color.green" # USE: scoped-name (as string) entity.color = 3 # USE: number (as int) assert entity.color is Color.green """ default_value = None info_text = "Trait type adapter to a Enum class" def __init__(self, enum_class, default_value=None, **kwargs): assert issubclass(enum_class, enum.Enum), \ "REQUIRE: enum.Enum, but was: %r" % enum_class allow_none = kwargs.get("allow_none", False) if default_value is None and not allow_none: default_value = list(enum_class.__members__.values())[0] super(UseEnum, self).__init__(default_value=default_value, **kwargs) self.enum_class = enum_class self.name_prefix = enum_class.__name__ + "." def select_by_number(self, value, default=Undefined): """Selects enum-value by using its number-constant.""" assert isinstance(value, int) enum_members = self.enum_class.__members__ for enum_item in enum_members.values(): if enum_item.value == value: return enum_item # -- NOT FOUND: return default def select_by_name(self, value, default=Undefined): """Selects enum-value by using its name or scoped-name.""" assert isinstance(value, six.string_types) if value.startswith(self.name_prefix): # -- SUPPORT SCOPED-NAMES, like: "Color.red" => "red" value = value.replace(self.name_prefix, "", 1) return self.enum_class.__members__.get(value, default) def validate(self, obj, value): if isinstance(value, self.enum_class): return value elif isinstance(value, int): # -- CONVERT: number => enum_value (item) value2 = self.select_by_number(value) if value2 is not Undefined: return value2 elif isinstance(value, six.string_types): # -- CONVERT: name or scoped_name (as string) => enum_value (item) value2 = self.select_by_name(value) if value2 is not Undefined: return value2 elif value is None: if self.allow_none: return None else: return self.default_value self.error(obj, value) def info(self): """Returns a description of this Enum trait (in case of errors).""" result = "Any of: %s" % ", ".join(self.enum_class.__members__.keys()) if self.allow_none: return result + " or None" return result ================================================ FILE: lib/client/traitlets/utils/__init__.py ================================================ ================================================ FILE: lib/client/traitlets/utils/bunch.py ================================================ """Yet another implementation of bunch attribute-access of items on a dict. """ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. class Bunch(dict): """A dict with attribute-access""" def __getattr__(self, key): try: return self.__getitem__(key) except KeyError: raise AttributeError(key) def __setattr__(self, key, value): self.__setitem__(key, value) def __dir__(self): # py2-compat: can't use super because dict doesn't have __dir__ names = dir({}) names.extend(self.keys()) return names ================================================ FILE: lib/client/traitlets/utils/getargspec.py ================================================ # -*- coding: utf-8 -*- """ getargspec excerpted from: sphinx.util.inspect ~~~~~~~~~~~~~~~~~~~ Helpers for inspecting Python modules. :copyright: Copyright 2007-2015 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ import inspect from six import PY3 # Unmodified from sphinx below this line if PY3: from functools import partial def getargspec(func): """Like inspect.getargspec but supports functools.partial as well.""" if inspect.ismethod(func): func = func.__func__ if type(func) is partial: orig_func = func.func argspec = getargspec(orig_func) args = list(argspec[0]) defaults = list(argspec[3] or ()) kwoargs = list(argspec[4]) kwodefs = dict(argspec[5] or {}) if func.args: args = args[len(func.args):] for arg in func.keywords or (): try: i = args.index(arg) - len(args) del args[i] try: del defaults[i] except IndexError: pass except ValueError: # must be a kwonly arg i = kwoargs.index(arg) del kwoargs[i] del kwodefs[arg] return inspect.FullArgSpec(args, argspec[1], argspec[2], tuple(defaults), kwoargs, kwodefs, argspec[6]) while hasattr(func, '__wrapped__'): func = func.__wrapped__ if not inspect.isfunction(func): raise TypeError('%r is not a Python function' % func) return inspect.getfullargspec(func) else: # 2.6, 2.7 from functools import partial def getargspec(func): """Like inspect.getargspec but supports functools.partial as well.""" if inspect.ismethod(func): func = func.__func__ parts = 0, () if type(func) is partial: keywords = func.keywords if keywords is None: keywords = {} parts = len(func.args), keywords.keys() func = func.func if not inspect.isfunction(func): raise TypeError('%r is not a Python function' % func) args, varargs, varkw = inspect.getargs(func.__code__) func_defaults = func.__defaults__ if func_defaults is None: func_defaults = [] else: func_defaults = list(func_defaults) if parts[0]: args = args[parts[0]:] if parts[1]: for arg in parts[1]: i = args.index(arg) - len(args) del args[i] try: del func_defaults[i] except IndexError: pass return inspect.ArgSpec(args, varargs, varkw, func_defaults) ================================================ FILE: lib/client/traitlets/utils/importstring.py ================================================ # encoding: utf-8 """ A simple utility to import something by its string name. """ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. from ipython_genutils.py3compat import cast_bytes_py2 from six import string_types def import_item(name): """Import and return ``bar`` given the string ``foo.bar``. Calling ``bar = import_item("foo.bar")`` is the functional equivalent of executing the code ``from foo import bar``. Parameters ---------- name : string The fully qualified name of the module/package being imported. Returns ------- mod : module object The module that was imported. """ if not isinstance(name, string_types): raise TypeError("import_item accepts strings, not '%s'." % type(name)) name = cast_bytes_py2(name) parts = name.rsplit('.', 1) if len(parts) == 2: # called with 'foo.bar....' package, obj = parts module = __import__(package, fromlist=[obj]) try: pak = getattr(module, obj) except AttributeError: raise ImportError('No module named %s' % obj) return pak else: # called with un-dotted string return __import__(parts[0]) ================================================ FILE: lib/client/traitlets/utils/sentinel.py ================================================ """Sentinel class for constants with useful reprs""" # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. class Sentinel(object): def __init__(self, name, module, docstring=None): self.name = name self.module = module if docstring: self.__doc__ = docstring def __repr__(self): return str(self.module)+'.'+self.name ================================================ FILE: lib/client/traitlets/utils/tests/__init__.py ================================================ ================================================ FILE: lib/client/traitlets/utils/tests/test_bunch.py ================================================ from ..bunch import Bunch def test_bunch(): b = Bunch(x=5, y=10) assert 'y' in b assert 'x' in b assert b.x == 5 b['a'] = 'hi' assert b.a == 'hi' def test_bunch_dir(): b = Bunch(x=5, y=10) assert 'x' in dir(b) assert 'keys' in dir(b) ================================================ FILE: lib/client/traitlets/utils/tests/test_importstring.py ================================================ # encoding: utf-8 # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. # # Adapted from enthought.traits, Copyright (c) Enthought, Inc., # also under the terms of the Modified BSD License. """Tests for traitlets.utils.importstring.""" import os from unittest import TestCase from ..importstring import import_item class TestImportItem(TestCase): def test_import_unicode(self): self.assertIs(os, import_item(u'os')) self.assertIs(os.path, import_item(u'os.path')) self.assertIs(os.path.join, import_item(u'os.path.join')) def test_bad_input(self): class NotAString(object): pass msg = ( "import_item accepts strings, " "not '%s'." % NotAString ) with self.assertRaisesRegexp(TypeError, msg): import_item(NotAString()) ================================================ FILE: lib/client/update_decorator.sh ================================================ #!/bin/bash # This script updates decorator to another version. Version is either the git tag # or the commit has of the respective release # # Usage: # ./update_traitlets.sh 4.0.0 # cur_path=`pwd` red='\033[0;31m' noc='\033[0m' if [[ $1 == "" ]]; then echo -e "${red}ERROR" echo -e "${noc}No version spefified." echo "" echo "Exiting..." exit 1; fi rm -rf /tmp/decorator git clone https://github.com/micheles/decorator /tmp/decorator cd /tmp/decorator git checkout $1 cd $cur_path git_status=`git status -- decorator | tail -n 1` git_clean="nothing to commit, working tree clean" if [[ $git_status == $git_clean ]]; then rm -rf decorator/ mv /tmp/decorator/src/decorator.py . echo $1 > decorator_version else echo "" echo -e "${red}ERROR" echo "There are uncommitted changes or untracked files in './decorator.py'." echo "" echo -e "${noc}See 'git status -- decorator.py' for more details." fi rm -rf /tmp/decorator ================================================ FILE: lib/client/update_ipython_genutils.sh ================================================ #!/bin/bash # This script updates ipython_genutils to another version. Version is either the git tag # or the commit has of the respective release # # Usage: # ./update_ipython_genutils.sh 4.0.0 # cur_path=`pwd` red='\033[0;31m' noc='\033[0m' if [[ $1 == "" ]]; then echo -e "${red}ERROR" echo -e "${noc}No version spefified." echo "" echo "Exiting..." exit 1; fi rm -rf /tmp/ipython_genutils git clone https://github.com/ipython/ipython_genutils /tmp/ipython_genutils cd /tmp/ipython_genutils git checkout $1 cd $cur_path git_status=`git status -- ipython_genutils | tail -n 1` git_clean="nothing to commit, working tree clean" if [[ $git_status == $git_clean ]]; then rm -rf ipython_genutils/ mv /tmp/ipython_genutils/ipython_genutils . rm -rf ipython_genutils/testing rm -rf ipython_genutils/tests else echo "" echo -e "${red}ERROR" echo "There are uncommitted changes or untracked files in './ipython_genutils/'." echo "" echo -e "${noc}See 'git status -- ipython_genutils' for more details." fi rm -rf /tmp/ipython_genutils ================================================ FILE: lib/client/update_jupyter_client.sh ================================================ #!/bin/bash # This script updates jupyter_client to another version. Version is either the git tag # or the commit has of the respective release # # Usage: # ./update_jupyter_client.sh 4.0.0 # cur_path=`pwd` red='\033[0;31m' noc='\033[0m' if [[ $1 == "" ]]; then echo -e "${red}ERROR" echo -e "${noc}No version spefified." echo "" echo "Exiting..." exit 1; fi rm -rf /tmp/jupyter_client git clone https://github.com/jupyter/jupyter_client /tmp/jupyter_client cd /tmp/jupyter_client git checkout $1 cd $cur_path git_status=`git status -- jupyter_client | tail -n 1` git_clean="nothing to commit, working tree clean" if [[ $git_status == $git_clean ]]; then rm -rf jupyter_client/ mv /tmp/jupyter_client/jupyter_client . rm -rf jupyter_client/tests else echo "" echo -e "${red}ERROR" echo "There are uncommitted changes or untracked files in './jupyter_client/'." echo "" echo -e "${noc}See 'git status -- jupyter_client' for more details." fi rm -rf /tmp/jupyter_client ================================================ FILE: lib/client/update_jupyter_core.sh ================================================ #!/bin/bash # This script updates jupyter_core to another version. Version is either the git tag # or the commit has of the respective release # # Usage: # ./update_jupyter_core.sh 4.0.0 # cur_path=`pwd` red='\033[0;31m' noc='\033[0m' if [[ $1 == "" ]]; then echo -e "${red}ERROR" echo -e "${noc}No version spefified." echo "" echo "Exiting..." exit 1; fi rm -rf /tmp/jupyter_core git clone https://github.com/jupyter/jupyter_core /tmp/jupyter_core cd /tmp/jupyter_core git checkout $1 cd $cur_path git_status=`git status -- jupyter_core | tail -n 1` git_clean="nothing to commit, working tree clean" if [[ $git_status == $git_clean ]]; then rm -rf jupyter_core/ mv /tmp/jupyter_core/jupyter_core . rm -rf jupyter_core/tests else echo "" echo -e "${red}ERROR" echo "There are uncommitted changes or untracked files in './jupyter_core/'." echo "" echo -e "${noc}See 'git status -- jupyter_core' for more details." fi rm -rf /tmp/jupyter_core ================================================ FILE: lib/client/update_traitlets.sh ================================================ #!/bin/bash # This script updates traitlets to another version. Version is either the git tag # or the commit has of the respective release # # Usage: # ./update_traitlets.sh 4.0.0 # cur_path=`pwd` red='\033[0;31m' noc='\033[0m' if [[ $1 == "" ]]; then echo -e "${red}ERROR" echo -e "${noc}No version spefified." echo "" echo "Exiting..." exit 1; fi rm -rf /tmp/traitlets git clone https://github.com/ipython/traitlets /tmp/traitlets cd /tmp/traitlets git checkout $1 cd $cur_path git_status=`git status -- traitlets | tail -n 1` git_clean="nothing to commit, working tree clean" if [[ $git_status == $git_clean ]]; then rm -rf traitlets/ mv /tmp/traitlets/traitlets . rm -rf traitlets/tests rm -rf traitlets/config/tests else echo "" echo -e "${red}ERROR" echo "There are uncommitted changes or untracked files in './traitlets/'." echo "" echo -e "${noc}See 'git status -- traitlets' for more details." fi rm -rf /tmp/traitlets ================================================ FILE: lib/kernel.py ================================================ """Definition of KernelConnection class. KernelConnection class provides interaction with Jupyter kernels. Copyright (c) 2017-2018, NEGORO Tetsuya (https://github.com/ngr-t) """ import re from collections import defaultdict from datetime import datetime from queue import Empty, Queue from threading import Event, RLock, Thread import sublime from .utils import get_cell, get_png_dimensions, show_password_input JUPYTER_PROTOCOL_VERSION = "5.0" REPLY_STATUS_OK = "ok" REPLY_STATUS_ERROR = "error" REPLY_STATUS_ABORT = "abort" MSG_TYPE_EXECUTE_INPUT = "execute_input" MSG_TYPE_EXECUTE_REQUEST = "execute_request" MSG_TYPE_EXECUTE_RESULT = "execute_result" MSG_TYPE_EXECUTE_REPLY = "execute_reply" MSG_TYPE_COMPLETE_REQUEST = "complete_request" MSG_TYPE_COMPLETE_REPLY = "complete_reply" MSG_TYPE_DISPLAY_DATA = "display_data" MSG_TYPE_INSPECT_REQUEST = "inspect_request" MSG_TYPE_INSPECT_REPLY = "inspect_reply" MSG_TYPE_INPUT_REQUEST = "input_request" MSG_TYPE_INPUT_REPLY = "input_reply" MSG_TYPE_ERROR = "error" MSG_TYPE_STREAM = "stream" MSG_TYPE_STATUS = "status" HELIUM_FIGURE_PHANTOMS = "helium_figure_phantoms" MAX_PHANTOMS = 65536 # Used as key of status bar. KERNEL_STATUS_KEY = "helium_kernel_status" HELIUM_OBJECT_INSPECT_PANEL = "helium_object_inspect" ANSI_ESCAPE_PATTERN = re.compile(r"\x1b[^m]*m") OUTPUT_VIEW_SEPARATOR = "-" * 80 TEXT_PHANTOM = """ × {content} """ IMAGE_PHANTOM = """ ×
Out """ STREAM_PHANTOM = "
{content}
" def fix_whitespace_for_phantom(text: str): """Transform output for proper display. This is important to display pandas DataFrames, for instance. """ text = text.replace(" ", r" ") text = "
".join(text.splitlines()) return text def extract_content(messages, msg_type): """Extract content from messages received from a kernel.""" return [ message["content"] for message in messages if message["header"]["msg_type"] == msg_type ] def remove_ansi_escape(text: str): return ANSI_ESCAPE_PATTERN.sub("", text) def get_msg_type(message): return message["header"]["msg_type"] def extract_data(result): """Extract plain text data.""" try: return result["data"] except KeyError: return "" class KernelConnection(object): """Interact with a Jupyter kernel.""" class MessageReceiver(Thread): # noqa def __init__(self, kernel): """Initialize AsyncCommunicator class.""" super().__init__() self._kernel = kernel self.exit = Event() def shutdown(self): self.exit.set() class ShellMessageReceiver(MessageReceiver): """Communicator that runs asynchroniously.""" def run(self): """Run main routine.""" # TODO: implement logging # TODO: remove view and regions from id2region while not self.exit.is_set(): try: msg = self._kernel.client.get_shell_msg(timeout=1) self._kernel.shell_msg_queues_lock.acquire() try: queue = self._kernel.shell_msg_queues[ msg["parent_header"]["msg_id"] ] finally: self._kernel.shell_msg_queues_lock.release() queue.put(msg) except Empty: pass except Exception as ex: self._kernel._logger.exception(ex) class IOPubMessageReceiver(MessageReceiver): """Receive and process IOPub messages.""" def run(self): """Run main routine.""" # TODO: log, handle other message types. while not self.exit.is_set(): try: msg = self._kernel.client.get_iopub_msg(timeout=1) self._kernel._logger.info(msg) content = msg.get("content", {}) execution_count = content.get("execution_count", None) msg_type = msg["msg_type"] view, region = self._kernel.id2region.get( msg["parent_header"].get("msg_id", None), (None, None) ) if msg_type == MSG_TYPE_STATUS: self._kernel._execution_state = content["execution_state"] elif msg_type == MSG_TYPE_EXECUTE_INPUT: # if code is executed deleted all phantoms in this region self._kernel._clear_phantoms_in_region(region, view) self._kernel._write_text_to_view("\n\n") if sublime.load_settings("Helium.sublime-settings").get( "output_code" ): self._kernel._output_input_code( content["code"], content["execution_count"] ) elif msg_type == MSG_TYPE_ERROR: self._kernel._logger.info("Handling error") self._kernel._handle_error( content["ename"], content["evalue"], content["traceback"], region, view, ) elif msg_type == MSG_TYPE_DISPLAY_DATA: self._kernel._write_mime_data_to_view( content["data"], region, view ) elif msg_type == MSG_TYPE_EXECUTE_RESULT: self._kernel._write_mime_data_to_view( content["data"], region, view ) elif msg_type == MSG_TYPE_STREAM: self._kernel._handle_stream( content["name"], content["text"], region, view, ) except Empty: pass except Exception as ex: self._kernel._logger.exception(ex) class StdInMessageReceiver(MessageReceiver): """Receive and process IOPub messages.""" def _handle_input_request(self, prompt, password): def interrupt(): self._kernel.interrupt_kernel(self.kernel_id) if password: show_password_input(prompt, self._kernel.input, interrupt) else: ( sublime.active_window().show_input_panel( prompt, "", self._kernel.client.input, lambda x: None, interrupt ) ) def run(self): """Run main routine.""" # TODO: log, handle other message types. while not self.exit.is_set(): try: msg = self._kernel.client.get_stdin_msg(timeout=1) msg_type = msg["msg_type"] content = msg["content"] if msg_type == MSG_TYPE_INPUT_REQUEST: self._handle_input_request( content["prompt"], content["password"] ) except Empty: pass except Exception as ex: self._kernel._logger.exception(ex) def _init_receivers(self): # Set the attributes refered by receivers before they start. self._shell_msg_receiver = self.ShellMessageReceiver(self) self._shell_msg_receiver.start() self._iopub_msg_receiver = self.IOPubMessageReceiver(self) self._iopub_msg_receiver.start() self._stdin_msg_receiver = self.StdInMessageReceiver(self) self._stdin_msg_receiver.start() def __init__( self, kernel_id, kernel_manager, parent, connection_name=None, logger=None, ): """Initialize KernelConnection class. parameters --------- kernel_id str: kernel ID parent parent kernel manager """ self._logger = logger self.shell_msg_queues = defaultdict(Queue) self._kernel_id = kernel_id self.parent = parent self.kernel_manager = kernel_manager self.client = self.kernel_manager.client() self.client.start_channels() self.shell_msg_queues_lock = RLock() self.id2region = {} self._connection_name = connection_name self._execution_state = "unknown" self._init_receivers() self.phantoms = {} def __del__(self): # noqa self._shell_msg_receiver.shutdown() self._iopub_msg_receiver.shutdown() self._stdin_msg_receiver.shutdown() @property def lang(self): """Language of kernel.""" return self.kernel_manager.kernel_name @property def kernel_id(self): """ID of kernel.""" return self._kernel_id def shutdown_kernel(self): self.kernel_manager.shutdown_kernel() def restart_kernel(self): self.kernel_manager.restart_kernel() def interrupt_kernel(self): self.kernel_manager.interrupt_kernel() def get_connection_name(self): return self._connection_name def set_connection_name(self, new_name): # We also have to change the view name now. view = self.get_view() self._connection_name = new_name view.set_name(self.view_name) def del_connection_name(self): self._connection_name = None connection_name = property( get_connection_name, set_connection_name, del_connection_name, "Name of kernel connection shown in a view title.", ) @property def view_name(self): """Return name of output view.""" return "*Helium Output* {repr}".format(repr=self.repr) @property def repr(self): """Return string representation of the connection.""" if self.connection_name: return "{connection_name} ([{lang}] {kernel_id})".format( connection_name=self.connection_name, lang=self.lang, kernel_id=self.kernel_id, ) else: return "[{lang}] {kernel_id}".format( lang=self.lang, kernel_id=self.kernel_id ) @property def execution_state(self): return self._execution_state @property def _show_inline_output(self): return sublime.load_settings("Helium.sublime-settings").get("inline_output") def activate_view(self): """Activate view to show the output of kernel.""" view = self.get_view() current_view = sublime.active_window().active_view() sublime.active_window().focus_view(view) view.set_scratch(True) # avoids prompting to save view.settings().set("word_wrap", "false") sublime.active_window().focus_view(current_view) def _output_input_code(self, code, execution_count): line = "In[{execution_count}]: {code}".format( execution_count=execution_count, code=code ) self._write_text_to_view(line) def _handle_error( self, ename, evalue, traceback, region: sublime.Region = None, view: sublime.View = None, ) -> None: try: lines = """\nError: {ename}, {evalue}. \nTraceback:\n{traceback}""".format( ename=ename, evalue=evalue, traceback="\n".join(traceback), ) lines = remove_ansi_escape(lines) self._write_text_to_view(lines) if region is not None: phantom_html = STREAM_PHANTOM.format( name="error", content=fix_whitespace_for_phantom(lines) ) self._write_inline_html_phantom(phantom_html, region, view) except AttributeError: # Just there is no error. pass def _handle_stream( self, name, text, region: sublime.Region = None, view: sublime.View = None ) -> None: # Currently don't consider real time catching of streams. try: lines = "\n({name}):\n{text}".format(name=name, text=text) phantom_html = STREAM_PHANTOM.format( name=name, content=fix_whitespace_for_phantom(text) ) self._write_text_to_view(lines) if phantom_html and (region is not None): self._write_inline_html_phantom(phantom_html, region, view) except AttributeError: # Just there is no error. pass def _write_out_execution_count(self, execution_count) -> None: self._write_text_to_view("\nOut[{}]: \n".format(execution_count)) def _write_text_to_view(self, text: str) -> None: if self._show_inline_output: return self.activate_view() view = self.get_view() view.set_read_only(False) view.run_command("append", {"characters": text}) view.set_read_only(True) view.show(view.size()) def _write_phantom(self, content: str): if self._show_inline_output: return self.activate_view() file_size = self.get_view().size() region = sublime.Region(file_size, file_size) self.get_view().add_phantom( HELIUM_FIGURE_PHANTOMS, region, content, sublime.LAYOUT_BLOCK ) self._logger.info("Created phantom {}".format(content)) def _write_inline_html_phantom( self, content: str, region: sublime.Region, view: sublime.View ): if self._show_inline_output: id = HELIUM_FIGURE_PHANTOMS + datetime.now().isoformat() html = TEXT_PHANTOM.format(content=content) self._add_phantom(view, id, region, html) def _write_inline_image_phantom( self, data: str, region: sublime.Region, view: sublime.View ): if self._show_inline_output: id = HELIUM_FIGURE_PHANTOMS + datetime.now().isoformat() img_size = sublime.load_settings("Helium.sublime-settings").get( "image_size", "optimal" ) width = view.viewport_extent()[0] - 2 dimensions = get_png_dimensions(data) if img_size == "original" or ( img_size == "optimal" and (dimensions[0] < width) ): html = IMAGE_PHANTOM.format( data=data, width=dimensions[0], height=dimensions[1] ) else: scale_factor = width / dimensions[0] height = dimensions[1] * scale_factor html = IMAGE_PHANTOM.format(data=data, width=width, height=height) self._add_phantom(view, id, region, html) def _add_phantom( self, view: sublime.View, id: str, region: sublime.Region, html: str ): int_id = view.add_phantom( id, region, html, sublime.LAYOUT_BLOCK, on_navigate=lambda href, id=id, view=view: self._erase_phantom( id, view=view ), ) self.phantoms[id] = int_id if int_id > MAX_PHANTOMS: sublime.message_dialog( "Please close and reopen your current tab. " + "Otherwise the 'Clear All Cells' command might not work as intended." ) self._logger.info("Created inline phantom image") def _clear_phantoms_in_region(self, region: sublime.Region, view: sublime.View): _, cell = get_cell(view, region, logger="") remove = [ pid for pid, int_id in self.phantoms.items() if cell.contains(view.query_phantom(int_id)[0]) ] for pid in remove: self._erase_phantom(pid, view=view) def _erase_phantom(self, pid: str, *, view: sublime.View): if pid in self.phantoms: _ = self.phantoms.pop(pid) view.erase_phantoms(pid) def _write_mime_data_to_view( self, mime_data: dict, region: sublime.Region, view: sublime.View ) -> None: # Now we use basically text/plain for text type. # Jupyter kernels often emits html whom minihtml cannot render. if "text/plain" in mime_data: content = mime_data["text/plain"] lines = "\n(display data): {content}".format(content=content) self._write_text_to_view(lines) self._write_inline_html_phantom( fix_whitespace_for_phantom(content), region, view ) elif "text/html" in mime_data: self._logger.info( "Caught 'text/html' output without plain text. " "Try to show with phantom." ) content = mime_data["text/html"] self._write_phantom(content) self._write_inline_html_phantom(content, region, view) if "image/png" in mime_data: data = mime_data["image/png"].strip() img_size = sublime.load_settings("Helium.sublime-settings").get( "image_size", "optimal" ) self._logger.info(self.get_view().viewport_extent()) width = self.get_view().viewport_extent()[0] - 2 dimensions = get_png_dimensions(data) if img_size == "original" or ( img_size == "optimal" and (dimensions[0] < width) ): width, height = dimensions else: scale_factor = width / dimensions[0] height = dimensions[1] * scale_factor content = ( '' 'Out' + "" ).format(data=data, width=width, height=height, bgcolor="white") self._write_phantom(content) self._write_inline_image_phantom(data, region, view) def _handle_inspect_reply(self, reply: dict): window = sublime.active_window() if window.find_output_panel(HELIUM_OBJECT_INSPECT_PANEL) is not None: window.destroy_output_panel(HELIUM_OBJECT_INSPECT_PANEL) view = window.create_output_panel(HELIUM_OBJECT_INSPECT_PANEL) try: self._logger.debug(reply) text = remove_ansi_escape(reply["text/plain"]) view.run_command("append", {"characters": text}) window.run_command( "show_panel", {"panel": "output." + HELIUM_OBJECT_INSPECT_PANEL} ) except KeyError as ex: self._logger.exception(ex) def get_view(self): """Get view corresponds to the KernelConnection.""" view = None view_name = self.view_name window = sublime.active_window() views = window.views() for view_candidate in views: if view_candidate.name() == view_name: return view_candidate if not view: active_group = window.active_group() view = window.new_file() view.set_name(view_name) view.settings().set("syntax", "Packages/Helium/Helium.sublime-syntax") num_group = window.num_groups() if num_group != 1: if active_group + 1 < num_group: new_group = active_group + 1 else: new_group = active_group - 1 window.set_view_index( view, new_group, len(window.sheets_in_group(new_group)) ) return view def execute_code(self, code, phantom_region, view): """Run code with Jupyter kernel.""" msg_id = self.client.execute(code) self.id2region[msg_id] = ( view, sublime.Region(phantom_region.end() - 1, phantom_region.end() - 1), ) info_message = "Kernel executed code ```{code}```.".format(code=code) self._logger.info(info_message) def is_alive(self): """Return True if kernel is alive.""" return self.client.hb_channel.is_beating() def get_complete(self, code, cursor_pos, timeout=None): """Generate complete request.""" if self.execution_state != "idle": return [] msg_id = self.client.complete(code, cursor_pos) self.shell_msg_queues_lock.acquire() try: queue = self.shell_msg_queues[msg_id] finally: self.shell_msg_queues_lock.release() try: recv_msg = queue.get(timeout=timeout) recv_content = recv_msg["content"] self._logger.info(recv_content) if "_jupyter_types_experimental" in recv_content.get("metadata", {}): # If the reply has typing metadata, use it. # This metadata for typing is obviously experimental # and not documented yet. return [ ( match["text"] + "\t" + ( "" if match["type"] is None else match["type"] ), match["text"], ) for match in recv_content["metadata"]["_jupyter_types_experimental"] ] else: # Just say the completion is came from this plugin, otherwise. return [ (match + "\tHelium", match) for match in recv_content["matches"] ] except Empty: self._logger.info("Completion timeout.") except Exception as ex: self._logger.exception(ex) finally: self.shell_msg_queues_lock.acquire() try: self.shell_msg_queues.pop(msg_id, None) finally: self.shell_msg_queues_lock.release() return [] def get_inspection(self, code, cursor_pos, detail_level=0, timeout=None): """Get object inspection by sending a `inspect_request` message to kernel.""" msg_id = self.client.inspect(code, cursor_pos, detail_level) self.shell_msg_queues_lock.acquire() try: queue = self.shell_msg_queues[msg_id] finally: self.shell_msg_queues_lock.release() try: recv_msg = queue.get(timeout=timeout) self._handle_inspect_reply(recv_msg["content"]["data"]) except Empty: self._logger.info("Object inspection timeout.") finally: self.shell_msg_queues_lock.acquire() try: self.shell_msg_queues.pop(msg_id, None) finally: self.shell_msg_queues_lock.release() ================================================ FILE: lib/utils.py ================================================ import bisect import re import sys from base64 import b64decode from functools import wraps import sublime from sublime_plugin import TextCommand class add_path(object): """Temporarily insert a path into sys.path.""" def __init__(self, path): self.path = path def __enter__(self): # noqa sys.path.insert(0, self.path) def __exit__(self, exc_type, exc_value, traceback): # noqa sys.path.remove(self.path) def chain_callbacks(f): """Decorate to mimic the promise pattern via an yield expression. Decorator function to make a wrapper which executes functions yielded by the given generator in order. """ @wraps(f) def wrapper(*args, **kwargs): chain = f(*args, **kwargs) try: next_f = next(chain) except StopIteration: return def cb(*args, **kwargs): nonlocal next_f try: if len(args) + len(kwargs) != 0: next_f = chain.send(*args, **kwargs) else: next_f = next(chain) next_f(cb) except StopIteration: return next_f(cb) return wrapper PASSWORD_INPUT_PATTERN = re.compile(r"^(\**)([^*]+)(\**)") class MaskInputPanelText(TextCommand): """Command to hide all the charatcters of view by '*'.""" def run(self, edit): s = self.view.size() region = sublime.Region(0, s) self.view.replace(edit, region, s * "*") def show_password_input(prompt, on_done, on_cancel): hidden_input = "" view = None def get_hidden_input(user_input): nonlocal hidden_input on_done(hidden_input) def hide_input(user_input): nonlocal view nonlocal hidden_input matches = PASSWORD_INPUT_PATTERN.match(user_input) if matches: # When there are characters other than "*" pre, new, post = matches.group(1, 2, 3) hidden_input = ( hidden_input[: len(pre)] + new + hidden_input[len(hidden_input) - len(post) : len(hidden_input)] ) view.run_command("mask_input_panel_text") else: try: pos = view.sel()[0].begin() hidden_input = hidden_input[:pos] + hidden_input[pos : len(user_input)] except AttributeError: # `view` is not assigned at first time this function is called. pass view = sublime.active_window().show_input_panel( prompt, "", get_hidden_input, hide_input, on_cancel ) def get_png_dimensions(base64): """ Extrac the dimension properties of the IHDR information encoded in base 64. """ wh = b64decode(base64[20:32]) iwidth = int.from_bytes(wh[1:5], byteorder="big") iheight = int.from_bytes(wh[5:], byteorder="big") return (iwidth, iheight) def get_cell( view: sublime.View, region: sublime.Region, *, logger: str ) -> (str, sublime.Region): """Get the code cell under the cursor. Cells are separated by markers. Those are defined in `cell_delimiter_pattern` in the config file. If `s` is a selected region, the code cell is it. """ if not region.empty(): return (view.substr(region), region) cell_delimiter_pattern = sublime.load_settings("Helium.sublime-settings").get( "cell_delimiter_pattern" ) separators = view.find_all(cell_delimiter_pattern) separators.append(sublime.Region(view.size() + 2, view.size() + 2)) r = sublime.Region(region.begin() + 1, region.begin() + 1) start_point = separators[bisect.bisect(separators, r) - 1].end() + 1 end_point = separators[bisect.bisect(separators, r)].begin() - 1 cell_region = sublime.Region(start_point, end_point) return (view.substr(cell_region), cell_region) ================================================ FILE: messages/0.3.1.txt ================================================ Update in 0.3.1 --------------- Fixed some problems (thanks @randy3k for feedback!) - Handle "text/markdown" MIME type in `display_data` messages. Some kernel (such as IRkernel) uses this to show object repr instead of `execute_result`. - Add `username` and `session` item in message headers. Some kernel (such as IJulia) require them. ================================================ FILE: messages/0.3.2.txt ================================================ Update in 0.3.2 --------------- - Use wss for https connections. This enables Hermes to connect Jupyterhub server using token. See https://github.com/ngr-t/SublimeHermes/issues/1 for details. (Thanks to @randy3k) ================================================ FILE: messages/0.3.3.txt ================================================ Update in 0.3.3 --------------- - Fixed issue #3. Now `text/plain` type data are primarily used when `display_data` are written to the view. - Fixed issue #5. `text/html` type data are shown with in phantoms unless there `text/plain` or `text/markdown` type exist in output data. ================================================ FILE: messages/0.3.4.txt ================================================ Update in 0.3.4 --------------- - Add request parameter on creating WebSocket connection to avoid "No session ID specified" warning dumped to the server log (#9). - Show progress bar while the kernel is busy(#12). - Disable "Hermes: Execute Block" command if the kernel is dead (#13). Now we plan to rename and refactor the commands as argued in #4 (working branch is #14). If you have any openion about it, feel free to express it in the issue. ================================================ FILE: messages/0.3.5.txt ================================================ Update in 0.3.5 --------------- - Scroll to the end of the result buffer on code block execution(#15). - Added `Hermes: Settings` command, which opens the side-by-side edit setting window. Now we plan to rename and refactor the commands as argued in #4 (working branch is #14). If you have any openion about it, feel free to express it in the issue. ================================================ FILE: messages/0.3.6.txt ================================================ Update in 0.3.6 --------------- - Fixed code block detection when there are code block(s) selected. - Better error handlings around communicating via Jupyter APIs (related to #19). Now we plan to rename and refactor the commands as argued in #4 (working branch is #14). If you have any openion about it, feel free to express it in the issue. ================================================ FILE: messages/0.4.0.txt ================================================ Update in 0.4.0 --------------- - `Hermes: Set URL` command is renamed to `Hermes: Connect Server`. - `Hermes: List Kernels` command is added. You can do several actions on kernels running in Jupyter process with this command. - You become to be able to name the connections to the kernels, when you want. - Execute, object inspection, interrupt, shutdown, and restart commands are now only shown when the active view is connected to a kernel. ================================================ FILE: messages/0.4.1.txt ================================================ Update in 0.4.1 --------------- Fixed the bug failing to connect a kernel created by Hermes. ================================================ FILE: messages/0.4.2.txt ================================================ Update in 0.4.2 --------------- Fixed the bug that the editor hangs up when view is connected to a dead kernel. ================================================ FILE: messages/0.4.3.txt ================================================ Update in 0.4.3 --------------- Fix to reuse an established WebSocket connection to Jupyter kernel. This should solve some performance issues. ================================================ FILE: messages/0.5.0.txt ================================================ Update in 0.5.0 --------------- This is update with big changes. Please read this release note. I had tackled with the known serious unstability issues, and large modification was required to solve it. Consequently it requires changes about the way to Jupyter kernel. The way to make connection with Jupyter kernels is changed as below. ### 1. The most basic way, start a kernelspec installed locally, as a subprocess of ST3 (the process stops when Sublime stops) 1. Run `Hermes: connect kernel` command. 2. Choose `New kernel`. 3. Choose the kernelspec you want to run. ### 2. Connect to the kernel already runnning and connected to Hermes 1. Run `Hermes: connect kernel` command. 2. Choose the kernel you want to connect. ### 3. Connect to a kernel already running under some other Jupyter app (such as Notebook) 1. Get connection info of the kernel. The way to get connection info differ among kernels, see the doc of each kernel (in ipython kernel, you can get it by `%connect_info` magic.) 2. Run `Hermes: connect kernel` command. 3. Choose `New kernel`. 4. Choose `(Enter connection info)`. 5. Enter the connection info (Hermes accepts a path or connection info itself). ### 4. Connect to a kernel already running under some other Jupyter app, in a SSH server 1. Configure SSH servers in the setting file (opened by `Hermes: Settings` command.) 2. Get connection info of the kernel. The way to get connection info differ among kernels, see the doc of each kernel (in ipython kernel, you can get it by `%connect_info` magic.) 3. Run `Hermes: connect kernel` command. 4. Choose `New kernel`. 5. Choose `(Connect remote kernel via SSH)`. 6. Choose the server, then enter the connection info. I'm afraid that this change might break the workflow of some users, but I think it's more convenient for many people who works with locally running Jupyter. I ask you for your kind understanding about the changes. ## New features Thanks to @SamiPirbay, there are also new cool features, which are inline phantom mode and cell evaluation support. ### Inline phantoms You can get evaluation result as inline phantoms in the place where you executed code like Light Table or Hydrogen. It's enabled by setting the `inline_output` option `True`. Each phantom has a small x mark to close it. ### Cell evaluation support Regions surrounded by `# %%` or `# ` (you can configure it in `cell_delimiter_pattern` option item) are considered as "code cells". You can execute a region by `Hermes: Execute cell` or `Hermes: Execute Cell and Move` command. Each cell has a clickable "Run Cell" phantom that appears next to the cell markers to run the cell. ================================================ FILE: messages/0.5.1.txt ================================================ Update in 0.5.1 --------------- Fixed that text data in display_data were not shown as inline output. This has caused problem such as no execution results shown when we use IRkernel (#46). ================================================ FILE: messages/0.6.0.txt ================================================ Update in 0.6.0 --------------- This release adds many quality of life features and fixes some bugs: - control the size of inline plots - transparent background for images - move the cursor after executing cell is the new default - selects the correct cell for execution - bugfixes Altogether, Helium should feel much smoother to use. If you have any issues, please report them on [github/issues](https://github.com/sschuhmann/Helium/issues) ================================================ FILE: messages/0.6.1.txt ================================================ Update in 0.6.1 --------------- - added package control message ================================================ FILE: messages/0.6.2.txt ================================================ Update 0.6.2 ------------ - fixed a bug where ST4 uses python 3.8 instead of python 3.3. Thanks to walterm128 for the fix. ================================================ FILE: messages/0.6.3.txt ================================================ Update 0.6.2 ------------ - updated internal packages ================================================ FILE: messages.json ================================================ { "install": "README.md", "0.3.1": "messages/0.3.1.txt", "0.3.2": "messages/0.3.2.txt", "0.3.3": "messages/0.3.3.txt", "0.3.4": "messages/0.3.4.txt", "0.3.5": "messages/0.3.5.txt", "0.3.6": "messages/0.3.6.txt", "0.4.0": "messages/0.4.0.txt", "0.4.1": "messages/0.4.1.txt", "0.4.2": "messages/0.4.2.txt", "0.4.3": "messages/0.4.3.txt", "0.5.0": "messages/0.5.0.txt", "0.5.1": "messages/0.5.1.txt", "0.6.0": "messages/0.6.0.txt", "0.6.1": "messages/0.6.1.txt", "0.6.2": "messages/0.6.2.txt" } ================================================ FILE: pyproject.toml ================================================ [tool.poetry] name = "Helium" version = "0.6.0" description = "Let Sublime Text 3 talk with Jupyter." authors = ["Ben Felder "] repository = "https://github.com/pykong/Helium" readme = "README.md" keywords = ["sublime-text-3", "jupyter"] license = "GPL-2.0-only" exclude = [] [tool.poetry.urls] "Bug Tracker" = "https://github.com/pykong/Helium/issues" [tool.poetry.dependencies] python = "^3.8" flake8-eradicate = "^0.2.4" pre-commit = "^2.0.1" [tool.poetry.dev-dependencies] black = {version = "^24.3", allow-prereleases = true} flake8 = "^3.7.9" flake8-blind-except = "^0.1.1" flake8-comprehensions = "^3.2.2" flake8-docstrings = "^1.5.0" isort = "^4.3.21" mypy = "^0.761" [tool.isort] line_length = 88 combine_star = true known_third_party = [] skip_glob = ["lib/client"] [tool.black] line-length = 88 target_version = ["py33"] exclude = "lib/client" [build-system] requires = ["poetry>=0.12"] build-backend = "poetry.masonry.api" [flake8] ignore = ",E203,D100,D101,D102,D103,D104,D105,D106,D107,D203,W503," exclude = ",.git, .venv, lib/client," max-line-length = 88 ================================================ FILE: tests/_helpers.py ================================================ import sublime from unittest import TestCase class ViewTestCase(TestCase): """Providing basic functionality for testing against views. Taken from: https://github.com/randy3k/AlignTab/blob/master/tests/test_basic.py https://github.com/randy3k/AutoWrap/blob/master/tests/test_python.py """ def setUp(self): # make sure we have a window to work with s = sublime.load_settings("Preferences.sublime-settings") s.set("close_windows_when_empty", False) self.view = sublime.active_window().new_file() def tearDown(self): if self.view: self.view.set_scratch(True) self.view.window().run_command("close_file") def set_text(self, string): self.view.run_command("insert", {"characters": string}) def clear_view(self): self.view.run_command("select_all") self.view.run_command("right_delete") ================================================ FILE: tests/test_cell_handling.py ================================================ import sublime from _helpers import ViewTestCase valid_delimiters = ( # %% pattern "#%%", "# %%", # `in` pattern "# in[5]:", "# in[]:", "# in:", "#in:" ) invalid_delimiters = ( "#", "#%", "#%?", "%", "% 123", "no way", "# normal comment", "#xin:", "# in:", "# in", "in:", "# In[5]:", "# In[]:", "# In:" ) class TestDelimiter(ViewTestCase): def find_all_delimiters(self): s = sublime.load_settings("Helium.sublime-settings") pattern = s.get("cell_delimiter_pattern") return self.view.find_all(pattern) def check_content_against_match_count(self, content, expected_count): self.clear_view() self.set_text(content) matches = self.find_all_delimiters() assert len(matches) == expected_count def test_pattern_against_delimiteres_valid(self): """Succeed if all of the valid delimiters match.""" for d in valid_delimiters: # TODO: Use subTest once on ST4 self.check_content_against_match_count(d, 1) def test_pattern_against_delimiteres_invalid(self): """Succeed if none of the invalid delimiters match.""" for d in invalid_delimiters: # TODO: Use subTest once on ST4 self.check_content_against_match_count(d, 0) def test_delimiter_match_count_against_pattern_one(self): """Succeed if match count from view.find_all equal its expectation.""" for i in range(10): # TODO: Use subTest once on ST4 self.check_content_against_match_count("# %% \n" * i, i) def test_delimiter_match_count_against_pattern_two(self): """Succeed if match count from view.find_all equal its expectation.""" for i in range(10): # TODO: Use subTest once on ST4 self.check_content_against_match_count("# in: \n" * i, i) def test_delimiter_match_count_against_pattern_mixed(self): """Succeed if match count from view.find_all equal its expectation.""" for i in range(10): # TODO: Use subTest once on ST4 self.check_content_against_match_count("# %% \n# in: \n" * i, i * 2) ================================================ FILE: tests/test_tests_running.py ================================================ import unittest class TestThatTestsAreRunning(unittest.TestCase): def test_that_never_fails(self): """Check that tests are indeed being run.""" self.assertTrue(True)