[ { "path": ".devcontainer/Dockerfile", "content": "FROM mcr.microsoft.com/devcontainers/python:3.13\n\nCOPY requirements-dev.txt /tmp/requirements-dev.txt\nCOPY docs/requirements.txt /tmp/requirements-docs.txt\n\nRUN pip install --no-cache-dir -r /tmp/requirements-dev.txt && \\\n pip install --no-cache-dir -r /tmp/requirements-docs.txt\n" }, { "path": ".devcontainer/devcontainer.json", "content": "{\n \"name\": \"Jupyter Docker Stacks\",\n \"build\": {\n \"context\": \"..\",\n \"dockerfile\": \"Dockerfile\"\n },\n\n \"features\": {\n \"ghcr.io/devcontainers/features/docker-in-docker:2\": {\n \"moby\": false\n }\n },\n\n \"postCreateCommand\": \"pre-commit install --install-hooks\",\n\n \"customizations\": {\n \"vscode\": {\n \"extensions\": [\n \"github.copilot-chat\",\n \"github.copilot\",\n \"github.vscode-github-actions\",\n \"github.vscode-pull-request-github\",\n \"ms-azuretools.vscode-containers\",\n \"ms-azuretools.vscode-docker\",\n \"ms-python.autopep8\",\n \"ms-vscode.makefile-tools\"\n ]\n }\n }\n}\n" }, { "path": ".flake8", "content": "[flake8]\nmax-line-length = 88\nselect = C, E, F, W, B, B950\nextend-ignore = E203, E501, E704, W503\n" }, { "path": ".gitattributes", "content": "* text=auto eol=lf\n" }, { "path": ".github/ISSUE_TEMPLATE/blank.yml", "content": "name: \"(maintainers only) Blank issue\"\ndescription: For maintainers only\nlabels: []\n\nbody:\n - type: markdown\n attributes:\n value: |\n This is a blank issue template for maintainers to use as needed.\n\n - type: checkboxes\n attributes:\n label: Are you a maintainer?\n description: Please confirm you are a maintainer before proceeding.\n options:\n - label: Yes, I am a maintainer\n required: true\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.yml", "content": "name: Bug report\ndescription: Create a report to help us improve\nlabels: [\"type:Bug\"]\n\nbody:\n - type: markdown\n attributes:\n value: |\n Hi! Thanks for using the Jupyter Docker Stacks and taking some time to contribute to this project.\n\n We'd appreciate it if you could check out the [Troubleshooting common problems](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/troubleshooting.html) section in the documentation,\n as well as [existing issues](https://github.com/jupyter/docker-stacks/issues?q=is%3Aissue) prior to submitting an issue to avoid duplication.\n\n Please answer the following sections to help us troubleshoot the problem.\n\n - type: dropdown\n attributes:\n label: What docker image(s) are you using?\n description: Select as many images as applicable\n multiple: true\n options:\n - all-spark-notebook\n - base-notebook\n - datascience-notebook\n - docker-stacks-foundation\n - julia-notebook\n - minimal-notebook\n - pyspark-notebook\n - pytorch-notebook\n - r-notebook\n - scipy-notebook\n - tensorflow-notebook\n validations:\n required: true\n\n - type: input\n attributes:\n label: Host OS\n placeholder: |\n Example:\n Ubuntu 24.04\n validations:\n required: true\n\n - type: dropdown\n attributes:\n label: Host architecture\n options:\n - x86_64\n - aarch64\n validations:\n required: true\n\n - type: textarea\n attributes:\n label: What Docker command are you running?\n description: |\n What complete docker command do you run to launch the container (omitting sensitive values)?\n placeholder: |\n Example:\n `docker run -it --rm -p 8888:8888 quay.io/jupyter/base-notebook`\n validations:\n required: true\n\n - type: textarea\n attributes:\n label: How to Reproduce the problem?\n description: Please provide steps to reproduce this bug (once the container is running).\n placeholder: |\n Example:\n\n 1. Visit \n\n 2. Start an R notebook\n\n 3. ...\n validations:\n required: true\n\n - type: textarea\n attributes:\n label: Command output\n render: bash session\n description: |\n Provide the output of the steps above, including the commands\n themselves and Docker's output/traceback etc. If you're familiar with\n Markdown, this block will have triple backticks added automatically\n around it -- you don't have to add them.\n\n If you want to present output from multiple commands, please present\n that as a shell session (commands you run get prefixed with `$ `).\n Please also ensure that the \"How to reproduce\" section contains matching\n instructions for reproducing this.\n\n - type: textarea\n attributes:\n label: Expected behavior\n description: |\n A clear and concise description of what you expected to happen.\n placeholder: |\n Example: `ggplot` output appears in my notebook.\n\n - type: textarea\n attributes:\n label: Actual behavior\n description: |\n A clear and concise description of what the bug is.\n placeholder: |\n Example: No output is visible in the notebook and the Server log contains messages about ...\n validations:\n required: true\n\n - type: textarea\n attributes:\n label: Anything else?\n description: |\n Links? References? Anything that will give us more context about the issue you are encountering!\n\n Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.\n validations:\n required: false\n\n - type: checkboxes\n attributes:\n label: Latest Docker version\n description: You should try to use the latest Docker version\n options:\n - label: I've updated my Docker version to the latest available, and the issue persists\n required: true\n" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "blank_issues_enabled: false\ncontact_links:\n - name: ๐Ÿ“– - Jupyter Docker Stacks documentation\n url: https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html\n about: Go to the project's documentation\n - name: ๐Ÿ” - Troubleshooting common problems\n url: https://jupyter-docker-stacks.readthedocs.io/en/latest/using/troubleshooting.html\n about: Documentation section on troubleshooting commonly encountered errors\n - name: ๐Ÿ’ฌ - Jupyter community Discourse\n url: https://discourse.jupyter.org/\n about: Interact with the rest of the Jupyter community\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.yml", "content": "name: Feature request\ndescription: Suggest a new feature for this project\nlabels: [\"type:Enhancement\"]\n\nbody:\n - type: markdown\n attributes:\n value: |\n Hi! Thanks for using the Jupyter Docker Stacks and taking some time to contribute to this project.\n\n We'd appreciate it if you could check out the [Suggesting a new feature](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/features.html#suggesting-a-new-feature)\n section in the documentation for our preferred processes before submitting a feature request.\n\n - type: dropdown\n attributes:\n label: What docker image(s) is this feature applicable to?\n description: Select as many images as applicable\n multiple: true\n options:\n - all-spark-notebook\n - base-notebook\n - datascience-notebook\n - docker-stacks-foundation\n - julia-notebook\n - minimal-notebook\n - pyspark-notebook\n - pytorch-notebook\n - r-notebook\n - scipy-notebook\n - tensorflow-notebook\n - new community stack\n validations:\n required: true\n\n - type: textarea\n attributes:\n label: What change(s) are you proposing?\n description: |\n Be concise and feel free to add supporting links or references.\n placeholder: |\n Example:\n - Add the [altair](https://altair-viz.github.io) package to the image.\n validations:\n required: true\n\n - type: textarea\n attributes:\n label: How does this affect the user?\n description: |\n How will the proposed feature affect the user's workflow?\n How will this feature make the image more robust, secure, etc.?\n placeholder: |\n Example:\n - Altair is a declarative statistical visualization library for Python, based on Vega and Vega-Lite, and the source is available on GitHub.\n - With Altair, you can spend more time understanding your data and its meaning.\n - Altair's API is simple, friendly, and consistent and built on top of the powerful Vega-Lite visualization grammar.\n - This elegant simplicity produces beautiful and effective visualizations with a minimal amount of code.\n validations:\n required: true\n\n - type: textarea\n attributes:\n label: Anything else?\n description: |\n Links? References? Anything that will give us more context about the feature you are proposing.\n validations:\n required: false\n" }, { "path": ".github/actions/apply-single-tags/action.yml", "content": "name: Apply single platform tags\ndescription: Download the image tar, load it to Docker and apply tags to it\n\ninputs:\n image:\n description: Image name\n required: true\n platform:\n description: Image platform\n required: true\n variant:\n description: Variant tag prefix\n required: true\n\nruns:\n using: composite\n steps:\n - name: Load image to Docker ๐Ÿ“ฅ\n uses: ./.github/actions/load-image\n with:\n image: ${{ inputs.image }}\n platform: ${{ inputs.platform }}\n variant: ${{ inputs.variant }}\n\n - name: Download tags file ๐Ÿ“ฅ\n uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1\n with:\n name: ${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}.txt\n path: /tmp/jupyter/tags/\n\n - name: Apply tags to the loaded image ๐Ÿท\n run: |\n python3 -m tagging.apps.apply_tags \\\n --registry ${{ env.REGISTRY }} \\\n --owner ${{ env.OWNER }} \\\n --image ${{ inputs.image }} \\\n --variant ${{ inputs.variant }} \\\n --platform ${{ inputs.platform }} \\\n --tags-dir /tmp/jupyter/tags/\n shell: bash\n\n - name: Upload SBOM for the image ๐Ÿงพ\n uses: anchore/sbom-action@57aae528053a48a3f6235f2d9461b05fbcb7366d # v0.23.1\n with:\n image: ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }}\n artifact-name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-sbom.spdx.json\n upload-artifact-retention: 40\n\n # This step is needed to prevent pushing non-multi-arch \"latest\" tag\n - name: Remove the \"latest\" tag from the image ๐Ÿ—‘๏ธ\n run: docker image rmi ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }}:latest\n shell: bash\n\n - name: Show Docker images ๐Ÿ“ฆ\n run: docker image ls --all\n shell: bash\n" }, { "path": ".github/actions/create-dev-env/action.yml", "content": "name: Build environment\ndescription: Create a build environment\n\nruns:\n using: composite\n steps:\n - name: Set Up Python ๐Ÿ\n uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: 3.12\n\n - name: Install Dev Dependencies ๐Ÿ“ฆ\n run: |\n pip install --upgrade pip\n pip install --upgrade -r requirements-dev.txt\n shell: bash\n\n # We need to have a recent docker version\n # More info: https://github.com/jupyter/docker-stacks/pull/2255\n # Can be removed after Docker Engine is updated\n # https://github.com/actions/runner-images/issues/11766\n - name: Set Up Docker ๐Ÿณ\n uses: docker/setup-docker-action@1a6edb0ba9ac496f6850236981f15d8f9a82254d # v5.0.0\n with:\n set-host: true\n" }, { "path": ".github/actions/free-disk-space/action.yml", "content": "name: \"Free Disk Space (Ubuntu)\"\ndescription: \"A GitHub Action to free up disk space on an Ubuntu GitHub Actions runner.\"\n\nruns:\n using: \"composite\"\n steps:\n - name: Installing rmz\n shell: bash\n run: |\n curl -fsSL --tlsv1.2 --proto '=https' https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash > /dev/null 2>&1\n cargo binstall -qy rmz\n ln -s ~/.cargo/bin/rmz /usr/local/bin/rmz\n\n - name: Freeing up disk space\n shell: bash\n run: |\n sudo rmz -f /usr/local/lib/android || true\n sudo rmz -f /usr/share/dotnet || true\n sudo rmz -f /opt/ghc /usr/local/.ghcup || true\n sudo rmz -f /usr/share/swift || true\n sudo rmz -f /usr/share/miniconda || true\n sudo rmz -f \"${AGENT_TOOLSDIRECTORY}\" || true\n" }, { "path": ".github/actions/load-image/action.yml", "content": "name: Load Docker image\ndescription: Download the image tar and load it to Docker\n\ninputs:\n image:\n description: Image name\n required: true\n platform:\n description: Image platform\n required: true\n variant:\n description: Variant tag prefix\n required: true\n\nruns:\n using: composite\n steps:\n - name: Download built image ๐Ÿ“ฅ\n uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1\n with:\n name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst\n path: /tmp/jupyter/images/\n\n - name: Load downloaded image to docker ๐Ÿ“ฅ\n run: |\n zstd \\\n --uncompress \\\n --stdout \\\n --rm \\\n /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst \\\n | docker load\n shell: bash\n\n - name: Show Docker images ๐Ÿ“ฆ\n run: docker image ls --all\n shell: bash\n" }, { "path": ".github/dependabot.yml", "content": "# To get started with Dependabot version updates, you'll need to specify which\n# package ecosystems to update and where the package manifests are located.\n# Please see the documentation for all configuration options:\n# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/dependabot-options-reference\n\n# We're adding `[FAST_BUILD]` prefix to commit messages (this adds it to PR title)\n# This triggers a faster build, see more info in `.github/workflows/docker.yml`\n\nversion: 2\nupdates:\n - package-ecosystem: github-actions\n directory: /\n schedule:\n interval: weekly\n commit-message:\n prefix: \"[FAST_BUILD] \"\n - package-ecosystem: github-actions\n directory: .github/actions/apply-single-tags/\n schedule:\n interval: weekly\n commit-message:\n prefix: \"[FAST_BUILD] \"\n - package-ecosystem: github-actions\n directory: .github/actions/create-dev-env/\n schedule:\n interval: weekly\n commit-message:\n prefix: \"[FAST_BUILD] \"\n # This action is only used for some images, so full build is required\n - package-ecosystem: github-actions\n directory: .github/actions/free-disk-space/\n schedule:\n interval: weekly\n - package-ecosystem: github-actions\n directory: .github/actions/load-image/\n schedule:\n interval: weekly\n commit-message:\n prefix: \"[FAST_BUILD] \"\n" }, { "path": ".github/pull_request_template.md", "content": "## Describe your changes\n\n## Issue ticket if applicable\n\n\n\n## Checklist (especially for first-time contributors)\n\n- [ ] I have performed a self-review of my code\n- [ ] If it is a core feature, I have added thorough tests\n- [ ] I will try not to use force-push to make the review process easier for reviewers\n- [ ] I have updated the documentation for significant changes\n\n\n" }, { "path": ".github/workflows/contributed-recipes.yml", "content": "name: Test the contributed recipes\n\nenv:\n REGISTRY: quay.io\n OWNER: ${{ github.repository_owner }}\n\non:\n schedule:\n # Images are rebuilt at 03:00 on Monday UTC\n # So we're testing recipes one hour in advance\n # They will also be tested after building images\n - cron: \"0 2 * * 1\"\n pull_request:\n paths:\n - \".github/workflows/contributed-recipes.yml\"\n - \"docs/using/recipe_code/**\"\n push:\n branches:\n - main\n paths:\n - \".github/workflows/contributed-recipes.yml\"\n - \"docs/using/recipe_code/**\"\n workflow_dispatch:\n workflow_call:\n inputs:\n # There is no good way to detect if the workflow was called using workflow_call\n # https://github.com/actions/runner/discussions/1884\n called-using-workflow-call:\n description: \"Was the workflow called using workflow_call\"\n required: true\n type: boolean\n\npermissions:\n contents: read\n\njobs:\n generate-matrix:\n runs-on: ubuntu-24.04\n timeout-minutes: 1\n outputs:\n matrix: ${{ steps.set-matrix.outputs.matrix }}\n steps:\n - name: Checkout Repo โšก๏ธ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Calculate recipes matrix ๐Ÿ› \n id: set-matrix\n run: docs/using/recipe_code/generate_matrix.py >> \"${GITHUB_OUTPUT}\"\n env:\n REPOSITORY_OWNER: ${{ github.repository_owner }}\n\n build:\n runs-on: ${{ matrix.runs-on }}\n timeout-minutes: 10\n needs: generate-matrix\n if: github.repository_owner == 'jupyter' || github.repository_owner == 'mathbunnyru'\n\n steps:\n - name: Checkout Repo โšก๏ธ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Load image to Docker ๐Ÿ“ฅ\n if: ${{ inputs.called-using-workflow-call && matrix.parent-image != '' }}\n uses: ./.github/actions/load-image\n with:\n image: ${{ matrix.parent-image }}\n platform: ${{ matrix.platform }}\n variant: default\n\n # Not pulling the image, because it might be loaded from previous step or will be downloaded automatically\n - name: Build recipe with parent image ๐Ÿ› \n if: ${{ matrix.parent-image != '' }}\n run: |\n docker build \\\n --rm --force-rm \\\n --tag my-custom-image \\\n -f ./${{ matrix.dockerfile }} \\\n --build-arg BASE_IMAGE=${{ env.REGISTRY }}/${{ env.OWNER }}/${{ matrix.parent-image }} \\\n ./\n env:\n DOCKER_BUILDKIT: 1\n # Full logs for CI build\n BUILDKIT_PROGRESS: plain\n working-directory: docs/using/recipe_code\n shell: bash\n\n # Not pulling the image, because it might be loaded from previous step or will be downloaded automatically\n - name: Build recipe without parent image ๐Ÿ› \n if: ${{ matrix.parent-image == '' }}\n run: |\n docker build \\\n --rm --force-rm \\\n --tag my-custom-image \\\n -f ./${{ matrix.dockerfile }} \\\n ./\n env:\n DOCKER_BUILDKIT: 1\n # Full logs for CI build\n BUILDKIT_PROGRESS: plain\n working-directory: docs/using/recipe_code\n shell: bash\n\n strategy:\n matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}\n" }, { "path": ".github/workflows/docker-build-test-upload.yml", "content": "name: Download a parent image, build a new one, and test it; upload the image, tags, build history line and manifest to GitHub artifacts\n\nenv:\n REGISTRY: quay.io\n OWNER: ${{ github.repository_owner }}\n\non:\n workflow_call:\n inputs:\n parent-image:\n description: Parent image name\n required: true\n type: string\n parent-variant:\n description: Parent variant tag prefix\n required: false\n type: string\n default: default\n image:\n description: Image name\n required: true\n type: string\n variant:\n description: Variant tag prefix\n required: false\n type: string\n default: default\n platform:\n description: Image platform\n required: true\n type: string\n runs-on:\n description: GitHub Actions Runner image\n required: true\n type: string\n timeout-minutes:\n description: Timeout in minutes\n required: true\n type: number\n\npermissions:\n contents: read\n\njobs:\n build-test-upload:\n runs-on: ${{ inputs.runs-on }}\n timeout-minutes: ${{ inputs.timeout-minutes }}\n\n steps:\n - name: Checkout Repo โšก๏ธ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n - name: Compute commit hash tag ๐Ÿท\n id: hash\n run: echo \"tag=${GITHUB_SHA::12}\" >> \"$GITHUB_OUTPUT\"\n shell: bash\n - name: Free disk space ๐Ÿงน\n uses: ./.github/actions/free-disk-space\n if: contains(inputs.variant, 'cuda') || inputs.image == 'datascience-notebook' || inputs.image == 'all-spark-notebook'\n - name: Create dev environment ๐Ÿ“ฆ\n uses: ./.github/actions/create-dev-env\n\n - name: Load parent built image to Docker ๐Ÿ“ฅ\n if: inputs.parent-image != ''\n uses: ./.github/actions/load-image\n with:\n image: ${{ inputs.parent-image }}\n platform: ${{ inputs.platform }}\n variant: ${{ inputs.parent-variant }}\n\n - name: Pull base ubuntu image ๐Ÿ“ฅ\n if: inputs.parent-image == ''\n run: docker pull ubuntu:24.04\n shell: bash\n\n - name: Build image ๐Ÿ› \n run: |\n docker build \\\n --rm --force-rm \\\n --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} \\\n images/${{ inputs.image }}/${{ inputs.variant != 'default' && inputs.variant || '.' }}/ \\\n --build-arg REGISTRY=${{ env.REGISTRY }} \\\n --build-arg OWNER=${{ env.OWNER }}\n env:\n DOCKER_BUILDKIT: 1\n # Full logs for CI build\n BUILDKIT_PROGRESS: plain\n shell: bash\n\n - name: Write tags file ๐Ÿท\n run: |\n python3 -m tagging.apps.write_tags_file \\\n --registry ${{ env.REGISTRY }} \\\n --owner ${{ env.OWNER }} \\\n --image ${{ inputs.image }} \\\n --variant ${{ inputs.variant }} \\\n --tags-dir /tmp/jupyter/tags/\n shell: bash\n - name: Upload tags file ๐Ÿ’พ\n uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0\n with:\n path: /tmp/jupyter/tags/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}.txt\n retention-days: 3\n archive: false\n\n - name: Write manifest and build history file ๐Ÿท\n run: |\n python3 -m tagging.apps.write_manifest \\\n --registry ${{ env.REGISTRY }} \\\n --owner ${{ env.OWNER }} \\\n --image ${{ inputs.image }} \\\n --variant ${{ inputs.variant }} \\\n --hist-lines-dir /tmp/jupyter/hist_lines/ \\\n --manifests-dir /tmp/jupyter/manifests/ \\\n --repository ${{ github.repository }}\n shell: bash\n - name: Upload manifest file ๐Ÿ’พ\n uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0\n with:\n path: /tmp/jupyter/manifests/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}-${{ steps.hash.outputs.tag }}.md\n retention-days: 3\n archive: false\n - name: Upload build history line ๐Ÿ’พ\n uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0\n with:\n path: /tmp/jupyter/hist_lines/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}-${{ steps.hash.outputs.tag }}.txt\n retention-days: 3\n archive: false\n\n - name: Save image as a tar for later use ๐Ÿ’พ\n run: |\n mkdir -p /tmp/jupyter/images/\n docker save \\\n ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} \\\n | zstd > /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst\n shell: bash\n - name: Upload image as artifact ๐Ÿ’พ\n uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0\n with:\n path: /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst\n retention-days: 3\n archive: false\n\n - name: Run tests โœ…\n run: |\n python3 -m tests.run_tests \\\n --registry ${{ env.REGISTRY }} \\\n --owner ${{ env.OWNER }} \\\n --image ${{ inputs.image }}\n shell: bash\n" }, { "path": ".github/workflows/docker-tag-merge.yml", "content": "name: Merge single platform tags\n\nenv:\n REGISTRY: quay.io\n PUSH_TO_REGISTRY: ${{ (github.repository_owner == 'jupyter' || github.repository_owner == 'mathbunnyru') && (github.ref == 'refs/heads/main' || github.event_name == 'schedule') }}\n\non:\n workflow_call:\n inputs:\n image:\n description: Image name\n required: true\n type: string\n variant:\n description: Variant tag prefix\n required: true\n type: string\n timeout-minutes:\n description: Timeout in minutes\n default: 5\n type: number\n secrets:\n REGISTRY_USERNAME:\n required: true\n REGISTRY_TOKEN:\n required: true\n\npermissions:\n contents: read\n\njobs:\n tag-merge:\n runs-on: ubuntu-24.04\n timeout-minutes: ${{ inputs.timeout-minutes }}\n\n steps:\n - name: Checkout Repo โšก๏ธ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Create dev environment ๐Ÿ“ฆ\n uses: ./.github/actions/create-dev-env\n\n - name: Download aarch64 tags file ๐Ÿท\n uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1\n with:\n name: aarch64-${{ inputs.variant }}-${{ inputs.image }}.txt\n path: /tmp/jupyter/tags/\n\n - name: Download x86_64 tags file ๐Ÿท\n uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1\n with:\n name: x86_64-${{ inputs.variant }}-${{ inputs.image }}.txt\n path: /tmp/jupyter/tags/\n\n - name: Login to Registry ๐Ÿ”\n if: env.PUSH_TO_REGISTRY == 'true'\n run: |\n docker login ${{ env.REGISTRY }} \\\n --username ${{ secrets.REGISTRY_USERNAME }} \\\n --password ${{ secrets.REGISTRY_TOKEN }} || \\\n docker login ${{ env.REGISTRY }} \\\n --username ${{ secrets.REGISTRY_USERNAME }} \\\n --password ${{ secrets.REGISTRY_TOKEN }}\n shell: bash\n id: login\n\n - name: Merge tags for the images ๐Ÿ”€\n run: |\n python3 -m tagging.apps.merge_tags \\\n --image ${{ inputs.image }} \\\n --variant ${{ inputs.variant }} \\\n --tags-dir /tmp/jupyter/tags/\n shell: bash\n\n - name: Logout from Registry ๐Ÿ”\n if: always() && env.PUSH_TO_REGISTRY == 'true' && steps.login.outcome == 'success'\n run: |\n docker logout ${{ env.REGISTRY }}\n shell: bash\n" }, { "path": ".github/workflows/docker-tag-push-merge.yml", "content": "name: Download a Docker image and its tags from GitHub artifacts, apply them, and push the image to the Registry; then merge them\n\non:\n workflow_call:\n inputs:\n image:\n description: Image name\n required: true\n type: string\n variant:\n description: Variant tag prefix\n required: true\n type: string\n secrets:\n REGISTRY_USERNAME:\n required: true\n REGISTRY_TOKEN:\n required: true\n\npermissions:\n contents: read\n\njobs:\n tag-push:\n uses: ./.github/workflows/docker-tag-push.yml\n with:\n image: ${{ inputs.image }}\n variant: ${{ inputs.variant }}\n secrets:\n REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}\n REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}\n\n tag-merge:\n uses: ./.github/workflows/docker-tag-merge.yml\n needs: tag-push\n with:\n image: ${{ inputs.image }}\n variant: ${{ inputs.variant }}\n secrets:\n REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}\n REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}\n" }, { "path": ".github/workflows/docker-tag-push.yml", "content": "name: Download a Docker image and its tags from GitHub artifacts, apply them, and push the image to the Registry\n\nenv:\n REGISTRY: quay.io\n OWNER: ${{ github.repository_owner }}\n PUSH_TO_REGISTRY: ${{ (github.repository_owner == 'jupyter' || github.repository_owner == 'mathbunnyru') && (github.ref == 'refs/heads/main' || github.event_name == 'schedule') }}\n\non:\n workflow_call:\n inputs:\n image:\n description: Image name\n required: true\n type: string\n variant:\n description: Variant tag prefix\n required: true\n type: string\n timeout-minutes:\n description: Timeout in minutes\n default: 25\n type: number\n secrets:\n REGISTRY_USERNAME:\n required: true\n REGISTRY_TOKEN:\n required: true\n\npermissions:\n contents: read\n\njobs:\n tag-push:\n runs-on: ubuntu-24.04\n timeout-minutes: ${{ inputs.timeout-minutes }}\n strategy:\n matrix:\n platform: [aarch64, x86_64]\n\n steps:\n - name: Checkout Repo โšก๏ธ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n - name: Free disk space ๐Ÿงน\n uses: ./.github/actions/free-disk-space\n if: contains(inputs.variant, 'cuda') || inputs.image == 'datascience-notebook' || inputs.image == 'all-spark-notebook'\n - name: Create dev environment ๐Ÿ“ฆ\n uses: ./.github/actions/create-dev-env\n\n - name: Download image tar and apply tags ๐Ÿท\n uses: ./.github/actions/apply-single-tags\n with:\n image: ${{ inputs.image }}\n variant: ${{ inputs.variant }}\n platform: ${{ matrix.platform }}\n\n - name: Login to Registry ๐Ÿ”\n if: env.PUSH_TO_REGISTRY == 'true'\n run: |\n docker login ${{ env.REGISTRY }} \\\n --username ${{ secrets.REGISTRY_USERNAME }} \\\n --password ${{ secrets.REGISTRY_TOKEN }} || \\\n docker login ${{ env.REGISTRY }} \\\n --username ${{ secrets.REGISTRY_USERNAME }} \\\n --password ${{ secrets.REGISTRY_TOKEN }}\n shell: bash\n id: login\n\n - name: Push single platform images to Registry ๐Ÿ“ค\n if: env.PUSH_TO_REGISTRY == 'true'\n run: |\n docker push --all-tags ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} || \\\n docker push --all-tags ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }}\n shell: bash\n\n - name: Logout from Registry ๐Ÿ”\n if: always() && env.PUSH_TO_REGISTRY == 'true' && steps.login.outcome == 'success'\n run: |\n docker logout ${{ env.REGISTRY }}\n shell: bash\n" }, { "path": ".github/workflows/docker-wiki-update.yml", "content": "name: Download build history lines and manifests from GitHub artifacts and push them to the GitHub wiki\n# We're doing everything in one workflow on purpose\n# This way we make sure we don't access wiki pages from several jobs simultaneously\n\nenv:\n PUSH_TO_REGISTRY: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' }}\n\non:\n workflow_call:\n\npermissions:\n contents: write\n\njobs:\n wiki-update:\n runs-on: ubuntu-24.04\n timeout-minutes: 1\n\n steps:\n - name: Checkout Repo โšก๏ธ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n - name: Create dev environment ๐Ÿ“ฆ\n uses: ./.github/actions/create-dev-env\n\n - name: Compute commit hash tag ๐Ÿท\n id: hash\n run: echo \"tag=${GITHUB_SHA::12}\" >> \"$GITHUB_OUTPUT\"\n shell: bash\n\n - name: Download all history lines ๐Ÿ“ฅ\n uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1\n with:\n pattern: \"*-${{ steps.hash.outputs.tag }}.txt\"\n path: /tmp/jupyter/hist_lines/\n merge-multiple: true\n\n - name: Download all manifests ๐Ÿ“ฅ\n uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1\n with:\n pattern: \"*-${{ steps.hash.outputs.tag }}.md\"\n path: /tmp/jupyter/manifests/\n merge-multiple: true\n\n - name: Checkout Wiki Repo ๐Ÿ“ƒ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n repository: ${{ github.repository }}.wiki\n path: wiki_src/\n\n - name: Update wiki ๐Ÿท\n run: |\n python3 -m wiki.update_wiki \\\n --wiki-dir wiki_src/ \\\n --hist-lines-dir /tmp/jupyter/hist_lines/ \\\n --manifests-dir /tmp/jupyter/manifests/ \\\n --repository ${{ github.repository }}\n shell: bash\n\n - name: Push Wiki to GitHub ๐Ÿ“ค\n if: env.PUSH_TO_REGISTRY == 'true'\n uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # v7.1.0\n with:\n commit_message: \"Automated wiki publish for ${{ github.sha }}\"\n repository: wiki_src/\n" }, { "path": ".github/workflows/docker.yml", "content": "name: Docker Stacks\n\n# [FAST_BUILD] in the PR title makes this workflow only build\n# the `jupyter/docker-stacks-foundation` and `jupyter/base-notebook` images\n# This allows to run CI faster if a full build is not required\n# This only works for a `pull_request` event and does not affect `push` to the `main` branch\n\non:\n schedule:\n # Weekly, at 03:00 on Monday UTC\n - cron: \"0 3 * * 1\"\n pull_request:\n paths:\n - \".github/workflows/docker.yml\"\n # We use local reusable workflows to make architecture clean and simple\n # https://docs.github.com/en/actions/sharing-automations/reusing-workflows\n - \".github/workflows/docker-build-test-upload.yml\"\n - \".github/workflows/docker-tag-merge.yml\"\n - \".github/workflows/docker-tag-push-merge.yml\"\n - \".github/workflows/docker-tag-push.yml\"\n - \".github/workflows/docker-wiki-update.yml\"\n\n # We use local composite actions to combine multiple workflow steps within one action\n # https://docs.github.com/en/actions/sharing-automations/creating-actions/about-custom-actions#composite-actions\n - \".github/actions/apply-single-tags/action.yml\"\n - \".github/actions/create-dev-env/action.yml\"\n - \".github/actions/free-disk-space/action.yml\"\n - \".github/actions/load-image/action.yml\"\n\n - \"images/**\"\n - \"!images/*/README.md\"\n - \"tagging/**\"\n - \"!tagging/README.md\"\n - \"tests/**\"\n - \"!tests/README.md\"\n - \"wiki/**\"\n - \"requirements-dev.txt\"\n push:\n branches:\n - main\n paths:\n - \".github/workflows/docker.yml\"\n - \".github/workflows/docker-build-test-upload.yml\"\n - \".github/workflows/docker-tag-merge.yml\"\n - \".github/workflows/docker-tag-push-merge.yml\"\n - \".github/workflows/docker-tag-push.yml\"\n - \".github/workflows/docker-wiki-update.yml\"\n\n - \".github/actions/apply-single-tags/action.yml\"\n - \".github/actions/create-dev-env/action.yml\"\n - \".github/actions/free-disk-space/action.yml\"\n - \".github/actions/load-image/action.yml\"\n\n - \"images/**\"\n - \"!images/*/README.md\"\n - \"tagging/**\"\n - \"!tagging/README.md\"\n - \"tests/**\"\n - \"!tests/README.md\"\n - \"wiki/**\"\n - \"requirements-dev.txt\"\n workflow_dispatch:\n\n# https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/control-the-concurrency-of-workflows-and-jobs\nconcurrency:\n # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags\n group: ${{ github.workflow }}-${{ github.ref }}\n cancel-in-progress: true\n\npermissions:\n contents: read\n\njobs:\n aarch64-foundation:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: \"\"\n image: docker-stacks-foundation\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 10\n\n x86_64-foundation:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: \"\"\n image: docker-stacks-foundation\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 10\n\n aarch64-base:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: docker-stacks-foundation\n image: base-notebook\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 15\n needs: aarch64-foundation\n\n x86_64-base:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: docker-stacks-foundation\n image: base-notebook\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 15\n needs: x86_64-foundation\n\n aarch64-minimal:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: base-notebook\n image: minimal-notebook\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 15\n needs: aarch64-base\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-minimal:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: base-notebook\n image: minimal-notebook\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 15\n needs: x86_64-base\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-scipy:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: minimal-notebook\n image: scipy-notebook\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 15\n needs: aarch64-minimal\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-scipy:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: minimal-notebook\n image: scipy-notebook\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 15\n needs: x86_64-minimal\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-r:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: minimal-notebook\n image: r-notebook\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 15\n needs: aarch64-minimal\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-r:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: minimal-notebook\n image: r-notebook\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 15\n needs: x86_64-minimal\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-julia:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: minimal-notebook\n image: julia-notebook\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n # This workflow sometimes takes quite long to build\n timeout-minutes: 30\n needs: aarch64-minimal\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-julia:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: minimal-notebook\n image: julia-notebook\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 20\n needs: x86_64-minimal\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-tensorflow:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: tensorflow-notebook\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 15\n needs: aarch64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-tensorflow:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: tensorflow-notebook\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 15\n needs: x86_64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-tensorflow-cuda:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: tensorflow-notebook\n variant: cuda\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 25\n needs: aarch64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-tensorflow-cuda:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: tensorflow-notebook\n variant: cuda\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 25\n needs: x86_64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-pytorch:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: pytorch-notebook\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 20\n needs: aarch64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-pytorch:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: pytorch-notebook\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 20\n needs: x86_64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-pytorch-cuda12:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: pytorch-notebook\n variant: cuda12\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 25\n needs: aarch64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-pytorch-cuda12:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: pytorch-notebook\n variant: cuda12\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 25\n needs: x86_64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-pytorch-cuda13:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: pytorch-notebook\n variant: cuda13\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 25\n needs: aarch64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-pytorch-cuda13:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: pytorch-notebook\n variant: cuda13\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 25\n needs: x86_64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-datascience:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: datascience-notebook\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n # This workflow sometimes takes quite long to build\n timeout-minutes: 30\n needs: aarch64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-datascience:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: datascience-notebook\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 25\n needs: x86_64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-pyspark:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: pyspark-notebook\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 20\n needs: aarch64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-pyspark:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: scipy-notebook\n image: pyspark-notebook\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 15\n needs: x86_64-scipy\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n aarch64-all-spark:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: pyspark-notebook\n image: all-spark-notebook\n platform: aarch64\n runs-on: ubuntu-24.04-arm\n timeout-minutes: 20\n needs: aarch64-pyspark\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n x86_64-all-spark:\n uses: ./.github/workflows/docker-build-test-upload.yml\n with:\n parent-image: pyspark-notebook\n image: all-spark-notebook\n platform: x86_64\n runs-on: ubuntu-24.04\n timeout-minutes: 15\n needs: x86_64-pyspark\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n contributed-recipes:\n uses: ./.github/workflows/contributed-recipes.yml\n with:\n called-using-workflow-call: true\n # Contributed recipes only use these images\n # If recipes using other images will be added, they should be added here as well\n #\n # contributed-recipes will give an error if the image is not yet built and uploaded\n needs: [aarch64-base, x86_64-base, aarch64-minimal, x86_64-minimal]\n\n tag-push-merge:\n uses: ./.github/workflows/docker-tag-push-merge.yml\n with:\n image: ${{ matrix.image }}\n variant: ${{ matrix.variant }}\n secrets:\n REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }}\n REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}\n strategy:\n matrix:\n image:\n [\n docker-stacks-foundation,\n base-notebook,\n minimal-notebook,\n scipy-notebook,\n r-notebook,\n julia-notebook,\n tensorflow-notebook,\n pytorch-notebook,\n datascience-notebook,\n pyspark-notebook,\n all-spark-notebook,\n ]\n variant: [default]\n include:\n - image: tensorflow-notebook\n variant: cuda\n - image: pytorch-notebook\n variant: cuda12\n - image: pytorch-notebook\n variant: cuda13\n needs:\n [\n contributed-recipes,\n\n aarch64-foundation,\n aarch64-base,\n aarch64-minimal,\n aarch64-scipy,\n aarch64-r,\n aarch64-julia,\n aarch64-tensorflow,\n aarch64-tensorflow-cuda,\n aarch64-pytorch,\n aarch64-pytorch-cuda12,\n aarch64-pytorch-cuda13,\n aarch64-datascience,\n aarch64-pyspark,\n aarch64-all-spark,\n\n x86_64-foundation,\n x86_64-base,\n x86_64-minimal,\n x86_64-scipy,\n x86_64-r,\n x86_64-julia,\n x86_64-tensorflow,\n x86_64-tensorflow-cuda,\n x86_64-pytorch,\n x86_64-pytorch-cuda12,\n x86_64-pytorch-cuda13,\n x86_64-datascience,\n x86_64-pyspark,\n x86_64-all-spark,\n ]\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n\n tag-push-merge-fast:\n uses: ./.github/workflows/docker-tag-push-merge.yml\n with:\n image: ${{ matrix.image }}\n variant: ${{ matrix.variant }}\n secrets:\n REGISTRY_USERNAME: ${{ secrets.QUAY_USERNAME }}\n REGISTRY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}\n strategy:\n matrix:\n image: [docker-stacks-foundation, base-notebook]\n variant: [default]\n needs: [aarch64-foundation, aarch64-base, x86_64-foundation, x86_64-base]\n if: contains(github.event.pull_request.title, '[FAST_BUILD]')\n\n wiki-update:\n uses: ./.github/workflows/docker-wiki-update.yml\n needs: tag-push-merge\n if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}\n permissions:\n contents: write\n\n wiki-update-fast:\n uses: ./.github/workflows/docker-wiki-update.yml\n needs: tag-push-merge-fast\n if: contains(github.event.pull_request.title, '[FAST_BUILD]')\n permissions:\n contents: write\n" }, { "path": ".github/workflows/pre-commit.yml", "content": "name: Run pre-commit hooks\n\non:\n pull_request:\n push:\n branches:\n - main\n workflow_dispatch:\n\npermissions:\n contents: read\n\njobs:\n run-hooks:\n runs-on: ubuntu-24.04\n timeout-minutes: 5\n\n steps:\n - name: Checkout Repo โšก๏ธ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Set Up Python ๐Ÿ\n uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: 3.12\n\n - name: Install pre-commit ๐Ÿ“ฆ\n run: |\n pip install --upgrade pip\n pip install --upgrade pre-commit\n\n - name: Run pre-commit hooks โœ…\n run: pre-commit run --all-files --hook-stage manual\n" }, { "path": ".github/workflows/registry-move.yml", "content": "name: Move some images from Docker Hub to Quay.io\n\nenv:\n OWNER: ${{ github.repository_owner }}\n PUSH_TO_REGISTRY: ${{ (github.repository_owner == 'jupyter' || github.repository_owner == 'mathbunnyru') && (github.ref == 'refs/heads/main') }}\n\non:\n pull_request:\n paths:\n - \".github/workflows/registry-move.yml\"\n push:\n branches:\n - main\n paths:\n - \".github/workflows/registry-move.yml\"\n workflow_dispatch:\n\npermissions:\n contents: read\n\njobs:\n registry-move:\n # To be able to use the latest skopeo\n runs-on: macos-latest\n timeout-minutes: 5\n if: github.repository_owner == 'jupyter' || github.repository_owner == 'mathbunnyru'\n\n steps:\n - name: Checkout Repo โšก๏ธ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Install skopeo and Docker ๐Ÿ“ฆ\n run: |\n brew install skopeo\n brew install --cask docker\n\n - name: Login to Quay.io ๐Ÿ”\n if: env.PUSH_TO_REGISTRY == 'true'\n run: |\n skopeo login quay.io \\\n --username ${{ secrets.QUAY_USERNAME }} \\\n --password ${{ secrets.QUAY_ROBOT_TOKEN }}\n\n - name: Move image from Docker Hub to Quay.io ๐Ÿณ\n if: env.PUSH_TO_REGISTRY == 'true' && matrix.tag != 'tag-for-testing'\n run: |\n skopeo copy \\\n --multi-arch all \\\n docker://${{ env.OWNER }}/${{ matrix.image }}:${{ matrix.tag }} \\\n docker://quay.io/${{ env.OWNER }}/${{ matrix.image }}:${{ matrix.tag }}\n\n strategy:\n fail-fast: false\n matrix:\n image:\n [\n docker-stacks-foundation,\n base-notebook,\n minimal-notebook,\n scipy-notebook,\n r-notebook,\n julia-notebook,\n tensorflow-notebook,\n pytorch-notebook,\n datascience-notebook,\n pyspark-notebook,\n all-spark-notebook,\n ]\n tag: [tag-for-testing]\n" }, { "path": ".github/workflows/registry-overviews.yml", "content": "name: Update Registry overviews\n\nenv:\n OWNER: ${{ github.repository_owner }}\n\non:\n push:\n branches:\n - main\n paths:\n - \".github/workflows/registry-overviews.yml\"\n\n - \"images/*/README.md\"\n workflow_dispatch:\n\npermissions:\n contents: read\n\njobs:\n update-overview:\n runs-on: ubuntu-24.04\n timeout-minutes: 1\n if: github.repository_owner == 'jupyter' || github.repository_owner == 'mathbunnyru'\n\n steps:\n - name: Checkout Repo โšก๏ธ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Push README to Registry ๐Ÿณ\n uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1\n env:\n DOCKER_APIKEY: ${{ secrets.APIKEY__QUAY_IO }}\n with:\n destination_container_repo: quay.io/${{ env.OWNER }}/${{ matrix.image }}\n provider: quay\n readme_file: images/${{ matrix.image }}/README.md\n\n strategy:\n matrix:\n image:\n [\n docker-stacks-foundation,\n base-notebook,\n minimal-notebook,\n scipy-notebook,\n r-notebook,\n julia-notebook,\n tensorflow-notebook,\n pytorch-notebook,\n datascience-notebook,\n pyspark-notebook,\n all-spark-notebook,\n ]\n" }, { "path": ".github/workflows/sphinx.yml", "content": "name: Build Sphinx Documentation and check links\n\non:\n schedule:\n # Weekly, at 03:00 on Monday UTC\n - cron: \"0 3 * * 1\"\n pull_request:\n paths:\n - \".github/workflows/sphinx.yml\"\n - \"Makefile\"\n\n - \"docs/**\"\n\n # These files are also rendered as docs pages\n - \"README.md\"\n - \"CHANGELOG.md\"\n\n # These files are used to generate some code snippets in the docs\n - \"tagging/manifests/apt_packages.py\"\n - \"tagging/manifests/manifest_interface.py\"\n - \"tagging/taggers/sha.py\"\n - \"tagging/taggers/tagger_interface.py\"\n push:\n branches:\n - main\n paths:\n - \".github/workflows/sphinx.yml\"\n - \"Makefile\"\n\n - \"docs/**\"\n\n - \"README.md\"\n - \"CHANGELOG.md\"\n\n - \"tagging/manifests/apt_packages.py\"\n - \"tagging/manifests/manifest_interface.py\"\n - \"tagging/taggers/sha.py\"\n - \"tagging/taggers/tagger_interface.py\"\n workflow_dispatch:\n\npermissions:\n contents: read\n\njobs:\n build-docs:\n runs-on: ubuntu-24.04\n timeout-minutes: 10\n if: github.repository_owner == 'jupyter' || github.repository_owner == 'mathbunnyru' || github.event_name != 'schedule'\n\n steps:\n - name: Checkout Repo โšก๏ธ\n uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n with:\n fetch-depth: 0\n\n - name: Set Up Python ๐Ÿ\n uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0\n with:\n python-version: 3.12\n\n - name: Install Doc Dependencies ๐Ÿ“ฆ\n run: |\n pip install --upgrade pip\n pip install --upgrade -r docs/requirements.txt\n\n - name: Build Documentation ๐Ÿ“–\n run: make docs\n\n - name: Check Documentation URLs ๐Ÿ”—\n run: make linkcheck-docs || make linkcheck-docs\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\ncover/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\n.pybuilder/\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n# For a library or package, you might want to ignore these files since the code is\n# intended to run in multiple environments; otherwise, check them in:\n# .python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# UV\n# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.\n# This is especially recommended for binary packages to ensure reproducibility, and is more\n# commonly ignored for libraries.\n#uv.lock\n\n# poetry\n# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.\n# This is especially recommended for binary packages to ensure reproducibility, and is more\n# commonly ignored for libraries.\n# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control\n#poetry.lock\n\n# pdm\n# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.\n#pdm.lock\n# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it\n# in version control.\n# https://pdm.fming.dev/latest/usage/project/#working-with-version-control\n.pdm.toml\n.pdm-python\n.pdm-build/\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# pytype static type analyzer\n.pytype/\n\n# Cython debug symbols\ncython_debug/\n\n# PyCharm\n# JetBrains specific template is maintained in a separate JetBrains.gitignore that can\n# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore\n# and can be added to the global gitignore or merged into this file. For a more nuclear\n# option (not recommended) you can uncomment the following to ignore the entire idea folder.\n#.idea/\n\n# Ruff stuff:\n.ruff_cache/\n\n# PyPI configuration file\n.pypirc\n\n##################################################################\n# The content above is copied from #\n# https://github.com/github/gitignore/blob/main/Python.gitignore #\n# Please, add the content only below these lines #\n##################################################################\n\n# Mac OS X\n.DS_Store\n\n# VS Code project configuration\n.vscode/\n\n# PyCharm project configuration\n.idea/\n" }, { "path": ".hadolint.yaml", "content": "---\nignored:\n - DL3006\n - DL3008\n - DL3013\n" }, { "path": ".markdownlint.yaml", "content": "# Default state for all rules\ndefault: true\n\n# MD013/line-length - Line length\nMD013:\n # Number of characters\n line_length: 200\n tables: false\n" }, { "path": ".pre-commit-config.yaml", "content": "---\n# pre-commit is a tool to perform a predefined set of tasks manually and/or\n# automatically before git commits are made.\n#\n# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level\n#\n# Common tasks\n#\n# - Run on all files: pre-commit run --all-files\n# - Register git hooks: pre-commit install --install-hooks\n#\n# See https://pre-commit.com for more information\n# See https://pre-commit.com/hooks.html for more hooks\n\nexclude: ^LICENSE.md$\n\nrepos:\n # Autoupdate: Python code\n - repo: https://github.com/asottile/pyupgrade\n rev: 75992aaa40730136014f34227e0135f63fc951b4 # frozen: v3.21.2\n hooks:\n - id: pyupgrade\n args: [--py312-plus]\n\n # Automatically sort python imports\n - repo: https://github.com/PyCQA/isort\n rev: a333737ed43df02b18e6c95477ea1b285b3de15a # frozen: 8.0.1\n hooks:\n - id: isort\n args: [--profile, black]\n\n # Autoformat: Python code\n - repo: https://github.com/psf/black-pre-commit-mirror\n rev: ea488cebbfd88a5f50b8bd95d5c829d0bb76feb8 # frozen: 26.1.0\n hooks:\n - id: black\n args: [--target-version=py312]\n\n # Check python code static typing\n - repo: https://github.com/pre-commit/mirrors-mypy\n rev: a66e98df7b4aeeb3724184b332785976d062b92e # frozen: v1.19.1\n hooks:\n - id: mypy\n args: [--config, ./mypy.ini]\n additional_dependencies:\n [\n \"beautifulsoup4\",\n \"numpy\",\n \"pytest\",\n \"requests\",\n \"tenacity\",\n \"urllib3\",\n \"types-beautifulsoup4\",\n \"types-python-dateutil\",\n \"types-requests\",\n \"types-tabulate\",\n \"types-urllib3\",\n ]\n # Unfortunately, `pre-commit` only runs on modified files\n # This doesn't work well with `mypy --follow-imports error`\n # See: https://github.com/pre-commit/mirrors-mypy/issues/34#issuecomment-1062160321\n #\n # To work around this we run `mypy` only in manual mode\n # So it won't run as part of `git commit` command,\n # but it will still be run as part of `pre-commit` workflow and give expected results\n stages: [manual]\n\n # Autoformat: YAML, JSON, Markdown, etc.\n - repo: https://github.com/rbubley/mirrors-prettier\n rev: c2bc67fe8f8f549cc489e00ba8b45aa18ee713b1 # frozen: v3.8.1\n hooks:\n - id: prettier\n\n # `pre-commit sample-config` default hooks\n - repo: https://github.com/pre-commit/pre-commit-hooks\n rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0\n hooks:\n - id: check-added-large-files\n - id: check-executables-have-shebangs\n - id: check-shebang-scripts-are-executable\n - id: end-of-file-fixer\n - id: requirements-txt-fixer\n - id: trailing-whitespace\n\n # Lint: Dockerfile\n - repo: https://github.com/hadolint/hadolint\n rev: 4e697ba704fd23b2409b947a319c19c3ee54d24f # frozen: v2.14.0\n hooks:\n - id: hadolint-docker\n entry: hadolint/hadolint:v2.14.0 hadolint\n\n # Lint: Dockerfile\n # We're linting .dockerfile files as well\n - repo: https://github.com/hadolint/hadolint\n rev: 4e697ba704fd23b2409b947a319c19c3ee54d24f # frozen: v2.14.0\n hooks:\n - id: hadolint-docker\n name: Lint *.dockerfile Dockerfiles\n entry: hadolint/hadolint:v2.12.1-beta hadolint\n types: [file]\n files: \\.dockerfile$\n\n # Lint: YAML\n - repo: https://github.com/adrienverge/yamllint\n rev: cba56bcde1fdd01c1deb3f945e69764c291a6530 # frozen: v1.38.0\n hooks:\n - id: yamllint\n args: [\"-d {extends: relaxed, rules: {line-length: disable}}\", \"-s\"]\n\n # Lint: Bash scripts\n - repo: https://github.com/openstack/bashate\n rev: 5798d24d571676fc407e81df574c1ef57b520f23 # frozen: 2.1.1\n hooks:\n - id: bashate\n args: [\"--ignore=E006\"]\n\n # Lint: Shell scripts\n - repo: https://github.com/shellcheck-py/shellcheck-py\n rev: 745eface02aef23e168a8afb6b5737818efbea95 # frozen: v0.11.0.1\n hooks:\n - id: shellcheck\n args: [\"-x\"]\n\n # Lint: Python\n - repo: https://github.com/PyCQA/flake8\n rev: d93590f5be797aabb60e3b09f2f52dddb02f349f # frozen: 7.3.0\n hooks:\n - id: flake8\n\n # Lint: Markdown\n - repo: https://github.com/DavidAnson/markdownlint-cli2\n rev: 5387279b3b4c24822c0f86d4df4f28b37e3e8992 # frozen: v0.21.0\n hooks:\n - id: markdownlint-cli2\n args: [--fix]\n\n # Strip output from Jupyter notebooks\n - repo: https://github.com/kynan/nbstripout\n rev: f5da19ce3b7b40e97c12ee9cd8ce97f48f97ddf7 # frozen: 0.9.1\n hooks:\n - id: nbstripout\n\n # nbQA provides tools from the Python ecosystem like\n # pyupgrade, isort, black, and flake8, adjusted for notebooks.\n - repo: https://github.com/nbQA-dev/nbQA\n rev: f96ec7f3b26a32619435686eb5813235f7e3327e # frozen: 1.9.1\n hooks:\n - id: nbqa-pyupgrade\n args: [--py312-plus]\n - id: nbqa-isort\n - id: nbqa-black\n args: [--target-version=py312]\n - id: nbqa-flake8\n\n # Run black on python code blocks in documentation files.\n - repo: https://github.com/adamchainz/blacken-docs\n rev: dda8db18cfc68df532abf33b185ecd12d5b7b326 # frozen: 1.20.0\n hooks:\n - id: blacken-docs\n # --skip-errors is added to allow us to have python syntax highlighting even if\n # the python code blocks include jupyter-specific additions such as % or !\n # See https://github.com/adamchainz/blacken-docs/issues/127 for an upstream\n # feature request about this.\n args: [--target-version=py312, --skip-errors]\n\n# pre-commit.ci config reference: https://pre-commit.ci/#configuration\nci:\n autoupdate_schedule: monthly\n # Docker hooks do not work in pre-commit.ci\n # See: \n skip: [hadolint-docker]\n" }, { "path": ".readthedocs.yaml", "content": "# Read the Docs configuration file for Sphinx projects\n# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details\n\n# Required\nversion: 2\n\n# Set the OS, Python version and other tools you might need\nbuild:\n os: ubuntu-22.04\n tools:\n python: \"3.12\"\n # You can also specify other tool versions:\n # nodejs: \"20\"\n # rust: \"1.70\"\n # golang: \"1.20\"\n jobs:\n post_checkout:\n - git fetch --unshallow || true\n\n# Build documentation in the \"docs/\" directory with Sphinx\nsphinx:\n configuration: docs/conf.py\n # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs\n # builder: \"dirhtml\"\n # Fail on all warnings to avoid broken references\n # fail_on_warning: true\n\n# Optionally build your docs in additional formats such as PDF and ePub\n# formats:\n# - pdf\n# - epub\n\n# Optional but recommended, declare the Python requirements required\n# to build your documentation\n# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html\npython:\n install:\n - requirements: docs/requirements.txt\n" }, { "path": "CHANGELOG.md", "content": "# Changelog\n\nThis changelog only contains breaking and/or significant changes manually introduced to this repository (using Pull Requests).\nAll image manifests can be found in [the wiki](https://github.com/jupyter/docker-stacks/wiki).\n\n## 2025-12-31\n\nAffected: `pytorch-notebook`.\n\n- **Non-breaking:** `pytorch-notebook`: Build pytorch cuda13 image instead of cuda11 ([#2391](https://github.com/jupyter/docker-stacks/pull/2391))\n\n## 2025-12-02\n\nAffected: `tensorflow-notebook`, `pytorch-notebook`.\n\n- **Non-breaking:** Enable CUDA build for ARM64 ([#2352](https://github.com/jupyter/docker-stacks/pull/2352)).\n\n## 2025-11-29\n\nAffected: all images.\n\n- **Breaking:** Use Docker v29 and `docker buildx imagetools create` ([#2368](https://github.com/jupyter/docker-stacks/pull/2368)).\n\n## 2025-11-24\n\nAffected: all images.\n\n- **Non-breaking:** Add Dev Container support ([#2358](https://github.com/jupyter/docker-stacks/pull/2358)).\n- **Non-breaking:** Add recipe on running Jupyter Docker Stacks with Singularity ([#2357](https://github.com/jupyter/docker-stacks/pull/2357)).\n\n## 2025-11-06\n\nAffected: `scipy-notebook`.\n\n- **Breaking:** `scipy-notebook`: Remove facets package installation ([#2347](https://github.com/jupyter/docker-stacks/pull/2347)).\n\n## 2025-09-16\n\nAffected: all images.\n\n- **Non-breaking:** Publish SBOM using anchore/sbom-action ([#2317](https://github.com/jupyter/docker-stacks/pull/2317)).\n\n## 2025-08-15\n\nAffected: all images.\n\n- **Breaking:** `docker-stacks-foundation`: switch to Python 3.13 ([#2163](https://github.com/jupyter/docker-stacks/pull/2163)).\n\n## 2025-04-13\n\nAffected: `tensorflow-notebook`.\n\n- **Non-breaking:** `tesnorflow-notebook`: Install latest tensorflow ([#2263](https://github.com/jupyter/docker-stacks/pull/2263)).\n\n## 2025-04-12\n\nAffected: all images.\n\n- **Non-breaking:** `docker-stacks-foundation`: Pin libxml2 to avoid ABI breakage ([#2283](https://github.com/jupyter/docker-stacks/pull/2283)).\n\n## 2025-04-11\n\nAffected: all images.\n\n- **Non-breaking:** Make docker tag-push depend on contributed recipes in CI ([#2282](https://github.com/jupyter/docker-stacks/pull/2282)).\n\n## 2025-04-01\n\nAffected: all images.\n\n- **Non-breaking:** Apply and merge tags in the same place ([#2274](https://github.com/jupyter/docker-stacks/pull/2274)).\n\n## 2025-03-23\n\nAffected: `tensorflow-notebook`.\n\n- **Non-breaking:** `tensorflow-notebook`: Use mamba to install jupyter-server-proxy ([#2262](https://github.com/jupyter/docker-stacks/pull/2262)).\n\n## 2025-03-22\n\nAffected: all images.\n\n- **Non-breaking:** Use tty for running docker commands by default ([#2260](https://github.com/jupyter/docker-stacks/pull/2260)).\n- **Non-breaking:** Improve logs around running docker ([#2261](https://github.com/jupyter/docker-stacks/pull/2261)).\n\n## 2025-03-21\n\nAffected: all images.\n\n- **Non-breaking:** Refactor TrackedContainer run_detached/exec_cmd functions ([#2256](https://github.com/jupyter/docker-stacks/pull/2256)).\n- **Non-breaking:** Do not allocate TTY in tests if not needed ([#2257](https://github.com/jupyter/docker-stacks/pull/2257)).\n- **Non-breaking:** `base-notebook`: Flush output in Python before running execvp ([#2258](https://github.com/jupyter/docker-stacks/pull/2258)).\n\n## 2025-03-20\n\nAffected: all images except `docker-stacks-foundation`.\n\n- **Non-breaking:** `base-notebook`: Refactor healthcheck tests to use one function ([#2254](https://github.com/jupyter/docker-stacks/pull/2254)).\n- **Non-breaking:** `base-notebook`: Test server listening on IPv4/IPv6 ([#2255](https://github.com/jupyter/docker-stacks/pull/2255)).\n\n## 2025-03-12\n\nAffected: all images.\n\n- **Non-breaking:** Add `conda` and `mamba` version taggers ([#2251](https://github.com/jupyter/docker-stacks/pull/2251)).\n- **Non-breaking:** Make taggers and manifests functions ([#2252](https://github.com/jupyter/docker-stacks/pull/2252)).\n\n## 2025-02-21\n\nAffected: all images.\n\n- **Non-breaking:** Better tagging directory structure ([#2228](https://github.com/jupyter/docker-stacks/pull/2228)).\n- **Non-breaking:** Better testing directory structure ([#2231](https://github.com/jupyter/docker-stacks/pull/2231)).\n\n## 2025-02-18\n\nAffected: all images.\n\n- **Non-breaking:** switch from `ubuntu-22.04-arm` to `ubuntu-24.04-arm` runners ([#2209](https://github.com/jupyter/docker-stacks/pull/2209)).\n- **Non-breaking:** don't create extra free space in runners for cuda images ([#2218](https://github.com/jupyter/docker-stacks/pull/2218)).\n- **Non-breaking:** revert \"Pin some packages to fix `r-notebook` and `datascience-notebook` under aarch64\" ([#2220](https://github.com/jupyter/docker-stacks/pull/2220)).\n- **Non-breaking:** Simplify and improve `test_packages.py` ([#2219](https://github.com/jupyter/docker-stacks/pull/2219)).\n- **Non-breaking:** Use Python 3.12 for internal code ([#2222](https://github.com/jupyter/docker-stacks/pull/2222)).\n\n## 2025-02-17\n\nAffected: all images.\n\n- **Non-breaking:** build contributed recipes in PRs ([#2212](https://github.com/jupyter/docker-stacks/pull/2212), [#2213](https://github.com/jupyter/docker-stacks/pull/2213)).\n- **Non-breaking:** remove information about Docker Hub images from Quay.io READMEs ([#2211](https://github.com/jupyter/docker-stacks/pull/2211)).\n- **Non-breaking:** first upload artifacts and then run tests to make sure we can easily debug broken images ([#2214](https://github.com/jupyter/docker-stacks/pull/2214)).\n- **Non-Breaking:** aarch64 `r-notebook`, `datascience-notebook`: pin some packages to fix `r-notebook` and `datascience-notebook` under aarch64 ([#2215](https://github.com/jupyter/docker-stacks/pull/2215)).\n- **Non-breaking:** don't use matrix.image-variant, use 2 separate variables ([#2217](https://github.com/jupyter/docker-stacks/pull/2217)).\n\n## 2025-02-11\n\nAffected: all images.\n\n- **Non-breaking:** start using `ubuntu-22.04-arm` GitHub-hosted `aarch64` runners ([#2202](https://github.com/jupyter/docker-stacks/pull/2202)).\n\n## 2024-12-03\n\nAffected: all images.\n\n- **Breaking:** `docker-stacks-foundation`: switch to `mamba` v2 ([#2147](https://github.com/jupyter/docker-stacks/pull/2147)).\n More information about changes made: .\n\n## 2024-11-08\n\nAffected: all images except `docker-stacks-foundation`.\n\n- **Breaking:** `base-notebook`: stop installing `nodejs` from `conda-forge` ([#2172](https://github.com/jupyter/docker-stacks/pull/2172)).\n\n Reason: It isn't a direct dependency on anything in the images anymore, and increased the image size by ~150MB.\n\n## 2024-11-06\n\nAffected: all images except `docker-stacks-foundation`.\n\n- **Non-breaking:** `base-notebook`: install `jupyterhub-base` and `nodejs` packages instead of `jupyterhub` package ([#2171](https://github.com/jupyter/docker-stacks/pull/2171)).\n\n## 2024-10-23\n\nAffected: all images.\n\n- **Breaking:** `docker-stacks-foundation`: switch to Python 3.12 ([#2072](https://github.com/jupyter/docker-stacks/pull/2072)).\n\n## 2024-10-22\n\nAffected: `pyspark-notebook` and `all-spark-notebook` images.\n\n- **Breaking:** `pyspark-notebook`: start using Spark 4.0.0 preview versions ([#2159](https://github.com/jupyter/docker-stacks/pull/2159)).\n `sparklyr` doesn't seem to support Spark v4 yet when using Spark locally.\n\n Reason: Spark v3 is not compatible with Python 3.12, and [the voting group has decided](https://github.com/jupyter/docker-stacks/pull/2072#issuecomment-2414123851) to switch to Spark v4 preview version.\n\n## 2024-10-09\n\nAffected: users building a custom set of images.\n\n- **Breaking:** rename: `ROOT_CONTAINER`->`ROOT_IMAGE`, `BASE_CONTAINER`->`BASE_IMAGE` ([#2154](https://github.com/jupyter/docker-stacks/issues/2154), [#2155](https://github.com/jupyter/docker-stacks/pull/2155)).\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Project `jupyter/docker-stacks` Code of Conduct\n\nPlease see the [Project Jupyter Code of Conduct](https://github.com/jupyter/governance/blob/HEAD/conduct/code_of_conduct.md).\n" }, { "path": "CONTRIBUTING.md", "content": "Thanks for contributing!\nPlease see the **Contributor Guide** section in [the documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/)\nfor information about how to contribute\n[issues](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/issues.html),\n[features](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/features.html),\n[recipes](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/recipes.html),\n[tests](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/tests.html),\nand [community-maintained stacks](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/stacks.html).\n\n## Our Copyright Policy\n\nJupyter uses a shared copyright model. Each contributor maintains copyright\nover their contributions to Jupyter. But, it is important to note that these\ncontributions are typically only changes to the repositories. Thus, the Jupyter\nsource code, in its entirety is not the copyright of any single person or\ninstitution. Instead, it is the collective copyright of the entire Jupyter\nDevelopment Team. If individual contributors want to maintain a record of what\nchanges/contributions they have specific copyright on, they should indicate\ntheir copyright in the commit message of the change, when they commit the\nchange to one of the Jupyter repositories.\n\nWith this in mind, the following banner should be used in any source code file\nto indicate the copyright and license terms:\n\n```text\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n```\n\n\n" }, { "path": "LICENSE.md", "content": "BSD 3-Clause License\n\nCopyright (c) 2001-2015, IPython Development Team\n\nCopyright (c) 2015-, Jupyter Development Team\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "Makefile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n.PHONY: docs help test\n\nSHELL:=bash\nREGISTRY?=quay.io\nOWNER?=jupyter\n\n# Enable BuildKit for Docker build\nexport DOCKER_BUILDKIT:=1\n\n# All the images listed in the build dependency order\nALL_IMAGES:= \\\n\tdocker-stacks-foundation \\\n\tbase-notebook \\\n\tminimal-notebook \\\n\tscipy-notebook \\\n\tr-notebook \\\n\tjulia-notebook \\\n\ttensorflow-notebook \\\n\tpytorch-notebook \\\n\tdatascience-notebook \\\n\tpyspark-notebook \\\n\tall-spark-notebook\n\n\n\n# https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html\nhelp:\n\t@echo \"jupyter/docker-stacks\"\n\t@echo \"=====================\"\n\t@echo \"Replace % with a stack directory name (e.g., make build/minimal-notebook)\"\n\t@echo\n\t@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = \":.*?## \"}; {printf \"\\033[36m%-30s\\033[0m %s\\n\", $$1, $$2}'\n\n\n\n# Note that `ROOT_IMAGE` and `PYTHON_VERSION` arguments are only applicable to the `docker-stacks-foundation` image\nbuild/%: DOCKER_BUILD_ARGS?=\nbuild/%: ROOT_IMAGE?=ubuntu:24.04\nbuild/%: PYTHON_VERSION?=3.13\nbuild/%: ## build the latest image for a stack using the system's architecture\n\tdocker build $(DOCKER_BUILD_ARGS) --rm --force-rm \\\n\t --tag \"$(REGISTRY)/$(OWNER)/$(notdir $@)\" \\\n\t \"./images/$(notdir $@)\" \\\n\t --build-arg REGISTRY=\"$(REGISTRY)\" \\\n\t --build-arg OWNER=\"$(OWNER)\" \\\n\t --build-arg ROOT_IMAGE=\"$(ROOT_IMAGE)\" \\\n\t --build-arg PYTHON_VERSION=\"$(PYTHON_VERSION)\"\n\t@echo -n \"Built image size: \"\n\t@docker images \"$(REGISTRY)/$(OWNER)/$(notdir $@):latest\" --format \"{{.Size}}\"\nbuild-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks\n\n\n\ncheck-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report\n\tpytest tests/by_image/docker-stacks-foundation/test_outdated.py \\\n\t --registry \"$(REGISTRY)\" \\\n\t --owner \"$(OWNER)\" \\\n\t --image \"$(notdir $@)\"\ncheck-outdated-all: $(foreach I, $(ALL_IMAGES), check-outdated/$(I)) ## check all the stacks for outdated packages\n\n\n\ncont-stop-all: ## stop all containers\n\t@echo \"Stopping all containers ...\"\n\t-docker stop --time 0 $(shell docker ps --all --quiet) 2> /dev/null\ncont-rm-all: ## remove all containers\n\t@echo \"Removing all containers ...\"\n\t-docker rm --force $(shell docker ps --all --quiet) 2> /dev/null\ncont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm)\n\n\n\ndocs: ## build HTML documentation\n\tsphinx-build -W --keep-going --color docs/ docs/_build/\nlinkcheck-docs: ## check for broken links\n\tsphinx-build -W --keep-going --color -b linkcheck docs/ docs/_build/\n\n\n\nhook/%: VARIANT?=default\nhook/%: REPOSITORY?=$(OWNER)/docker-stacks\nhook/%: ## run post-build hooks for an image\n\tpython3 -m tagging.apps.write_tags_file \\\n\t --registry \"$(REGISTRY)\" \\\n\t --owner \"$(OWNER)\" \\\n\t --image \"$(notdir $@)\" \\\n\t --variant \"$(VARIANT)\" \\\n\t --tags-dir /tmp/jupyter/tags/\n\tpython3 -m tagging.apps.write_manifest \\\n\t --registry \"$(REGISTRY)\" \\\n\t --owner \"$(OWNER)\" \\\n\t --image \"$(notdir $@)\" \\\n\t --variant \"$(VARIANT)\" \\\n\t --hist-lines-dir /tmp/jupyter/hist_lines/ \\\n\t --manifests-dir /tmp/jupyter/manifests/ \\\n\t --repository \"$(REPOSITORY)\"\n\tpython3 -m tagging.apps.apply_tags \\\n\t --registry \"$(REGISTRY)\" \\\n\t --owner \"$(OWNER)\" \\\n\t --image \"$(notdir $@)\" \\\n\t --variant \"$(VARIANT)\" \\\n\t --platform \"$(shell uname -m)\" \\\n\t --tags-dir /tmp/jupyter/tags/\nhook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images\n\n\n\nimg-list: ## list jupyter images\n\t@echo \"Listing $(OWNER) images ...\"\n\tdocker images \"$(OWNER)/*\"\n\tdocker images \"*/$(OWNER)/*\"\nimg-rm-dang: ## remove dangling images (tagged None)\n\t@echo \"Removing dangling images ...\"\n\t-docker rmi --force $(shell docker images -f \"dangling=true\" --quiet) 2> /dev/null\nimg-rm-jupyter: ## remove jupyter images\n\t@echo \"Removing $(OWNER) images ...\"\n\t-docker rmi --force $(shell docker images --quiet \"$(OWNER)/*\") 2> /dev/null\n\t-docker rmi --force $(shell docker images --quiet \"*/$(OWNER)/*\") 2> /dev/null\nimg-rm: img-rm-dang img-rm-jupyter ## remove dangling and jupyter images\n\n\n\npull/%: ## pull a jupyter image\n\tdocker pull \"$(REGISTRY)/$(OWNER)/$(notdir $@)\"\npull-all: $(foreach I, $(ALL_IMAGES), pull/$(I)) ## pull all images\npush/%: ## push all tags for a jupyter image\n\tdocker push --all-tags \"$(REGISTRY)/$(OWNER)/$(notdir $@)\"\npush-all: $(foreach I, $(ALL_IMAGES), push/$(I)) ## push all tagged images\n\n\n\nrun-shell/%: ## run a bash in interactive mode in a stack\n\tdocker run -it --rm \"$(REGISTRY)/$(OWNER)/$(notdir $@)\" $(SHELL)\nrun-sudo-shell/%: ## run bash in interactive mode as root in a stack\n\tdocker run -it --rm --user root \"$(REGISTRY)/$(OWNER)/$(notdir $@)\" $(SHELL)\n\n\n\ntest/%: ## run tests against a stack\n\tpython3 -m tests.run_tests \\\n\t --registry \"$(REGISTRY)\" \\\n\t --owner \"$(OWNER)\" \\\n\t --image \"$(notdir $@)\"\ntest-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks\n" }, { "path": "README.md", "content": "# Jupyter Docker Stacks\n\n[![GitHub Actions badge](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml/badge.svg)](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml?query=branch%3Amain \"Docker images build status\")\n[![Read the Docs badge](https://img.shields.io/readthedocs/jupyter-docker-stacks.svg)](https://jupyter-docker-stacks.readthedocs.io/en/latest/ \"Documentation build status\")\n[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/jupyter/docker-stacks/main.svg)](https://results.pre-commit.ci/latest/github/jupyter/docker-stacks/main \"pre-commit.ci build status\")\n[![Discourse badge](https://img.shields.io/discourse/users.svg?color=%23f37626&server=https%3A%2F%2Fdiscourse.jupyter.org)](https://discourse.jupyter.org/ \"Jupyter Discourse Forum\")\n[![Binder badge](https://static.mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/jupyter/docker-stacks/main?urlpath=lab/tree/README.ipynb \"Launch a quay.io/jupyter/base-notebook container on mybinder.org\")\n\nJupyter Docker Stacks are a set of ready-to-run [Docker images](https://quay.io/organization/jupyter) containing Jupyter applications and interactive computing tools.\nYou can use a stack image to do any of the following (and more):\n\n- Start a personal Jupyter Server with the JupyterLab frontend (default)\n- Run JupyterLab for a team using JupyterHub\n- Start a personal Jupyter Server with the Jupyter Notebook frontend in a local Docker container\n- Write your own project Dockerfile\n\n## Quick Start\n\nYou can [try the quay.io/jupyter/base-notebook image](https://mybinder.org/v2/gh/jupyter/docker-stacks/main?urlpath=lab/tree/README.ipynb) on .\nOtherwise, the examples below may help you get started if you [have Docker installed](https://docs.docker.com/get-started/get-docker/),\nknow [which Docker image](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html) you want to use, and want to launch a single Jupyter Application in a container.\n\nThe [User Guide on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/) describes additional uses and features in detail.\n\n```{note}\nSince `2023-10-20` our images are only pushed to `Quay.io` registry.\nOlder images are available on Docker Hub, but they will no longer be updated.\n```\n\n### Example 1\n\nThis command pulls the `jupyter/scipy-notebook` image tagged `2025-12-31` from Quay.io if it is not already present on the local host.\nIt then starts a container running a Jupyter Server with the JupyterLab frontend and exposes the container's internal port `8888` to port `10000` of the host machine:\n\n```bash\ndocker run -p 10000:8888 quay.io/jupyter/scipy-notebook:2025-12-31\n```\n\nYou can modify the port on which the container's port is exposed by [changing the value of the `-p` option](https://docs.docker.com/engine/containers/run/#exposed-ports) to `-p 8888:8888`.\n\nVisiting `http://:10000/?token=` in a browser loads JupyterLab,\nwhere:\n\n- The `hostname` is the name of the computer running Docker\n- The `token` is the secret token printed in the console.\n\nThe container remains intact for restart after the Server exits.\n\n### Example 2\n\nThis command pulls the `jupyter/datascience-notebook` image tagged `2025-12-31` from Quay.io if it is not already present on the local host.\nIt then starts an _ephemeral_ container running a Jupyter Server with the JupyterLab frontend and exposes the server on host port 10000.\n\n```bash\ndocker run -it --rm -p 10000:8888 -v \"${PWD}\":/home/jovyan/work quay.io/jupyter/datascience-notebook:2025-12-31\n```\n\nThe use of the `-v` flag in the command mounts the current working directory on the host (`${PWD}` in the example command) as `/home/jovyan/work` in the container.\nThe server logs appear in the terminal.\n\nVisiting `http://:10000/?token=` in a browser loads JupyterLab.\n\nDue to the usage of [the `--rm` flag](https://docs.docker.com/reference/cli/docker/container/run/#rm)\nDocker automatically cleans up the container and removes the file system when the container exits,\nbut any changes made to the `~/work` directory and its files in the container will remain intact on the host.\n[The `-i` flag](https://docs.docker.com/reference/cli/docker/container/run/#interactive) keeps the container's `STDIN` open, and lets you send input to the container through standard input.\n[The `-t` flag](https://docs.docker.com/reference/cli/docker/container/run/#tty) attaches a pseudo-TTY to the container.\n\n```{note}\nBy default, [jupyter's root_dir](https://jupyter-server.readthedocs.io/en/latest/other/full-config.html) is `/home/jovyan`.\nSo, new notebooks will be saved there, unless you change the directory in the file browser.\n\nTo change the default directory, you must specify `ServerApp.root_dir` by adding this line to the previous command: `start-notebook.py --ServerApp.root_dir=/home/jovyan/work`.\n```\n\n## Choosing Jupyter frontend\n\nJupyterLab is the default for all the Jupyter Docker Stacks images.\nIt is still possible to switch back to Jupyter Notebook (or to launch a different startup command).\nYou can achieve this by passing the environment variable `DOCKER_STACKS_JUPYTER_CMD=notebook` (or any other valid `jupyter` subcommand) at container startup;\nmore information is available in the [documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html#alternative-commands).\n\n## Resources\n\n- [Documentation on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/)\n- [Issue Tracker on GitHub](https://github.com/jupyter/docker-stacks/issues)\n- [Jupyter Discourse Forum](https://discourse.jupyter.org/)\n- [Jupyter Website](https://jupyter.org)\n- [Images on Quay.io](https://quay.io/organization/jupyter)\n\n## Acknowledgments\n\n- Starting from `2022-07-05`, `aarch64` self-hosted runners were sponsored by [`@mathbunnyru`](https://github.com/mathbunnyru/).\n Please, consider [sponsoring his work](https://github.com/sponsors/mathbunnyru) on GitHub\n- Starting from `2023-10-31`, `aarch64` self-hosted runners are sponsored by an amazing [`2i2c non-profit organization`](https://2i2c.org)\n- Starting from `2025-02-11`, we use GitHub-hosted `aarch64` runners\n\n## CPU Architectures\n\n- We publish containers for both `x86_64` and `aarch64` platforms\n- Single-platform images have either `aarch64-` or `x86_64-` tag prefixes, for example, `quay.io/jupyter/base-notebook:aarch64-python-3.11.6`\n- Starting from `2022-09-21`, we create multi-platform images (except `tensorflow-notebook`)\n- Starting from `2023-06-01`, we create a multi-platform `tensorflow-notebook` image as well\n- Starting from `2024-02-24`, we create CUDA enabled variants of `pytorch-notebook` image for `x86_64` platform\n- Starting from `2024-03-26`, we create CUDA enabled variant of `tensorflow-notebook` image for `x86_64` platform\n\n## Using old images\n\n[![Python versions badge](https://img.shields.io/badge/python-3.7%20%7C%203.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue.svg)](https://www.python.org/downloads/ \"Python versions supported\")\n\nThis project only builds one set of images at a time.\nIf you want to use the older `Ubuntu` and/or `Python` version, you can use the following images:\n\n| Build Date | Ubuntu | Python | Tag |\n| ------------ | ------ | ------ | -------------- |\n| 2022-10-09 | 20.04 | 3.7 | `1aac87eb7fa5` |\n| 2022-10-09 | 20.04 | 3.8 | `a374cab4fcb6` |\n| 2022-10-09 | 20.04 | 3.9 | `5ae537728c69` |\n| 2022-10-09 | 20.04 | 3.10 | `f3079808ca8c` |\n| 2022-10-09 | 22.04 | 3.7 | `b86753318aa1` |\n| 2022-10-09 | 22.04 | 3.8 | `7285848c0a11` |\n| 2022-10-09 | 22.04 | 3.9 | `ed2908bbb62e` |\n| 2023-05-30 | 22.04 | 3.10 | `4d70cf8da953` |\n| 2024-08-26 | 22.04 | 3.11 | `00987883e58d` |\n| 2024-10-22 | 24.04 | 3.11 | `b74418220768` |\n| 2025-08-11 | 24.04 | 3.12 | `82d322f00937` |\n| weekly build | 24.04 | 3.13 | `latest` |\n\n## Contributing\n\nPlease see the [the documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/)\nfor information about how to contribute\n[issues](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/issues.html),\n[features](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/features.html),\n[recipes](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/recipes.html),\n[tests](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/tests.html),\nand [community-maintained stacks](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/stacks.html).\n\n## LICENSE\n\nThis project is licensed under the terms of the Modified BSD License (also known as New or Revised or 3-Clause BSD).\n\n## About the Jupyter Development Team\n\nThe Jupyter Development Team is the set of all contributors to the Jupyter project.\nThis includes all of the Jupyter subprojects.\n\nThe core team that coordinates development on GitHub can be found here:\n.\n\n## Our Copyright Policy\n\nJupyter uses a shared copyright model. Each contributor maintains copyright\nover their contributions to Jupyter. But, it is important to note that these\ncontributions are typically only changes to the repositories. Thus, the Jupyter\nsource code, in its entirety is not the copyright of any single person or\ninstitution. Instead, it is the collective copyright of the entire Jupyter\nDevelopment Team. If individual contributors want to maintain a record of what\nchanges/contributions they have specific copyright on, they should indicate\ntheir copyright in the commit message of the change, when they commit the\nchange to one of the Jupyter repositories.\n\nWith this in mind, the following banner should be used in any source code file\nto indicate the copyright and license terms:\n\n```text\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n```\n\n## Alternatives\n\n- [b-data](https://github.com/b-data)'s JupyterLab docker stacks - For\n [R](https://github.com/b-data/jupyterlab-r-docker-stack),\n [Python](https://github.com/b-data/jupyterlab-python-docker-stack),\n [MAX/Mojo](https://github.com/b-data/jupyterlab-mojo-docker-stack) and\n [Julia](https://github.com/b-data/jupyterlab-julia-docker-stack).\n With [code-server](https://github.com/coder/code-server) next to JupyterLab.\n Just Python โ€“ no [Conda](https://github.com/conda/conda) /\n [Mamba](https://github.com/mamba-org/mamba).\n- [rocker/binder](https://rocker-project.org/images/versioned/binder.html) -\n From the R focused [rocker-project](https://rocker-project.org),\n lets you run both RStudio and Jupyter either standalone or in a JupyterHub\n- [jupyter/repo2docker](https://github.com/jupyterhub/repo2docker) -\n Turn git repositories into Jupyter-enabled Docker Images\n- [openshift/source-to-image](https://github.com/openshift/source-to-image) -\n A tool for building artifacts from source code and injecting them into docker images\n- [jupyter-on-openshift/jupyter-notebooks](https://github.com/jupyter-on-openshift/jupyter-notebooks) -\n OpenShift compatible S2I builder for basic notebook images\n" }, { "path": "SECURITY.md", "content": "# Security Policy\n\n## Supported Versions\n\nJupyter Docker Stacks only provides security updates for the latest version of each image.\n\n## Reporting a Vulnerability\n\nThe Jupyter Vulnerability Handling Process is described in detail in the [security documentation](https://github.com/jupyter/security/blob/main/docs/vulnerability-handling.md).\n" }, { "path": "binder/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# https://quay.io/repository/jupyter/base-notebook?tab=tags\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/base-notebook:2025-12-31\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nENV TAG=\"2025-12-31\"\n\nCOPY --chown=${NB_UID}:${NB_GID} binder/README.ipynb \"${HOME}\"/README.ipynb\n\nRUN jupyter labextension disable \"@jupyterlab/apputils-extension:announcements\"\n" }, { "path": "binder/README.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# jupyter/base-notebook on Binder\\n\",\n \"\\n\",\n \"Run the cells below to inspect what's in the [jupyter/base-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-base-notebook) image from the Jupyter Docker Stacks project.\\n\",\n \"\\n\",\n \"You can launch the classic notebook interface in Binder by replacing `lab/tree/*` with `tree/` in the JupyterLab URL.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"\\n\",\n \"print(\\n\",\n \" f\\\"This container is using tag {os.environ['TAG']} of the jupyter/base-notebook image\\\"\\n\",\n \")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"The Server is running as the following user.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!id\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Here's the contents of that user's home directory, the default notebook directory for the server.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!ls -al\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"`mamba` is available in the user's path.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!which mamba\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"The user has read/write access to the root mamba environment.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!ls -l /opt/conda\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"The following packages are mamba-installed in the base image to support [Jupyter Notebook](https://github.com/jupyter/notebook), [JupyterLab](https://github.com/jupyterlab/jupyterlab), and their use in [JupyterHub](https://github.com/jupyterhub/jupyterhub) environments (e.g., [MyBinder](https://mybinder.org/)).\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"!mamba list\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Other images in the [jupyter/docker-stacks project](https://github.com/jupyter/docker-stacks) include additional libraries. See the [Jupyter Docker Stacks documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/) for full details.\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.4\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n" }, { "path": "docs/conf.py", "content": "# Configuration file for the Sphinx documentation builder.\n#\n# For the full list of built-in configuration values, see the documentation:\n# https://www.sphinx-doc.org/en/master/usage/configuration.html\n\n# -- Project information -----------------------------------------------------\n# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information\n\nproject = \"docker-stacks\"\ncopyright = \"2025, Project Jupyter\"\nauthor = \"Project Jupyter\"\n\nversion = \"latest\"\nrelease = \"latest\"\n\n# -- General configuration ---------------------------------------------------\n# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration\n\nextensions = []\n\ntemplates_path = [\"_templates\"]\nexclude_patterns = [\"_build\", \"Thumbs.db\", \".DS_Store\"]\n\nlanguage = \"en\"\n\n# -- Options for HTML output -------------------------------------------------\n# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output\n\nhtml_theme = \"alabaster\"\nhtml_static_path = [\"_static\"]\n\n# The file above was generated using sphinx 8.1.3 with this command:\n# sphinx-quickstart --project \"docker-stacks\" --author \"Project Jupyter\" -v \"latest\" -r \"latest\" -l en --no-sep --no-makefile --no-batchfile\n# These are custom options for this project\n\nhtml_theme = \"sphinx_book_theme\"\nhtml_title = \"Docker Stacks documentation\"\nhtml_logo = \"_static/jupyter-logo.svg\"\nhtml_theme_options = {\n \"logo\": {\n \"text\": html_title,\n },\n \"navigation_with_keys\": False,\n \"path_to_docs\": \"docs\",\n \"repository_branch\": \"main\",\n \"repository_url\": \"https://github.com/jupyter/docker-stacks\",\n \"use_download_button\": True,\n \"use_edit_page_button\": True,\n \"use_issues_button\": True,\n \"use_repository_button\": True,\n}\nhtml_last_updated_fmt = \"%Y-%m-%d\"\n\nextensions = [\"myst_parser\", \"sphinx_copybutton\", \"sphinx_last_updated_by_git\"]\nsource_suffix = {\n \".rst\": \"restructuredtext\",\n \".md\": \"markdown\",\n}\npygments_style = \"sphinx\"\n\n# MyST configuration reference: https://myst-parser.readthedocs.io/en/latest/configuration.html\nmyst_heading_anchors = 3\n\nlinkcheck_ignore = [\n r\".*github\\.com.*#\", # javascript based anchors\n r\"http://127\\.0\\.0\\.1:.*\", # various examples\n r\"https://mybinder\\.org/v2/gh/.*\", # lots of 500 errors\n r\"https://packages\\.ubuntu\\.com/search\\?keywords=openjdk\", # frequent read timeouts\n r\"https://anaconda\\.org\\/conda-forge\", # frequent read timeouts\n]\n\nlinkcheck_allowed_redirects = {\n r\"https://results\\.pre-commit\\.ci/latest/github/jupyter/docker-stacks/main\": r\"https://results\\.pre-commit\\.ci/run/github/.*\", # Latest main CI build\n r\"https://github\\.com/jupyter/docker-stacks/issues/new.*\": r\"https://github\\.com/login.*\", # GitHub wants user to be logon to use this features\n r\"https://github\\.com/orgs/jupyter/teams/docker-image-maintainers/members\": r\"https://github\\.com/login.*\",\n}\n" }, { "path": "docs/contributing/features.md", "content": "# New Features\n\nThank you for contributing to the Jupyter Docker Stacks!\nWe review pull requests for new features (e.g., new packages, new scripts, new flags)\nto balance the value of the images to the Jupyter community with the cost of maintaining the images over time.\n\n## Suggesting a New Feature\n\nPlease follow the process below to suggest a new feature for inclusion in one of the core stacks:\n\n1. Open a [GitHub feature request issue](https://github.com/jupyter/docker-stacks/issues/new?template=feature_request.yml)\n describing the feature you'd like to contribute.\n2. Discuss with the maintainers whether the addition makes sense\n in [one of the core stacks](../using/selecting.md#core-stacks),\n as a [way to build a custom set of images](../using/custom-images.md),\n as a [recipe in the documentation](recipes.md),\n as a [community stack](stacks.md),\n or as something else entirely.\n\n## Selection Criteria\n\nRoughly speaking, we evaluate new features based on the following criteria:\n\n- **Usefulness to Jupyter users**:\n Is the feature generally applicable across domains?\n Does it work with JupyterLab, Jupyter Notebook, JupyterHub, etc.?\n- **Fit with the image purpose**:\n Does the feature match the theme of the stack to which it will be added?\n Would it fit better in a new community stack?\n- **The complexity of build/runtime configuration**:\n How many lines of code does the feature require in one of the Dockerfiles or startup scripts?\n Does it require new scripts entirely?\n Do users need to adjust how they use the images?\n- **Impact on image metrics**:\n How many bytes does the feature and its dependencies add to the image(s)?\n How many minutes do they add to the build time?\n- **Ability to support the addition**:\n Can existing maintainers answer user questions and address future build issues?\n Are the contributors interested in helping with long-term maintenance?\n Can we write tests to ensure the feature continues to work over the years?\n\n## Submitting a Pull Request\n\nIf there's agreement that the feature belongs in one or more of the core stacks:\n\n1. Implement the feature in a local clone of the `jupyter/docker-stacks` project.\n2. Please, build the image locally before submitting a pull request.\n It shortens the debugging cycle by taking some load off GitHub Actions,\n which graciously provides free build services for open-source projects like this one.\n If you use `make`, call:\n\n ```bash\n make build/\n ```\n\n3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) (PR) with your changes.\n4. Watch for GitHub to report a build success or failure for your PR on GitHub.\n5. Discuss changes with the maintainers and address any build issues.\n" }, { "path": "docs/contributing/issues.md", "content": "# Project Issues\n\nWe appreciate you taking the time to report an issue you encountered while using the Jupyter Docker Stacks.\nPlease review the following guidelines when reporting your problem.\n\n- Please use GitHub's \"Report a Vulnerability\" button under Security > Advisories on the appropriate repo,\n e.g. [report here for Jupyter Docker Stacks](https://github.com/jupyter/docker-stacks/security/advisories).\n You may also send an email to , but the GitHub reporting system is preferred.\n- If you think your problem is unique to the Jupyter Docker Stacks images,\n please search the [jupyter/docker-stacks issue tracker](https://github.com/jupyter/docker-stacks/issues?q=is%3Aissue%20)\n to see if someone else has already reported the same problem.\n If not, please open a [GitHub bug report issue](https://github.com/jupyter/docker-stacks/issues/new?template=bug_report.yml)\n and provide all the information requested in the issue template.\n Additionally, check the [Troubleshooting Common Problems](../using/troubleshooting.md) page in the documentation before submitting an issue.\n- If the issue you're seeing is with one of the open-source libraries included in the Docker images and is reproducible outside the images,\n please file a bug with the appropriate open-source project.\n- If you have a general question about how to use the Jupyter Docker Stacks in your environment,\n in conjunction with other tools, customizations, and so on,\n please post your question on the [Jupyter Discourse site](https://discourse.jupyter.org).\n" }, { "path": "docs/contributing/lint.md", "content": "# Lint\n\nTo enforce some rules, **linters** are used in this project.\nLinters can be run either during the **development phase** (by the developer) or the **integration phase** (by GitHub Actions).\nTo integrate and enforce this process in the project lifecycle, we are using **git hooks** through [pre-commit](https://pre-commit.com/).\n\n## Using pre-commit hooks\n\n### Pre-commit hook installation\n\n_pre-commit_ is a Python package that needs to be installed.\nTo achieve this, use the generic task to install all Python development dependencies.\n\n```sh\n# Install all development dependencies for the project\npip install --upgrade -r requirements-dev.txt\n# It can also be installed directly\npip install pre-commit\n```\n\nThen the git hooks scripts configured for the project in `.pre-commit-config.yaml` need to be installed in the local git repository.\n\n```sh\npre-commit install\n```\n\n### Run\n\nNow, _pre-commit_ (and so configured hooks) will run automatically on `git commit` on each changed file.\nHowever, you can also run it against all files manually.\n\n```{note}\nHadolint pre-commit uses Docker to run, so `docker` should be running while executing this command.\n```\n\n```sh\npre-commit run --all-files --hook-stage manual\n```\n\n```{note}\nWe're running `pre-commit` with `--hook-stage manual`, because `pre-commit` is run on modified files only, which doesn't work well with `mypy --follow-imports error`.\nMore information can be found in [`.pre-commit-config.yaml` file](https://github.com/jupyter/docker-stacks/blob/main/.pre-commit-config.yaml)\n```\n\n## Image Lint\n\nTo comply with [Docker best practices](https://docs.docker.com/build/building/best-practices/),\nwe are using the [Hadolint](https://github.com/hadolint/hadolint) tool to analyze each `Dockerfile`.\n\n### Ignoring Rules\n\nSometimes it is necessary to ignore [some rules](https://github.com/hadolint/hadolint#rules).\nThe following rules are ignored by default for all images in the `.hadolint.yaml` file.\n\n- [`DL3006`][dl3006]: We use a specific policy to manage image tags.\n - The `docker-stacks-foundation` `FROM` clause is fixed but based on an argument (`ARG`).\n - Building downstream images from (`FROM`) the latest is done on purpose.\n- [`DL3008`][dl3008]: System packages are always updated (`apt-get`) to the latest version.\n- [`DL3013`][dl3013]: We always install the latest packages using `pip`\n\nThe preferred way to ignore other rules is to flag them in the `Dockerfile`.\nYou can use a special comment directly above the Dockerfile instruction you want to make an exception for.\nIgnore rule comments look like `# hadolint ignore=DL3001,SC1081`.\nFor example:\n\n```dockerfile\nFROM ubuntu\n\n# hadolint ignore=DL3003,SC1035\nRUN cd /tmp && echo \"hello!\"\n```\n\n[dl3006]: https://github.com/hadolint/hadolint/wiki/DL3006\n[dl3008]: https://github.com/hadolint/hadolint/wiki/DL3008\n[dl3013]: https://github.com/hadolint/hadolint/wiki/DL3013\n" }, { "path": "docs/contributing/packages.md", "content": "# Package Updates\n\nGenerally, we do not pin package versions in our `Dockerfile`s.\nDependency resolution is a difficult thing to do.\nAll this means that packages might have old versions.\nImages are rebuilt weekly, so usually, packages receive updates quite frequently.\n\n```{note}\nWe pin major.minor version of Python, so this will stay the same even after invoking the `mamba update` command.\n```\n\n## Outdated packages\n\nTo help identify packages that can be updated, you can use the following helper tool.\nIt will list all the outdated packages installed in the `Dockerfile`.\nDependencies are filtered to display only the requested packages.\n\n```bash\nmake check-outdated/base-notebook\n\n# INFO test_outdated:test_outdated.py:80 3/8 (38%) packages could be updated\n# INFO test_outdated:test_outdated.py:82\n# Package Current Newest\n# ---------- --------- --------\n# conda 4.7.12 4.8.2\n# jupyterlab 1.2.5 2.0.0\n# python 3.7.4 3.8.2\n```\n" }, { "path": "docs/contributing/recipes.md", "content": "# New Recipes\n\nWe welcome contributions of [recipes](../using/recipes.md), which are short examples of using, configuring, or extending the Docker Stacks for inclusion in the documentation site.\nFollow the process below to add a new recipe:\n\n1. Open the `docs/using/recipes.md` source file.\n2. Add a second-level Markdown heading naming your recipe at the bottom of the file (e.g., `## Slideshows with JupyterLab and RISE`)\n3. Write the body of your recipe under the heading, including whatever command line, links, etc. you need.\n4. If you have a Dockerfile, please put it in a `recipe_code` subdirectory.\n This file will be built automatically by [contributed-recipes workflow](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/contributed-recipes.yml).\n5. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) (PR) with your changes.\n Maintainers will respond and work with you to address any formatting or content issues.\n" }, { "path": "docs/contributing/stacks.md", "content": "# Community Stacks\n\nWe love to see the community create and share new Jupyter Docker images.\nWe've put together a [cookiecutter project](https://github.com/jupyter/cookiecutter-docker-stacks)\nand the documentation below to help you get started defining, building, and sharing your Jupyter environments in Docker.\n\nFollowing these steps will:\n\n1. Set up a project on GitHub containing a Dockerfile based on any image we provide.\n2. Configure GitHub Actions to build and test your image when users submit pull requests to your repository.\n3. Configure Quay.io to host your images for others to use.\n4. Update the [list of community stacks](../using/selecting.md#community-stacks) in this documentation to include your image.\n\nThis approach mirrors how we build and share the core stack images.\nFeel free to follow it or pave your path using alternative services and build tools.\n\n## Creating a Project\n\nFirst, install [cookiecutter](https://github.com/cookiecutter/cookiecutter) using _pip_ or _mamba_:\n\n```bash\npip install cookiecutter # or mamba install cookiecutter\n```\n\nRun the cookiecutter command pointing to the [jupyter/cookiecutter-docker-stacks](https://github.com/jupyter/cookiecutter-docker-stacks) project on GitHub.\n\n```bash\ncookiecutter https://github.com/jupyter/cookiecutter-docker-stacks.git\n```\n\nEnter a name for your new stack image.\nThis will serve as both the git repository name and the part of the Docker image name after the slash.\n\n```text\nstack_name [my-jupyter-stack]:\n```\n\nEnter the user or organization name under which this stack will reside on Docker Hub.\nYou must have access to manage this Docker Hub organization to push images here.\n\n```text\nstack_org [my-project]:\n```\n\nSelect an image from the `jupyter/docker-stacks` project that will serve as the base for your new image.\n\n```text\nstack_base_image [quay.io/jupyter/base-notebook]:\n```\n\nEnter a longer description of the stack for your README.\n\n```text\nstack_description [my-jupyter-stack is a community-maintained Jupyter Docker Stack image]:\n```\n\nCreate a GitHub repository to store your project.\nInitialize your project as a Git repository and push it to GitHub.\n\n```bash\ncd \n\ngit init\ngit add .\ngit commit -m 'Seed repo'\ngit remote add origin \ngit push -u origin main\n```\n\n## Exploring GitHub Actions\n\n1. By default, the newly `.github/workflows/docker.yaml` will trigger the CI pipeline whenever you push to your `main` branch\n and when any Pull Requests are made to your repository.\n For more details on this configuration, visit the [GitHub Actions documentation on triggers](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows).\n\n2. Go to your repository and click on the **Actions** tab.\n From there, you can click on the workflows on the left-hand side of the screen.\n\n ![GitHub page for jupyter/docker-stacks with the Actions tab active and a rectangle around the \"Build Docker Images\" workflow in the UI](../_static/contributing/stacks/github-actions-tab.png)\n\n ```{note}\n The first run is expected to fail because we haven't yet added Docker credentials to push the image\n ```\n\n3. In the next screen, you will see information about the workflow run and duration.\n If you click the button with the workflow name again, you will see the logs for the workflow steps.\n\n ![GitHub Actions page showing the \"Build Docker Images\" workflow](../_static/contributing/stacks/github-actions-workflow.png)\n\n## Configuring Docker Hub\n\n```{note}\nJupyter Docker Stacks are hosted on Quay.io, but in this example, we show you how to host your image on Docker Hub.\n```\n\nNow, configure Docker Hub to build your stack image and push it to the Docker Hub repository whenever\nyou merge a GitHub pull request to the main branch of your project.\n\n1. Visit [https://hub.docker.com/](https://hub.docker.com/) and log in.\n2. Create a new repository - make sure to use the correct namespace (account or organization).\n Enter the name of the image matching the one you entered when prompted with `stack_name` by the cookiecutter.\n\n ![Docker Hub - 'Create repository' page with the name field set to \"My specialized jupyter stack\"](../_static/contributing/stacks/docker-repo-name.png)\n\n3. Enter a description for your image.\n4. Click on your avatar in the top-right corner and select Account Settings.\n\n ![The Docker Hub page zoomed into the user's settings and accounts menu](../_static/contributing/stacks/docker-user-dropdown.png)\n\n5. Click on **Security** and then click on the **New Access Token** button.\n\n ![Docker Hub - Account page with the \"Security\" tab active and a rectangle highlighting the \"New Access Token\" button in the UI](../_static/contributing/stacks/docker-org-security.png)\n\n6. Enter a meaningful name for your token and click on **Generate**\n\n ![Docker Hub - New Access Token page with the name field set to \"test-stack token\"](../_static/contributing/stacks/docker-org-create-token.png)\n\n7. Copy the personal access token displayed on the next screen.\n\n ```{note}\n **You will not be able to see it again after you close the pop-up window**.\n ```\n\n8. Head back to your GitHub repository and click on the **Settings tab**.\n9. Click on the **Secrets and variables->Actions** section and then on the **New repository secret** button in the top right corner.\n\n ![GitHub page with the \"Setting\" tab active and a rectangle highlighting the \"New repository secret\" button in the UI](../_static/contributing/stacks/github-create-secrets.png)\n\n10. Create a **DOCKERHUB_TOKEN** secret and paste the Personal Access Token from Docker Hub in the **value** field.\n\n ![GitHub - Actions/New secret page with the Name field set to \"DOCKERHUB_TOKEN\"](../_static/contributing/stacks/github-secret-token.png)\n\n11. Now you're ready to go and you can restart a failed workflow.\n\n## Defining Your Image\n\nMake edits to the Dockerfile in your project to add third-party libraries and configure Jupyter applications.\nRefer to the Dockerfiles for the core stacks (e.g., [jupyter/datascience-notebook](https://github.com/jupyter/docker-stacks/blob/main/images/datascience-notebook/Dockerfile))\nto get a feel for what's possible and the best practices.\n\n[Submit pull requests](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)\nto your project repository on GitHub.\nEnsure your image builds correctly on GitHub Actions before merging to the main branch.\nAfter merging to the main branch, your image will be built and pushed to the Docker Hub automatically.\n\n## Sharing Your Image\n\nFinally, if you'd like to add a link to your project to this documentation site, please do the following:\n\n1. Fork the [jupyter/docker-stacks](https://github.com/jupyter/docker-stacks) GitHub repository.\n2. Open the `docs/using/selecting.md` source file and locate the **Community Stacks** section in your fork.\n3. Add a table entry with a link to your project, a binder link, and a short description of what your Docker image contains.\n4. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) (PR) with your changes.\n Maintainers will respond and work with you to address any formatting or content issues.\n" }, { "path": "docs/contributing/tests.md", "content": "# Image Tests\n\nWe greatly appreciate Pull Requests that extend the automated tests that vet the basic functionality of the Docker images.\n\n## How the Tests Work\n\nA [GitHub Actions workflow](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/docker-build-test-upload.yml)\nruns tests against pull requests submitted to the `jupyter/docker-stacks` repository.\n\nWe use the `pytest` module to run tests on the image.\n`conftest.py` and `pytest.ini` in the `tests` folder define the environment in which tests are run.\n[Read `pytest` documentation](https://docs.pytest.org/en/latest/contents.html).\n\nThe actual image-specific test files are located in folders like `tests/by_image//` (e.g., `tests/by_image/docker-stacks-foundation/`, etc.).\n\n```{note}\nIf your test is located in `tests/by_image//`, it will be run against the `jupyter/` image and against all the [images inherited from this image](../using/selecting.md#image-relationships).\n```\n\nMany tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/reference/fixtures.html)\ndefined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/main/tests/conftest.py) file.\n\n## Unit tests\n\nYou can add a unit test if you want to run a Python script in one of our images.\nYou should create a `tests/by_image//units/` directory, if it doesn't already exist, and put your file there.\nFiles in this folder will be executed in the container when tests are run.\nYou can see an [TensorFlow package example here](https://github.com/jupyter/docker-stacks/blob/HEAD/tests/by_image/tensorflow-notebook/units/unit_tensorflow.py).\n\n## Contributing New Tests\n\nPlease follow the process below to add new tests:\n\n1. Add your test code to one of the modules in the `tests/by_image//` directory or create a new module.\n2. Build one or more images you intend to test and run the tests locally.\n If you use `make`, call:\n\n ```bash\n make build/\n make test/\n ```\n\n3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)\n (PR) with your changes.\n4. Watch for GitHub to report a build success or failure for your PR on GitHub.\n5. Discuss changes with the maintainers and address any issues running the tests on GitHub.\n" }, { "path": "docs/index.rst", "content": ".. include:: ../README.md\n :parser: myst_parser.sphinx_\n\nTable of Contents\n-----------------\n\n.. toctree::\n :maxdepth: 2\n :caption: User Guide\n\n using/selecting\n using/running\n using/common\n using/specifics\n using/recipes\n using/custom-images\n using/troubleshooting\n using/faq\n using/changelog\n\n.. toctree::\n :maxdepth: 2\n :caption: Contributor Guide\n\n contributing/issues\n contributing/features\n contributing/tests\n contributing/lint\n contributing/recipes\n contributing/stacks\n contributing/packages\n\n.. toctree::\n :maxdepth: 2\n :caption: Maintainer Guide\n\n maintaining/new-images-and-packages-policy\n maintaining/tagging\n maintaining/tasks\n\n.. toctree::\n :maxdepth: 2\n :caption: Getting Help\n\n Issue Tracker on GitHub \n Jupyter Discourse Forum \n Jupyter Website \n" }, { "path": "docs/maintaining/new-images-and-packages-policy.md", "content": "# New images / packages policy\n\nThere are many things we consider while adding new images and packages.\n\nHere is a non-exhaustive list of things we do care about:\n\n1. **Software health**, details, and maintenance status\n - reasonable versioning is adopted, and the version is considered to be stable\n - has been around for several years\n - the package maintains documentation\n - a changelog is actively maintained\n - a release procedure with helpful automation is established\n - multiple people are involved in the maintenance of the project\n - provides a `conda-forge` package besides a `pypi` package, where both are kept up to date\n - supports both `x86_64` and `aarch64` architectures\n2. **Installation consequences**\n - GitHub Actions build time\n - Image sizes\n - All requirements should be installed as well\n3. Jupyter Docker Stacks _**image fit**_\n - new package or stack is changing (or inherits from) the most suitable stack\n4. **Software impact** for users of docker-stacks images\n - How this image can help existing users, or maybe reduce the need to build new images\n5. Why it shouldn't just be a documented **recipe**\n6. Impact on **security**\n - Does the package open additional ports, or add new web endpoints, that could be exploited?\n\nWith all this in mind, we have a voting group, that consists of\n[@mathbunnyru](https://github.com/mathbunnyru),\n[@consideRatio](https://github.com/consideRatio),\n[@yuvipanda](https://github.com/yuvipanda), and\n[@manics](https://github.com/manics).\n\nThis voting group is responsible for accepting or declining new packages and stacks.\nThe change is accepted, if there are **at least 2 positive votes**.\n" }, { "path": "docs/maintaining/tagging.md", "content": "# Tags and manifests\n\nThe main purpose of the source code in [the `tagging` folder](https://github.com/jupyter/docker-stacks/tree/main/tagging) is to\nproperly write tags file, build history line and manifest for a single-platform image,\napply these tags, and then merge single-platform images into one multi-arch image.\n\n## What is a tag and a manifest\n\nA tag is a label attached to a Docker image identifying specific attributes or versions.\nFor example, an image `jupyter/base-notebook` with Python 3.10.5 will have a full image name `quay.io/jupyter/base-notebook:python-3.10.5`.\nThese tags are pushed to our [Quay.io registry](https://quay.io/organization/jupyter).\n\nA manifest is a description of important image attributes written in Markdown format.\nFor example, we dump all `conda` packages with their versions into the manifest.\n\n## Main principles\n\n- All images are organized in a hierarchical tree.\n More info on [image relationships](../using/selecting.md#image-relationships).\n- `TaggerInterface` and `ManifestInterface` are interfaces for functions to generate tags and manifest pieces by running commands in Docker containers.\n- Tags and manifests are reevaluated for each image in the hierarchy since values may change between parent and child images.\n- To tag an image and create its manifest and build history line, run `make hook/` (e.g., `make hook/base-notebook`).\n\n## Utils\n\n### DockerRunner\n\n`DockerRunner` is a helper class to easily run a docker container and execute commands inside this container:\n\n```{literalinclude} tagging_examples/docker_runner.py\n:language: py\n:lines: 3-\n```\n\n### GitHelper\n\n`GitHelper` methods are run in the current `git` repo and give the information about the last commit hash and commit message:\n\n```{literalinclude} tagging_examples/git_helper.py\n:language: py\n:lines: 3-\n```\n\nThe prefix of commit hash (namely, 12 letters) is used as an image tag to make it easy to inherit from a fixed version of a docker image.\n\n## Taggers and Manifests\n\n### Tagger\n\n`Tagger` is a function that runs commands inside a docker container to calculate a tag for an image.\n\nAll the taggers follow `TaggerInterface`:\n\n```{literalinclude} ../../tagging/taggers/tagger_interface.py\n:language: py\n:start-at: TaggerInterface\n```\n\nSo, the `tagger(container)` gets a docker container as an input and returns a tag.\n\nFor example:\n\n```{literalinclude} ../../tagging/taggers/sha.py\n:language: py\n:start-at: def\n```\n\n- `taggers/` subdirectory contains all taggers.\n- `apps/write_tags_file.py`, `apps/apply_tags.py`, and `apps/merge_tags.py` are Python executables used to write tags for an image, apply tags from a file, and create multi-arch images.\n\n### Manifest\n\nAll manifest functions except `build_info_manifest` follow `ManifestInterface`\nand `manifest(container)` method returns a piece of the manifest.\n\n```{literalinclude} ../../tagging/manifests/manifest_interface.py\n:language: py\n:start-at: ManifestInterface\n```\n\nFor example:\n\n```{literalinclude} ../../tagging/manifests/apt_packages.py\n:language: py\n:start-at: def\n```\n\nwhere:\n\n- `quoted_output(container, cmd)` simply runs the command inside a container using `DockerRunner.exec_cmd` and wraps it to triple quotes to create a valid markdown piece.\n It also adds the command which was run to the markdown piece.\n- `manifests/` subdirectory contains all the manifests.\n- `apps/write_manifest.py` is a Python executable to create the build manifest and history line for an image.\n\n## Images Hierarchy\n\nAll images' dependencies on each other and what taggers and manifests are applicable to them are defined in `hierarchy/images_hierarchy.py`.\n\n`hierarchy/get_taggers.py` and `hierarchy/get_manifests.py` define functions to get the taggers and manifests for a specific image.\n" }, { "path": "docs/maintaining/tagging_examples/docker_runner.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom tagging.utils.docker_runner import DockerRunner\n\nwith DockerRunner(\"ubuntu\") as container:\n DockerRunner.exec_cmd(container, cmd=\"env\")\n" }, { "path": "docs/maintaining/tagging_examples/git_helper.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom tagging.utils.git_helper import GitHelper\n\nprint(\"Git hash:\", GitHelper.commit_hash())\nprint(\"Git message:\", GitHelper.commit_message())\n" }, { "path": "docs/maintaining/tasks.md", "content": "# Maintainer Playbook\n\n## Merging Pull Requests\n\nTo build new images and publish them to the Registry, do the following:\n\n1. Make sure GitHub Actions status checks pass for the PR.\n2. Merge the PR.\n3. Monitor the merge commit GitHub Actions status.\n\n ```{note}\n GitHub Actions are pretty reliable, so please investigate if some error occurs.\n Building Docker images in PRs is the same as building them in the default branch.\n The only difference is that single-platform images are pushed to Registry and then tags are merged for `x86_64` and `aarch64`.\n ```\n\n4. Avoid merging another PR to the main branch until all pending builds in the main branch are complete.\n This way, you will know which commit might have broken the build\n and also have the correct tags for moving tags (like the `Python` version).\n\n## Updating Python version\n\nWhen a new `Python` version is released, we wait for:\n\n- all the dependencies to be available (as wheels or in `conda-forge`).\n- the first `Python` patch release for this version.\n This allows us to avoid many bugs, which can happen in a major release.\n\n## Updating the Ubuntu Base Image\n\n`jupyter/docker-stacks-foundation` is based on the LTS Ubuntu docker image.\nWe wait for the first point release of the new LTS Ubuntu before updating the version.\nOther images are directly or indirectly inherited from `jupyter/docker-stacks-foundation`.\nWe rebuild our images automatically each week, which means they frequently receive updates.\n\nWhen there's a security fix in the Ubuntu base image, it's a good idea to manually trigger the rebuild of images\n[from the GitHub Actions workflow UI](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml).\nPushing the `Run Workflow` button will trigger this process.\n\n## Adding a New Core Image to the Registry\n\n```{note}\nIn general, we do not add new core images and ask contributors to either\ncreate a [recipe](../using/recipes.md) or [community stack](../contributing/stacks.md).\nWe have a [policy](./new-images-and-packages-policy.md), which we consider when adding new images or new packages to existing images.\n```\n\n[Take a look at an example](https://github.com/jupyter/docker-stacks/pull/1936/files) of adding a new image.\n\nWhen there's a new stack definition, check before merging the PR:\n\n1. PR includes an update to the stack overview diagram\n [in the documentation](../using/selecting.md#image-relationships).\n The image links to the [blockdiag source](http://interactive.blockdiag.com/) used to create it.\n2. PR updates the [Makefile](https://github.com/jupyter/docker-stacks/blob/main/Makefile).\n3. Necessary Tagger(s)/Manifest(s) are added for the new image\n in the [tagging](https://github.com/jupyter/docker-stacks/tree/main/tagging) folder.\n4. A new repository is created in the `jupyter` organization in the Registry,\n and it's named after the stack folder in the git repo.\n5. Robot `Write` permission is added in the `Repository Settings`.\n\n## Adding a New Registry Owner Account\n\n1. Visit \n2. Add the maintainer's username.\n\n## Restarting a failed build\n\nIf an automated build in GitHub Actions fails, you can restart the failed jobs on GitHub.\nYou can also download the artifacts and investigate them for any issues.\n" }, { "path": "docs/requirements.txt", "content": "# ReadTheDocs environment contains old package versions preinstalled\n# So, to ensure we have modern packages, we pin minimum versions of the packages we need\ndocutils>=0.17.1\nmyst-parser>=0.18.0\nsphinx>=4.5.0\nsphinx-book-theme>=1.0.0\nsphinx-copybutton>=0.5.0\nsphinx-last-updated-by-git>=0.3.4\n" }, { "path": "docs/using/changelog.md", "content": "```{include} ../../CHANGELOG.md\n\n```\n\n\n" }, { "path": "docs/using/common.md", "content": "# Common Features\n\nExcept for `jupyter/docker-stacks-foundation`, a container launched from any Jupyter Docker Stacks image runs a Jupyter Server with the JupyterLab frontend.\nThe container does so by executing a `start-notebook.py` script.\nThis script configures the internal container environment and then runs `jupyter lab`, passing any command-line arguments received.\n\nThis page describes the options supported by the startup script and how to bypass it to run alternative commands.\n\n## Jupyter Server Options\n\nYou can pass [Jupyter Server options](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html) to the `start-notebook.py` script when launching the container.\n\n1. For example, to secure the Jupyter Server with a [custom password](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html#preparing-a-hashed-password)\n hashed using `jupyter_server.auth.passwd()` instead of the default token,\n you can run the following (this hash was generated for the `my-password` password):\n\n ```bash\n docker run -it --rm -p 8888:8888 quay.io/jupyter/base-notebook \\\n start-notebook.py --PasswordIdentityProvider.hashed_password='argon2:$argon2id$v=19$m=10240,t=10,p=8$JdAN3fe9J45NvK/EPuGCvA$O/tbxglbwRpOFuBNTYrymAEH6370Q2z+eS1eF4GM6Do'\n ```\n\n2. To set the [base URL](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html#running-the-notebook-with-a-customized-url-prefix) of the Jupyter Server, you can run the following:\n\n ```bash\n docker run -it --rm -p 8888:8888 quay.io/jupyter/base-notebook \\\n start-notebook.py --ServerApp.base_url=/customized/url/prefix/\n ```\n\n## Docker Options\n\nYou may instruct the `start-notebook.py` script to customize the container environment before launching the Server.\nYou do so by passing arguments to the `docker run` command.\n\n### User-related configurations\n\n- `-e NB_USER=` - The desired username and associated home folder.\n The default value is `jovyan`.\n Setting `NB_USER` redefines the `jovyan` default user and ensures that the desired user has the correct file permissions\n for the new home directory created at `/home/`.\n For this option to take effect, you **must** run the container with `--user root`, set the working directory `-w \"/home/\"`\n and set the environment variable `-e CHOWN_HOME=yes`.\n\n _Example usage:_\n\n ```bash\n docker run -it --rm \\\n -p 8888:8888 \\\n --user root \\\n -e NB_USER=\"my-username\" \\\n -e CHOWN_HOME=yes \\\n -w \"/home/my-username\" \\\n quay.io/jupyter/base-notebook\n ```\n\n ```{note}\n If you set `NB_USER` to `root`, the `root` home dir will be set to `/home/root`.\n See discussion [here](https://github.com/jupyter/docker-stacks/issues/2042).\n ```\n\n- `-e NB_UID=` - Instructs the startup script to switch the numeric user ID of `${NB_USER}` to the given value.\n The default value is `1000`.\n This feature is useful when mounting host volumes with specific owner permissions.\n You **must** run the container with `--user root` for this option to take effect.\n (The startup script will `su ${NB_USER}` after adjusting the user ID.)\n Instead, you might consider using the modern Docker-native options [`--user`](https://docs.docker.com/engine/containers/run/#user) and\n [`--group-add`](https://docs.docker.com/engine/containers/run/#additional-groups) - see the last bullet in this section for more details.\n See bullet points regarding `--user` and `--group-add`.\n\n- `-e NB_GID=` - Instructs the startup script to change the primary group of `${NB_USER}` to `${NB_GID}`\n (the new group is added with a name of `${NB_GROUP}` if it is defined. Otherwise, the group is named `${NB_USER}`).\n This feature is useful when mounting host volumes with specific group permissions.\n You **must** run the container with `--user root` for this option to take effect.\n (The startup script will `su ${NB_USER}` after adjusting the group ID.)\n Instead, you might consider using modern Docker options `--user` and `--group-add`.\n See bullet points regarding `--user` and `--group-add`.\n The user is added to the supplemental group `users` (gid 100) to grant write access to the home directory and `/opt/conda`.\n If you override the user/group logic, ensure the user stays in the group `users` if you want them to be able to modify files in the image.\n\n- `-e NB_GROUP=` - The name used for `${NB_GID}`, which defaults to `${NB_USER}`.\n This group name is only used if `${NB_GID}` is specified and completely optional: there is only a cosmetic effect.\n\n- `--user 5000 --group-add users` - Launches the container with a specific user ID and adds that user to the `users` group so that it can modify files in the default home directory and `/opt/conda`.\n You can use these arguments as alternatives to setting `${NB_UID}` and `${NB_GID}`.\n\n## Permission-specific configurations\n\n- `-e NB_UMASK=` - Configures Jupyter to use a different `umask` value from default, i.e. `022`.\n For example, if setting `umask` to `002`, new files will be readable and writable by group members instead of the owner only.\n [Check this Wikipedia article](https://en.wikipedia.org/wiki/Umask) for an in-depth description of `umask` and suitable values for multiple needs.\n While the default `umask` value should be sufficient for most use cases, you can set the `NB_UMASK` value to fit your requirements.\n\n ```{note}\n When `NB_UMASK` is set, it only applies to the Jupyter process itself -\n you cannot use it to set a `umask` for additional files created during `run-hooks.sh`.\n For example, via `pip` or `conda`.\n If you need to set a `umask` for these, you **must** set the `umask` value for each command.\n ```\n\n- `-e CHOWN_HOME=yes` - Instructs the startup script to change the `${NB_USER}` home directory owner and group to the current value of `${NB_UID}` and `${NB_GID}`.\n This change will take effect even if the user home directory is mounted from the host using `-v` as described below.\n The change is **not** applied recursively by default.\n You can modify the `chown` behavior by setting `CHOWN_HOME_OPTS` (e.g., `-e CHOWN_HOME_OPTS='-R'`).\n\n- `-e CHOWN_EXTRA=\",\"` - Instructs the startup script to change the owner and group of each comma-separated container directory to the current value of `${NB_UID}` and `${NB_GID}`.\n The change is **not** applied recursively by default.\n You can modify the `chown` behavior by setting `CHOWN_EXTRA_OPTS` (e.g., `-e CHOWN_EXTRA_OPTS='-R'`).\n\n- `-e GRANT_SUDO=yes` - Instructs the startup script to grant the `NB_USER` user passwordless `sudo` capability.\n You do **not** need this option to allow the user to `conda` or `pip` install additional packages.\n This option is helpful for cases when you wish to give `${NB_USER}` the ability to install OS packages with `apt` or modify other root-owned files in the container.\n You **must** run the container with `--user root` for this option to take effect.\n (The `start-notebook.py` script will `su ${NB_USER}` after adding `${NB_USER}` to sudoers.)\n **You should only enable `sudo` if you trust the user or if the container runs on an isolated host.**\n\n### Additional runtime configurations\n\n- `-e GEN_CERT=yes` - Instructs the startup script to generate a self-signed SSL certificate.\n Configures Jupyter Server to use it to accept encrypted HTTPS connections.\n- `-e DOCKER_STACKS_JUPYTER_CMD=` - Instructs the startup script to run `jupyter ${DOCKER_STACKS_JUPYTER_CMD}` instead of the default `jupyter lab` command.\n See [Switching back to the classic notebook or using a different startup command][switch_back] for available options.\n This setting is helpful in container orchestration environments where setting environment variables is more straightforward than changing command line parameters.\n- `-e RESTARTABLE=yes` - Runs Jupyter in a loop so that quitting Jupyter does not cause the container to exit.\n This may be useful when installing extensions that require restarting Jupyter.\n- `-v /some/host/folder/for/work:/home/jovyan/work` - Mounts a host machine directory as a folder in the container.\n This configuration is useful for preserving notebooks and other work even after the container has been destroyed.\n **You must grant the within-container notebook user or group (`NB_UID` or `NB_GID`) write access to the host directory (e.g., `sudo chown 1000 /some/host/folder/for/work`).**\n- `-e JUPYTER_ENV_VARS_TO_UNSET=ADMIN_SECRET_1,ADMIN_SECRET_2` - Unsets specified environment variables in the default startup script.\n The variables are unset after the hooks have been executed but before the command provided to the startup script runs.\n- `-e NOTEBOOK_ARGS=\"--log-level='DEBUG' --dev-mode\"` - Adds custom options to add to `jupyter` commands.\n This way, the user could use any option supported by the `jupyter` subcommand.\n- `-e JUPYTER_PORT=8117` - Changes the port in the container that Jupyter is using to the value of the `${JUPYTER_PORT}` environment variable.\n This may be useful if you run multiple instances of Jupyter in swarm mode and want to use a different port for each instance.\n\n## Startup Hooks\n\nYou can further customize the container environment by adding shell scripts (`*.sh`) to be sourced\nor executables (`chmod +x`) to be run to the paths below:\n\n- `/usr/local/bin/start-notebook.d/` - handled **before** any of the standard options noted above is applied\n- `/usr/local/bin/before-notebook.d/` - handled **after** all the standard options noted above are applied\n and ran right before the Server launches\n\n[Open the `run-hooks.sh` script](https://github.com/jupyter/docker-stacks/blob/main/images/docker-stacks-foundation/run-hooks.sh) and how it's used in the [`start.sh`](https://github.com/jupyter/docker-stacks/blob/main/images/docker-stacks-foundation/start.sh)\nscript for execution details.\n\n## SSL Certificates\n\nYou may mount an SSL key and certificate file into a container and configure the Jupyter Server to use them to accept HTTPS connections.\nFor example, to mount a host folder containing a `notebook.key` and `notebook.crt` and use them, you might run the following:\n\n```bash\ndocker run -it --rm -p 8888:8888 \\\n -v /some/host/folder:/etc/ssl/notebook \\\n quay.io/jupyter/base-notebook \\\n start-notebook.py \\\n --ServerApp.keyfile=/etc/ssl/notebook/notebook.key \\\n --ServerApp.certfile=/etc/ssl/notebook/notebook.crt\n```\n\nAlternatively, you may mount a single PEM file containing both the key and certificate.\nFor example:\n\n```bash\ndocker run -it --rm -p 8888:8888 \\\n -v /some/host/folder/notebook.pem:/etc/ssl/notebook.pem \\\n quay.io/jupyter/base-notebook \\\n start-notebook.py \\\n --ServerApp.certfile=/etc/ssl/notebook.pem\n```\n\nIn either case, Jupyter Server expects the key and certificate to be a **base64 encoded text file**.\nThe certificate file or PEM may contain one or more certificates (e.g., server, intermediate, and root).\n\nFor additional information about using SSL, see the following:\n\n- The [docker-stacks/examples](https://github.com/jupyter/docker-stacks/tree/main/examples)\n for information about how to use\n [Let's Encrypt](https://letsencrypt.org/) certificates when you run these stacks on a publicly visible domain.\n- The [`jupyter_server_config.py`](https://github.com/jupyter/docker-stacks/blob/main/images/base-notebook/jupyter_server_config.py)\n file for how this Docker image generates a self-signed certificate.\n- The [Jupyter Server documentation](https://jupyter-server.readthedocs.io/en/latest/operators/public-server.html#securing-a-jupyter-server)\n for best practices about securing a public Server in general.\n\n## Alternative Commands\n\n### Switching back to the classic notebook or using a different startup command\n\nJupyterLab, built on top of Jupyter Server, is now the default for all the images of the stack.\nHowever, switching back to the classic notebook or using a different startup command is still possible.\nYou can achieve this by setting the environment variable `DOCKER_STACKS_JUPYTER_CMD` at container startup.\nThe table below shows some options.\nSince `Jupyter Notebook v7` `jupyter-server` is used as a backend.\n\n| `DOCKER_STACKS_JUPYTER_CMD` | Frontend |\n| --------------------------- | ---------------- |\n| `lab` (default) | JupyterLab |\n| `notebook` | Jupyter Notebook |\n| `nbclassic` | NbClassic |\n| `server` | None |\n| `retro`\\* | RetroLab |\n\n```{note}\n- Changing frontend for **JupyterHub singleuser image** is described in [JupyterHub docs](https://jupyterhub.readthedocs.io/en/latest/howto/configuration/config-user-env.html#switching-back-to-the-classic-notebook).\n- \\* `retro` is not installed at this time, but it could be the case in the future or in a community stack.\n- Any other valid `jupyter` subcommand that starts the Jupyter Application can be used.\n```\n\nExample:\n\n```bash\n# Run Jupyter Server with the Jupyter Notebook frontend\ndocker run -it --rm \\\n -p 8888:8888 \\\n -e DOCKER_STACKS_JUPYTER_CMD=notebook \\\n quay.io/jupyter/base-notebook\n\n# Executing the command: start-notebook.py\n# Executing: jupyter notebook\n# ...\n\n# Use Jupyter NBClassic frontend\ndocker run -it --rm \\\n -p 8888:8888 \\\n -e DOCKER_STACKS_JUPYTER_CMD=nbclassic \\\n quay.io/jupyter/base-notebook\n\n# Executing the command: start-notebook.py\n# Executing: jupyter nbclassic\n# ...\n```\n\n### `start.sh`\n\nMost of the configuration options in the `start-notebook.py` script are handled by an internal `start.sh` script that automatically runs before the command provided to the container\n(it's set as the container entrypoint).\nThis allows you to specify an arbitrary command that takes advantage of all these features.\nFor example, to run the text-based `ipython` console in a container, do the following:\n\n```bash\ndocker run -it --rm quay.io/jupyter/base-notebook ipython\n```\n\nThis script is handy when you derive a new Dockerfile from this image and install additional Jupyter applications with subcommands like `jupyter console`, `jupyter kernelgateway`, etc.\n\n## Conda Environments\n\nThe default Python 3.x [Conda environment](https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/environments.html) resides in `/opt/conda`.\nThe `/opt/conda/bin` directory is part of the default `jovyan` user's `${PATH}`.\nThat directory is also searched for binaries when run using `sudo` (`sudo my_binary` will search for `my_binary` in `/opt/conda/bin/`).\n\nThe `jovyan` user has full read/write access to the `/opt/conda` directory.\nYou can use either `mamba`, `pip`, or `conda` (`mamba` is recommended) to install new packages without any additional permissions.\n\n```bash\n# install a package into the default (python 3.x) environment and cleanup it after\n# the installation\nmamba install --yes some-package && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\npip install --no-cache-dir some-package && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\nconda install --yes some-package && \\\n conda clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n```\n\n### Using Alternative Channels\n\nConda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel.\nHowever, you can use alternative channels, either one-shot by overwriting the default channel in the installation command or by configuring `mamba` to use different channels.\nThe examples below show how to use the [anaconda default channels](https://repo.anaconda.com/pkgs/main) instead of `conda-forge` to install packages.\n\n```bash\n# using defaults channels to install a package\nmamba install --channel defaults humanize\n\n# configure conda to add default channels at the top of the list\nconda config --system --prepend channels defaults\n\n# install a package\nmamba install --yes humanize && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n```\n\n[switch_back]: #switching-back-to-the-classic-notebook-or-using-a-different-startup-command\n" }, { "path": "docs/using/custom-images.md", "content": "# Building a custom set of images\n\nThis section describes how to build a custom set of images.\nIt may be helpful if you need to change the Ubuntu or Python version, or to make a significant change to the build process itself.\n\nThis project only builds one set of images at a time.\nIf you want to use older images, [take a look here](../index.rst/#using-old-images).\n\n## Automating your build using template cookiecutter project\n\nIf you wish to build your own image on top of one of our images and automate your build process,\nplease, [take a look at cookiecutter template](../contributing/stacks.md).\n\n## Custom arguments\n\nOur repository provides several customization points:\n\n- `ROOT_IMAGE` (docker argument) - the parent image for `docker-stacks-foundation` image\n- `PYTHON_VERSION` (docker argument) - the Python version to install in `docker-stacks-foundation` image\n- `REGISTRY`, `OWNER`, `BASE_IMAGE` (docker arguments) - they allow to specify parent image for all the other images\n- `REGISTRY`, `OWNER` (part of `env` in some GitHub workflows) - these allow to properly tag and refer to images during following steps:\n - [`build-test-upload`](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/docker-build-test-upload.yml)\n - [`contributed-recipes`](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/contributed-recipes.yml)\n - [`tag-push-merge`](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/docker-tag-push-merge.yml)\n\nThese customization points can't be changed during runtime.\nRead more about [Docker build arguments](https://docs.docker.com/build/building/variables/#arg-usage-example) and [GitHub environment variables for a single workflow](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#defining-environment-variables-for-a-single-workflow).\n\n## Building stack images with custom arguments\n\nA selection of prebuilt images are available from [Quay.io](https://quay.io/organization/jupyter),\nhowever, it's impossible to cater to everybody's needs.\nFor extensive customization with an automated build pipeline,\nyou may wish to create a [community-maintained stack](../contributing/stacks),\nhowever, for minor customizations, this may be overkill.\nFor example, you may wish to use the same Jupyter stacks but built on a different base image,\nor built with a different Python version.\n\nTo achieve this you can use [Docker Bake](https://docs.docker.com/build/bake/)\nto build the stacks locally with custom arguments.\n\n```{note}\nCustom arguments may result in build errors due to incompatibility.\nIf so your use-case may require a fully customized stack.\n```\n\nAs a basic example, if you want to build a custom image based on the `minimal-notebook` image using `Python 3.12`,\nthen with a Dockerfile like:\n\n```{code-block} Dockerfile\n:caption: Dockerfile\n\nARG BASE_IMAGE=minimal-notebook\nFROM $BASE_IMAGE\n...\n```\n\nInclude the file below in your project:\n\n```{literalinclude} recipe_code/docker-bake.custom-python.hcl\n:force:\n:language: hcl\n:caption: docker-bake.hcl\n```\n\nTo build this stack, in the same directory run:\n\n```bash\ndocker buildx bake\n```\n\nDocker Bake then determines the correct build order from the `contexts` parameters\nand builds the stack as requested.\n\nThis image can then be run the same way as any other image provided by this project, for example:\n\n```bash\ndocker run -it --rm -p 8888:8888 custom-jupyter\n```\n\nor referenced in a Docker Compose file.\n\n## Forking our repository\n\nIf for some reason, you need to change more things in our images, feel free to fork it and change it any way you want.\nIf your customization is easy to backport to the main repo and might be helpful for other users, feel free to create a PR.\n\nIt is almost always a great idea to keep your diff as small as possible and to merge/rebase the latest version of our repo in your project.\n" }, { "path": "docs/using/faq.md", "content": "# Frequently Asked Questions (FAQ)\n\n## How to persist user data\n\nThere are two types of data you might want to persist.\n\n1. If you want to persist your environment (i.e. packages installed by `mamba`, `conda`, `pip`, `apt-get`, and so on),\n then you should create an inherited image and install packages only once while building your Dockerfile.\n Take a look at [an example of using `mamba` and `pip`](./recipes.md#using-mamba-install-recommended-or-pip-install-in-a-child-docker-image) in a child image.\n\n ```{note}\n If you install a package inside a running container (for example you run `pip install ` in a terminal),\n it won't be preserved when you next run your image.\n To make it work, install this package in your inherited image and rerun the `docker build` command.\n ```\n\n2. If you want to persist user data (files created by you, like `Python` scripts, notebooks, text files, and so on),\n then you should use a\n [Docker bind mount](https://docs.docker.com/engine/storage/bind-mounts/) or\n [Docker Volume](https://docs.docker.com/engine/storage/volumes/).\n You can find [an example of using a bind mount here](./running.md#example-2).\n There is also [a mount troubleshooting section](./troubleshooting.md#permission-denied-when-mounting-volumes) if you experience any issues.\n\n## Why we don't add your favorite package\n\nWe have lots of users with different packages they want to use.\nAdding them all is impossible, so we have several images to choose from.\n[Choose the image](selecting.md) that is closest to your needs, and feel free to [add your package on top of our images](recipes.md#using-mamba-install-recommended-or-pip-install-in-a-child-docker-image).\n\n## Who is `jovyan`\n\nAs described [in this issue comment](https://github.com/jupyter/docker-stacks/issues/358#issuecomment-288844834):\n\n```text\nJoยทvyยทan\n/หˆjลvฤ“ษ™n/\nnoun โ€“ an inhabitant of Jupyter\n```\n\n`Jovyan` is often a special term used to describe members of the Jupyter community.\nIt is also used as the user ID in the Jupyter Docker stacks or referenced in conversations.\nYou can find more information on [the Jupyter Community documentation](https://docs.jupyter.org/en/latest/community/content-community.html#what-is-a-jovyan).\n\n## How to give root permissions to the user\n\nWe have a [recipe for enabling root permissions](recipes.md#using-sudo-within-a-container).\n" }, { "path": "docs/using/recipe_code/custom_environment.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/minimal-notebook\nFROM $BASE_IMAGE\n\n# Name your environment and choose the Python version\nARG env_name=python313\nARG py_ver=3.13\n\n# You can add additional libraries here\nRUN mamba create --yes -p \"${CONDA_DIR}/envs/${env_name}\" \\\n python=${py_ver} \\\n 'ipykernel' \\\n 'jupyterlab' && \\\n mamba clean --all -f -y\n\n# Alternatively, you can comment out the lines above and uncomment those below\n# if you'd prefer to use a YAML file present in the docker build context\n\n# COPY --chown=${NB_UID}:${NB_GID} environment.yml /tmp/\n# RUN mamba env create -p \"${CONDA_DIR}/envs/${env_name}\" -f /tmp/environment.yml && \\\n# mamba clean --all -f -y\n\n# Create Python kernel and link it to jupyter\nRUN \"${CONDA_DIR}/envs/${env_name}/bin/python\" -m ipykernel install --user --name=\"${env_name}\" && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# Any additional `pip` installs can be added by using the following line\n# Using `mamba` is highly recommended though\nRUN \"${CONDA_DIR}/envs/${env_name}/bin/pip\" install --no-cache-dir \\\n 'flake8'\n\n# This changes the custom Python kernel so that the custom environment will\n# be activated for the respective Jupyter Notebook and Jupyter Console\n# hadolint ignore=DL3059\nRUN /opt/setup-scripts/activate_notebook_custom_env.py \"${env_name}\"\n\n# Comment the line above and uncomment the section below instead to activate the custom environment by default\n# Note: uncommenting this section makes \"${env_name}\" default both for Jupyter Notebook and Terminals\n# More information here: https://github.com/jupyter/docker-stacks/pull/2047\n# USER root\n# RUN \\\n# # This changes a startup hook, which will activate the custom environment for the process\n# echo conda activate \"${env_name}\" >> /usr/local/bin/before-notebook.d/10activate-conda-env.sh && \\\n# # This makes the custom environment default in Jupyter Terminals for all users which might be created later\n# echo conda activate \"${env_name}\" >> /etc/skel/.bashrc && \\\n# # This makes the custom environment default in Jupyter Terminals for already existing NB_USER\n# echo conda activate \"${env_name}\" >> \"/home/${NB_USER}/.bashrc\"\n\nUSER ${NB_UID}\n" }, { "path": "docs/using/recipe_code/dask_jupyterlab.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/base-notebook\nFROM $BASE_IMAGE\n\n# Install the Dask dashboard\nRUN mamba install --yes 'dask-labextension' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# Dask Scheduler port\nEXPOSE 8787\n" }, { "path": "docs/using/recipe_code/docker-bake.custom-python.hcl", "content": "group \"default\" {\n targets = [\"custom-notebook\"]\n}\n\ntarget \"foundation\" {\n context = \"https://github.com/jupyter/docker-stacks.git#main:images/docker-stacks-foundation\"\n args = {\n PYTHON_VERSION = \"3.13\"\n }\n tags = [\"docker-stacks-foundation\"]\n}\n\ntarget \"base-notebook\" {\n context = \"https://github.com/jupyter/docker-stacks.git#main:images/base-notebook\"\n contexts = {\n docker-stacks-foundation = \"target:foundation\"\n }\n args = {\n BASE_IMAGE = \"docker-stacks-foundation\"\n }\n tags = [\"base-notebook\"]\n}\n\ntarget \"minimal-notebook\" {\n context = \"https://github.com/jupyter/docker-stacks.git#main:images/minimal-notebook\"\n contexts = {\n base-notebook = \"target:base-notebook\"\n }\n args = {\n BASE_IMAGE = \"base-notebook\"\n }\n tags = [\"minimal-notebook\"]\n}\n\ntarget \"custom-notebook\" {\n context = \".\"\n contexts = {\n minimal-notebook = \"target:minimal-notebook\"\n }\n args = {\n BASE_IMAGE = \"minimal-notebook\"\n }\n tags = [\"custom-jupyter\"]\n}\n" }, { "path": "docs/using/recipe_code/generate_matrix.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport json\nfrom pathlib import Path\n\nTHIS_DIR = Path(__file__).parent.resolve()\n\nRUNS_ON = [\"ubuntu-24.04\", \"ubuntu-24.04-arm\"]\nARM_INCOMPATIBLE_IMAGES = {\"oracledb.dockerfile\"}\nBASE_IMAGE_PREFIX = \"ARG BASE_IMAGE=\"\n\n\ndef extract_base_image(dockerfile: Path) -> str:\n \"\"\"Extract base image from dockerfile\"\"\"\n for line in dockerfile.read_text().splitlines():\n if line.startswith(BASE_IMAGE_PREFIX):\n full_image = line[len(BASE_IMAGE_PREFIX) :]\n image_name = full_image[full_image.rfind(\"/\") + 1 :]\n return \"\" if \":\" in image_name else image_name\n raise RuntimeError(f\"Base image not found in {dockerfile}\")\n\n\ndef get_platform(runs_on: str) -> str:\n \"\"\"Get platform architecture based on runner\"\"\"\n return \"x86_64\" if runs_on == \"ubuntu-24.04\" else \"aarch64\"\n\n\ndef generate_matrix() -> dict[str, list[dict[str, str]]]:\n \"\"\"Generate build matrix for GitHub Actions\"\"\"\n dockerfiles = sorted(THIS_DIR.glob(\"*.dockerfile\"))\n configurations: list[dict[str, str]] = []\n\n for dockerfile in dockerfiles:\n dockerfile_name = dockerfile.name\n\n for run in RUNS_ON:\n # Skip ARM builds for incompatible images\n if dockerfile_name in ARM_INCOMPATIBLE_IMAGES and run == \"ubuntu-24.04-arm\":\n continue\n\n configurations.append(\n {\n \"dockerfile\": dockerfile_name,\n \"runs-on\": run,\n \"platform\": get_platform(run),\n \"parent-image\": extract_base_image(dockerfile),\n }\n )\n\n return {\"include\": configurations}\n\n\nif __name__ == \"__main__\":\n print(f\"matrix={json.dumps(generate_matrix())}\")\n" }, { "path": "docs/using/recipe_code/ijavascript.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/base-notebook\nFROM $BASE_IMAGE\n\nUSER root\n\nRUN apt-get update --yes && \\\n apt-get install --yes --no-install-recommends \\\n make \\\n g++ && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\nUSER ${NB_UID}\n\n# NodeJS <= 20 is required\n# https://github.com/n-riesco/ijavascript/issues/184\nRUN mamba install --yes nodejs=20.* && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# hadolint ignore=DL3016\nRUN npm install -g ijavascript\n# hadolint ignore=DL3059\nRUN ijsinstall\n" }, { "path": "docs/using/recipe_code/jupyterhub_version.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/base-notebook\nFROM $BASE_IMAGE\n\nRUN mamba install --yes 'jupyterhub-singleuser==5.2.1' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "docs/using/recipe_code/mamba_install.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/base-notebook\nFROM $BASE_IMAGE\n\nRUN mamba install --yes 'flake8' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# Install from the requirements.txt file\nCOPY --chown=${NB_UID}:${NB_GID} requirements.txt /tmp/\nRUN mamba install --yes --file /tmp/requirements.txt && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "docs/using/recipe_code/manpage_install.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/base-notebook\nFROM $BASE_IMAGE\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\n# `/etc/dpkg/dpkg.cfg.d/excludes` contains several `path-exclude`s, including man pages\n# Remove it, then install man, install docs\nRUN rm /etc/dpkg/dpkg.cfg.d/excludes && \\\n apt-get update --yes && \\\n dpkg -l | grep ^ii | cut -d' ' -f3 | xargs apt-get install --yes --no-install-recommends --reinstall man && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\nUSER ${NB_UID}\n" }, { "path": "docs/using/recipe_code/microsoft_odbc.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/base-notebook\nFROM $BASE_IMAGE\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\nENV MSSQL_DRIVER=\"ODBC Driver 18 for SQL Server\"\nENV PATH=\"/opt/mssql-tools18/bin:${PATH}\"\n\nRUN apt-get update --yes && \\\n apt-get install --yes --no-install-recommends curl gnupg2 lsb-release && \\\n curl -fsSL \"https://packages.microsoft.com/keys/microsoft.asc\" | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg && \\\n curl \"https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list\" > /etc/apt/sources.list.d/mssql-release.list && \\\n apt-get update --yes && \\\n ACCEPT_EULA=Y apt-get install --yes --no-install-recommends msodbcsql18 && \\\n # optional: for bcp and sqlcmd\n ACCEPT_EULA=Y apt-get install --yes --no-install-recommends mssql-tools18 && \\\n # optional: for unixODBC development headers\n apt-get install --yes --no-install-recommends unixodbc-dev && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# Switch back to jovyan to avoid accidental container runs as root\nUSER ${NB_UID}\n\nRUN mamba install --yes 'pyodbc' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "docs/using/recipe_code/oracledb.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/base-notebook\nFROM $BASE_IMAGE\n\nUSER root\n\n# Install Java & Oracle SQL Instant Client\nRUN apt-get update --yes && \\\n apt-get install --yes --no-install-recommends software-properties-common && \\\n add-apt-repository universe && \\\n apt-get update --yes && \\\n apt-get install --yes --no-install-recommends alien default-jre default-jdk openjdk-11-jdk libaio1t64 && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# Oracle\nARG INSTANTCLIENT_MAJOR_VERSION=23\nARG INSTANTCLIENT_BIN_SUFFIX=${INSTANTCLIENT_MAJOR_VERSION}.6.0.24.10-1.el9.x86_64.rpm\nARG INSTANTCLIENT_URL=https://download.oracle.com/otn_software/linux/instantclient/2360000\n\n# Then install Oracle SQL Instant client, SQL+Plus, tools, and JDBC.\n# Note: You may need to change the URL to a newer version.\n# See: https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html\nRUN mkdir \"/opt/oracle\"\nWORKDIR \"/opt/oracle\"\n# alien doesn't work well with sqlplus, so skipping it for now\nRUN wget --progress=dot:giga \"${INSTANTCLIENT_URL}/oracle-instantclient-basiclite-${INSTANTCLIENT_BIN_SUFFIX}\" && \\\n alien --install --scripts \"oracle-instantclient-basiclite-${INSTANTCLIENT_BIN_SUFFIX}\" && \\\n wget --progress=dot:giga \"${INSTANTCLIENT_URL}/oracle-instantclient-sqlplus-${INSTANTCLIENT_BIN_SUFFIX}\" && \\\n # alien --install --scripts \"oracle-instantclient-sqlplus-${INSTANTCLIENT_BIN_SUFFIX}\" && \\\n wget --progress=dot:giga \"${INSTANTCLIENT_URL}/oracle-instantclient-tools-${INSTANTCLIENT_BIN_SUFFIX}\" && \\\n alien --install --scripts \"oracle-instantclient-tools-${INSTANTCLIENT_BIN_SUFFIX}\" && \\\n wget --progress=dot:giga \"${INSTANTCLIENT_URL}/oracle-instantclient-jdbc-${INSTANTCLIENT_BIN_SUFFIX}\" && \\\n alien --install --scripts \"oracle-instantclient-jdbc-${INSTANTCLIENT_BIN_SUFFIX}\" && \\\n chown -R \"${NB_UID}\":\"${NB_GID}\" \"${HOME}/.rpmdb\" && \\\n rm -f ./*.rpm\n\n# And configure variables\nRUN echo \"ORACLE_HOME=/usr/lib/oracle/${INSTANTCLIENT_MAJOR_VERSION}/client64\" >> \"${HOME}/.bashrc\" && \\\n echo \"PATH=\\\"${ORACLE_HOME}/bin:${PATH}\\\"\" >> \"${HOME}/.bashrc\" && \\\n echo \"LD_LIBRARY_PATH=\\\"${ORACLE_HOME}/lib:${LD_LIBRARY_PATH}\\\"\" >> \"${HOME}/.bashrc\" && \\\n echo \"export ORACLE_HOME\" >> \"${HOME}/.bashrc\" && \\\n echo \"export PATH\" >> \"${HOME}/.bashrc\" && \\\n echo \"export LD_LIBRARY_PATH\" >> \"${HOME}/.bashrc\"\n\n# Add credentials for /redacted/ using Oracle DB.\nWORKDIR /usr/lib/oracle/${INSTANTCLIENT_MAJOR_VERSION}/client64/lib/network/admin/\n# Add a wildcard `[]` on the last letter of the filename to avoid throwing an error if the file does not exist.\n# See: https://stackoverflow.com/questions/31528384/conditional-copy-add-in-dockerfile\nCOPY cwallet.ss[o] ./\nCOPY sqlnet.or[a] ./\nCOPY tnsnames.or[a] ./\n\n# Switch back to jovyan to avoid accidental container runs as root\nUSER \"${NB_UID}\"\n\nWORKDIR \"${HOME}\"\n\n# Install `oracledb` Python library to use Oracle SQL Instant Client\nRUN mamba install --yes 'oracledb' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "docs/using/recipe_code/pip_install.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/base-notebook\nFROM $BASE_IMAGE\n\n# Install in the default python3 environment\nRUN pip install --no-cache-dir 'flake8' && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# Install from the requirements.txt file\nCOPY --chown=${NB_UID}:${NB_GID} requirements.txt /tmp/\nRUN pip install --no-cache-dir --requirement /tmp/requirements.txt && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "docs/using/recipe_code/requirements.txt", "content": "autoflake\n" }, { "path": "docs/using/recipe_code/rise_jupyterlab.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/base-notebook\nFROM $BASE_IMAGE\n\nRUN mamba install --yes 'jupyterlab_rise' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "docs/using/recipe_code/spellcheck_notebook_v6.dockerfile", "content": "# Using Docker Hub here, because this image is old and not pushed to Quay.io\nARG BASE_IMAGE=docker.io/jupyter/base-notebook:notebook-6.5.4\nFROM $BASE_IMAGE\n\nRUN pip install --no-cache-dir 'jupyter_contrib_nbextensions' && \\\n jupyter contrib nbextension install --user && \\\n # can modify or enable additional extensions here\n jupyter nbclassic-extension enable spellchecker/main --user && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "docs/using/recipe_code/xgboost.dockerfile", "content": "ARG BASE_IMAGE=quay.io/jupyter/base-notebook\nFROM $BASE_IMAGE\n\nRUN mamba install --yes 'py-xgboost' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "docs/using/recipes.md", "content": "# Contributed Recipes\n\nUsers sometimes share interesting ways of using the Jupyter Docker Stacks.\nWe encourage users to [contribute these recipes](../contributing/recipes.md) to the documentation in case they prove helpful to other community members by submitting a pull request to `docs/using/recipes.md`.\nThe sections below capture this knowledge.\n\nAll the recipes here assume you would like to use an image built by this project and install some things on top of it.\nIf you would like to build a custom set of images, [take a look at the docs](custom-images.md).\n\n## Using `sudo` within a container\n\nPassword authentication is disabled for the `NB_USER` (e.g., `jovyan`).\nWe made this choice to avoid distributing images with a weak default password that users ~might~ will forget to change before running a container on a publicly accessible host.\n\nYou can grant the within-container `NB_USER` passwordless `sudo` access by adding `--user root` and `-e GRANT_SUDO=yes` to your Docker command line or appropriate container orchestrator config.\n\nFor example:\n\n```bash\ndocker run -it --rm \\\n --user root \\\n -e GRANT_SUDO=yes \\\n quay.io/jupyter/base-notebook\n```\n\n**You should only enable `sudo` if you trust the user and/or if the container is running on an isolated host.**\nSee [Docker security documentation](https://docs.docker.com/engine/security/userns-remap/) for more information about running containers as `root`.\n\n## Using `mamba install` (recommended) or `pip install` in a Child Docker image\n\nCreate a new Dockerfile like the one shown below.\nTo use a requirements.txt file, first, create your `requirements.txt` file with the listing of packages desired.\n\n```{literalinclude} recipe_code/mamba_install.dockerfile\n:language: docker\n```\n\n`pip` usage is similar:\n\n```{literalinclude} recipe_code/pip_install.dockerfile\n:language: docker\n```\n\nThen build a new image.\n\n```bash\ndocker build --rm --tag my-custom-image .\n```\n\nYou can then run the image as follows:\n\n```bash\ndocker run -it --rm \\\n -p 8888:8888 \\\n my-custom-image\n```\n\n## Add a custom conda environment and Jupyter kernel\n\nThe default version of `Python` that ships with the image may not be the version you want.\nThe instructions below permit adding a conda environment with a different `Python` version and making it accessible to Jupyter.\nYou may also use older images like `jupyter/base-notebook:python-3.10`.\nWe also maintain a [full build history](https://github.com/jupyter/docker-stacks/wiki).\n\n```{literalinclude} recipe_code/custom_environment.dockerfile\n:language: docker\n```\n\n## Dask JupyterLab Extension\n\n[Dask JupyterLab Extension](https://github.com/dask/dask-labextension) provides a JupyterLab extension to manage Dask clusters, as well as embed Dask's dashboard plots directly into JupyterLab panes.\nCreate the Dockerfile as:\n\n```{literalinclude} recipe_code/dask_jupyterlab.dockerfile\n:language: docker\n```\n\nAnd build the image as:\n\n```bash\ndocker build --rm --tag my-custom-image .\n```\n\nOnce built, run using the command:\n\n```bash\ndocker run -it --rm \\\n -p 8888:8888 \\\n -p 8787:8787 \\\n my-custom-image\n```\n\n## Let's Encrypt a Server\n\n```{warning}\nThis recipe is not tested and might be broken.\n```\n\nSee the README for basic automation here\n\nwhich includes steps for requesting and renewing a Let's Encrypt certificate.\n\nRef: \n\n## Slideshows with JupyterLab and RISE\n\n[RISE](https://github.com/jupyterlab-contrib/rise): \"Live\" Reveal.js JupyterLab Slideshow Extension.\n\n```{note}\nWe're providing the recipe to install the JupyterLab extension.\nYou can find the original Jupyter Notebook extension [here](https://github.com/damianavila/RISE)\n```\n\n```{literalinclude} recipe_code/rise_jupyterlab.dockerfile\n:language: docker\n```\n\n## xgboost\n\n```{literalinclude} recipe_code/xgboost.dockerfile\n:language: docker\n```\n\n## Running behind an nginx proxy\n\n```{warning}\nThis recipe is not tested and might be broken.\n```\n\nSometimes it is helpful to run the Jupyter instance behind an nginx proxy, for example:\n\n- you would prefer to access the notebook at a server URL with a path\n (`https://example.com/jupyter`) rather than a port (`https://example.com:8888`)\n- you may have many services in addition to Jupyter running on the same server\n and want nginx to help improve server performance in managing the connections\n\nHere is a [quick example of NGINX configuration](https://gist.github.com/cboettig/8643341bd3c93b62b5c2) to get started.\nYou'll need a server, a `.crt`, and a `.key` file for your server, and `docker` & `docker-compose` installed.\nThen download the files at that gist and run `docker-compose up` to test it out.\nCustomize the `nginx.conf` file to set the desired paths and add other services.\n\n## Host volume mounts and notebook errors\n\nIf you are mounting a host directory as `/home/jovyan/work` in your container,\nand you receive permission errors or connection errors when you create a notebook,\nbe sure that the `jovyan` user (`UID=1000` by default) has read/write access to the directory on the host.\nAlternatively, specify the UID of the `jovyan` user on container startup using the `-e NB_UID` option\ndescribed in the [Common Features, Docker Options section](common.md#docker-options)\n\nRef: \n\n## Manpage installation\n\nMost images, including our Ubuntu base image, ship without manpages installed to save space.\nYou can use the following Dockerfile to inherit from one of our images to enable manpages:\n\n```{literalinclude} recipe_code/manpage_install.dockerfile\n:language: docker\n```\n\nAdding the documentation on top of the existing image wastes a lot of space\nand requires reinstalling every system package,\nwhich can take additional time and bandwidth.\nEnabling manpages in the base Ubuntu layer prevents this image bloat.\nTo achieve this, use the previous `Dockerfile`'s commands with the original `ubuntu` image as your base image:\n\n```dockerfile\nFROM ubuntu:24.04\n```\n\nBe sure to check the current base image in `jupyter/docker-stacks-foundation` before building.\n\n## JupyterHub\n\nWe also have contributed recipes for using JupyterHub.\n\n### Use JupyterHub's DockerSpawner\n\nYou can find [an example of using DockerSpawner](https://github.com/jupyterhub/jupyterhub-deploy-docker/tree/main/basic-example).\n\n### Containers with a specific version of JupyterHub\n\nThe version of `jupyterhub` in your image should match the\nversion in JupyterHub itself.\nTo use a specific version of JupyterHub, do the following:\n\n```{literalinclude} recipe_code/jupyterhub_version.dockerfile\n:language: docker\n```\n\n## Spark\n\nA few suggestions have been made regarding using Docker Stacks with Spark.\n\n### Using PySpark with AWS S3\n\n```{warning}\nThis recipe is not tested and might be broken.\n```\n\nUsing Spark session for Hadoop 2.7.3\n\n```python\nimport os\n\n# To figure out what version of Hadoop, run:\n# ls /usr/local/spark/jars/hadoop*\nos.environ[\"PYSPARK_SUBMIT_ARGS\"] = (\n '--packages \"org.apache.hadoop:hadoop-aws:2.7.3\" pyspark-shell'\n)\n\nimport pyspark\n\nmyAccessKey = input()\nmySecretKey = input()\n\nspark = (\n pyspark.sql.SparkSession.builder.master(\"local[*]\")\n .config(\"spark.hadoop.fs.s3a.access.key\", myAccessKey)\n .config(\"spark.hadoop.fs.s3a.secret.key\", mySecretKey)\n .getOrCreate()\n)\n\ndf = spark.read.parquet(\"s3://myBucket/myKey\")\n```\n\nUsing Spark context for Hadoop 2.6.0\n\n```python\nimport os\n\nos.environ[\"PYSPARK_SUBMIT_ARGS\"] = (\n \"--packages com.amazonaws:aws-java-sdk:1.10.34,org.apache.hadoop:hadoop-aws:2.6.0 pyspark-shell\"\n)\n\nimport pyspark\n\nsc = pyspark.SparkContext(\"local[*]\")\n\nfrom pyspark.sql import SQLContext\n\nsqlContext = SQLContext(sc)\n\nhadoopConf = sc._jsc.hadoopConfiguration()\nmyAccessKey = input()\nmySecretKey = input()\nhadoopConf.set(\"fs.s3.impl\", \"org.apache.hadoop.fs.s3native.NativeS3FileSystem\")\nhadoopConf.set(\"fs.s3.awsAccessKeyId\", myAccessKey)\nhadoopConf.set(\"fs.s3.awsSecretAccessKey\", mySecretKey)\n\ndf = sqlContext.read.parquet(\"s3://myBucket/myKey\")\n```\n\nRef: \n\n### Using Local Spark JARs\n\n```{warning}\nThis recipe is not tested and might be broken.\n```\n\n```python\nimport os\n\nos.environ[\"PYSPARK_SUBMIT_ARGS\"] = (\n \"--jars /home/jovyan/spark-streaming-kafka-assembly_2.10-1.6.1.jar pyspark-shell\"\n)\nimport pyspark\nfrom pyspark.streaming.kafka import KafkaUtils\nfrom pyspark.streaming import StreamingContext\n\nsc = pyspark.SparkContext()\nssc = StreamingContext(sc, 1)\nbroker = \"\"\ndirectKafkaStream = KafkaUtils.createDirectStream(\n ssc, [\"test1\"], {\"metadata.broker.list\": broker}\n)\ndirectKafkaStream.pprint()\nssc.start()\n```\n\nRef: \n\n### Using spark-packages.org\n\n```{warning}\nThis recipe is not tested and might be broken.\n```\n\nIf you'd like to use packages from [spark-packages.org](https://spark-packages.org/), see\n[https://gist.github.com/parente/c95fdaba5a9a066efaab](https://gist.github.com/parente/c95fdaba5a9a066efaab)\nfor an example of how to specify the package identifier in the environment before creating a\nSparkContext.\n\nRef: \n\n### Use jupyter/all-spark-notebooks with an existing Spark/YARN cluster\n\n```{warning}\nThis recipe is not tested and might be broken.\n```\n\n```dockerfile\nFROM quay.io/jupyter/all-spark-notebook\n\n# Set env vars for pydoop\nENV HADOOP_HOME=/usr/local/hadoop-2.7.3\nENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64\nENV HADOOP_CONF_HOME=/usr/local/hadoop-2.7.3/etc/hadoop\nENV HADOOP_CONF_DIR=/usr/local/hadoop-2.7.3/etc/hadoop\n\nUSER root\n# Add proper open-jdk-8 not the jre only, needed for pydoop\nRUN echo 'deb https://cdn-fastly.deb.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/jessie-backports.list && \\\n apt-get update --yes && \\\n apt-get install --yes --no-install-recommends -t jessie-backports openjdk-8-jdk && \\\n rm /etc/apt/sources.list.d/jessie-backports.list && \\\n apt-get clean && rm -rf /var/lib/apt/lists/* && \\\n# Add Hadoop binaries\n wget --progress=dot:giga https://mirrors.ukfast.co.uk/sites/ftp.apache.org/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz && \\\n tar -xvf hadoop-2.7.3.tar.gz -C /usr/local && \\\n chown -R \"${NB_USER}:users\" /usr/local/hadoop-2.7.3 && \\\n rm -f hadoop-2.7.3.tar.gz && \\\n# Install os dependencies required for pydoop, pyhive\n apt-get update --yes && \\\n apt-get install --yes --no-install-recommends build-essential python-dev libsasl2-dev && \\\n apt-get clean && rm -rf /var/lib/apt/lists/* && \\\n# Remove the example hadoop configs and replace\n# with those for our cluster.\n# Alternatively, this could be mounted as a volume\n rm -f /usr/local/hadoop-2.7.3/etc/hadoop/*\n\n# Download this from ambari/cloudera manager and copy it here\nCOPY example-hadoop-conf/ /usr/local/hadoop-2.7.3/etc/hadoop/\n\n# Spark-Submit doesn't work unless I set the following\nRUN echo \"spark.driver.extraJavaOptions -Dhdp.version=2.5.3.0-37\" >> /usr/local/spark/conf/spark-defaults.conf && \\\n echo \"spark.yarn.am.extraJavaOptions -Dhdp.version=2.5.3.0-37\" >> /usr/local/spark/conf/spark-defaults.conf && \\\n echo \"spark.master=yarn\" >> /usr/local/spark/conf/spark-defaults.conf && \\\n echo \"spark.hadoop.yarn.timeline-service.enabled=false\" >> /usr/local/spark/conf/spark-defaults.conf && \\\n chown -R \"${NB_USER}:users\" /usr/local/spark/conf/spark-defaults.conf && \\\n # Create an alternative HADOOP_CONF_HOME so we can mount as a volume and repoint\n # using ENV var if needed\n mkdir -p /etc/hadoop/conf/ && \\\n chown \"${NB_USER}\":users /etc/hadoop/conf/\n\nUSER ${NB_UID}\n\n# Install useful jupyter extensions and python libraries like :\n# - Dashboards\n# - PyDoop\n# - PyHive\nRUN pip install --no-cache-dir 'jupyter_dashboards' 'faker' && \\\n jupyter dashboards quick-setup --sys-prefix && \\\n pip2 install --no-cache-dir 'pyhive' 'pydoop' 'thrift' 'sasl' 'thrift_sasl' 'faker' && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\nUSER root\n# Ensure we overwrite the kernel config so that toree connects to cluster\nRUN jupyter toree install --sys-prefix --spark_opts=\"\\\n --master yarn \\\n --deploy-mode client \\\n --driver-memory 512m \\\n --executor-memory 512m \\\n --executor-cores 1 \\\n --driver-java-options \\\n -Dhdp.version=2.5.3.0-37 \\\n --conf spark.hadoop.yarn.timeline-service.enabled=false \\\n\"\nUSER ${NB_UID}\n```\n\nCredit: [britishbadger](https://github.com/britishbadger) from [docker-stacks/issues/369](https://github.com/jupyter/docker-stacks/issues/369)\n\n## Run Server inside an already secured environment (i.e., with no token)\n\nThe default security is very good.\nThere are use cases, encouraged by containers, where the jupyter container and the system it runs within lie inside the security boundary.\nIt is convenient to launch the server without a password or token in these use cases.\nIn this case, you should use the `start-notebook.py` script to launch the server with no token:\n\nFor JupyterLab:\n\n```bash\ndocker run -it --rm \\\n quay.io/jupyter/base-notebook \\\n start-notebook.py --IdentityProvider.token=''\n```\n\nFor Jupyter Notebook:\n\n```bash\ndocker run -it --rm \\\n -e DOCKER_STACKS_JUPYTER_CMD=notebook \\\n quay.io/jupyter/base-notebook \\\n start-notebook.py --IdentityProvider.token=''\n```\n\n## Enable nbclassic-extension spellchecker for markdown (or any other nbclassic-extension)\n\n```{note}\nThis recipe only works for NBClassic with Jupyter Notebook < 7.\nIt is recommended to use [jupyterlab-spellchecker](https://github.com/jupyterlab-contrib/spellchecker) in modern environments.\n```\n\n```{literalinclude} recipe_code/spellcheck_notebook_v6.dockerfile\n:language: docker\n```\n\n## Enable Delta Lake in Spark notebooks\n\n```{warning}\nThis recipe is not tested and might be broken.\n```\n\nPlease note that the [Delta Lake](https://delta.io/) packages are only available for Spark version > `3.0`.\nBy adding the properties to `spark-defaults.conf`, the user no longer needs to enable Delta support in each notebook.\n\n```dockerfile\nFROM quay.io/jupyter/pyspark-notebook\n\nRUN mamba install --yes 'delta-spark' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\nUSER root\n\nRUN echo 'spark.sql.extensions io.delta.sql.DeltaSparkSessionExtension' >> \"${SPARK_HOME}/conf/spark-defaults.conf\" && \\\n echo 'spark.sql.catalog.spark_catalog org.apache.spark.sql.delta.catalog.DeltaCatalog' >> \"${SPARK_HOME}/conf/spark-defaults.conf\"\n\nUSER ${NB_UID}\n\n# Trigger download of delta lake files\nRUN echo \"from pyspark.sql import SparkSession\" > /tmp/init-delta.py && \\\n echo \"from delta import *\" >> /tmp/init-delta.py && \\\n echo \"spark = configure_spark_with_delta_pip(SparkSession.builder).getOrCreate()\" >> /tmp/init-delta.py && \\\n python /tmp/init-delta.py && \\\n rm /tmp/init-delta.py\n```\n\n## Add Custom Fonts in Scipy notebook\n\n```{warning}\nThis recipe is not tested and might be broken.\n```\n\nThe example below is a Dockerfile to load Source Han Sans with normal weight, usually used for the web.\n\n```dockerfile\nFROM quay.io/jupyter/scipy-notebook\n\nRUN PYV=$(ls \"${CONDA_DIR}/lib\" | grep ^python) && \\\n MPL_DATA=\"${CONDA_DIR}/lib/${PYV}/site-packages/matplotlib/mpl-data\" && \\\n wget --progress=dot:giga -P \"${MPL_DATA}/fonts/ttf/\" https://mirrors.cloud.tencent.com/adobe-fonts/source-han-sans/SubsetOTF/CN/SourceHanSansCN-Normal.otf && \\\n sed -i 's/#font.family/font.family/g' \"${MPL_DATA}/matplotlibrc\" && \\\n sed -i 's/#font.sans-serif:/font.sans-serif: Source Han Sans CN,/g' \"${MPL_DATA}/matplotlibrc\" && \\\n sed -i 's/#axes.unicode_minus: True/axes.unicode_minus: False/g' \"${MPL_DATA}/matplotlibrc\" && \\\n rm -rf \"/home/${NB_USER}/.cache/matplotlib\" && \\\n python -c 'import matplotlib.font_manager;print(\"font loaded: \",(\"Source Han Sans CN\" in [f.name for f in matplotlib.font_manager.fontManager.ttflist]))'\n```\n\n## Enable clipboard in pandas on Linux systems\n\n```{warning}\nThis recipe is not tested and might be broken.\n```\n\n```{admonition} Additional notes\n This solution works on Linux host systems.\n It is not required on Windows and won't work on macOS.\n```\n\nTo enable the `pandas.read_clipboard()` functionality, you need to have `xclip` installed\n(installed in `minimal-notebook` and all the inherited images)\nand add these options when running `docker`: `-e DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix`, i.e.:\n\n```bash\ndocker run -it --rm \\\n -e DISPLAY \\\n -v /tmp/.X11-unix:/tmp/.X11-unix \\\n quay.io/jupyter/minimal-notebook\n```\n\n## Install ijavascript kernel in your image\n\nThe example below is a Dockerfile to install the [IJavascript kernel](https://github.com/n-riesco/ijavascript).\n\n```{literalinclude} recipe_code/ijavascript.dockerfile\n:language: docker\n```\n\n## Add Microsoft SQL Server ODBC driver\n\nThe following recipe demonstrates how to add functionality to read from and write to an instance of Microsoft SQL server in your notebook.\n\n```{literalinclude} recipe_code/microsoft_odbc.dockerfile\n:language: docker\n```\n\nYou can now use `pyodbc` and `sqlalchemy` to interact with the database.\n\nPre-built images are hosted in the [Realiserad/jupyter-docker-mssql](https://github.com/Realiserad/jupyter-docker-mssql) repository.\n\n## Add Oracle SQL Instant client, SQL\\*Plus, and other tools (Version 21.x)\n\n```{note}\nThis recipe only works for x86_64 architecture.\n```\n\nThe following recipe demonstrates how to add functionality to connect to an Oracle Database using [Oracle Instant Client](https://www.oracle.com/database/technologies/instant-client.html)\nin your notebook.\nThis recipe installs version `21.11.0.0.0`.\n\nNonetheless, go to the [Oracle Instant Client Download page](https://www.oracle.com/es/database/technologies/instant-client/linux-x86-64-downloads.html) for the complete list of versions available.\nYou may need to perform different steps for older versions;\nthey may be explained in the \"Installation instructions\" section of the Downloads page.\n\n```{literalinclude} recipe_code/oracledb.dockerfile\n:language: docker\n```\n\n## Running Jupyter Docker Stacks with Singularity\n\nYou can also start Jupyter Docker Stacks containers using **Singularity** instead of Docker. For example:\n\n```bash\nsingularity run --bind \"${PWD}:/home/${USER}/work\" --containall docker://quay.io/jupyter/datascience-notebook:2025-12-31\n```\n\n- `--bind \"${PWD}:/home/${USER}/work\"` mounts your current working directory into the container at `/home/$USER/work`.\n When running the image with Singularity, the container uses your host username inside the container.\n Therefore, the bind target is `/home/${USER}/work` instead of the usual `/home/jovyan/work`.\n\n- `--containall` starts the container in a fully isolated environment, ignoring most of the hostโ€™s filesystem and environment except for explicitly bound paths.\n By default, Singularity would bind your home directory automatically.\n If you have Python packages installed there, this may cause conflicts with packages inside the container.\n Using `--containall` avoids such interference.\n" }, { "path": "docs/using/running.md", "content": "# Running a Container\n\nUsing one of the Jupyter Docker Stacks requires two choices:\n\n1. Which Docker image you wish to use\n2. How you wish to start Docker containers from that image\n\nThis section provides details about the second.\n\n## Using the Docker CLI\n\nYou can launch a local Docker container from the Jupyter Docker Stacks using the [Docker command-line interface](https://docs.docker.com/reference/cli/docker/).\nThere are numerous ways to configure containers using CLI.\nThe following are some common patterns.\n\n### Example 1\n\nThis command pulls the `jupyter/scipy-notebook` image tagged `2025-12-31` from Quay.io if it is not already present on the local host.\nIt then starts a container running a Jupyter Server with the JupyterLab frontend and exposes the server on host port 8888.\nThe server logs appear in the terminal and include a URL to the server.\n\n```bash\ndocker run -it -p 8888:8888 quay.io/jupyter/scipy-notebook:2025-12-31\n\n# Entered start.sh with args: jupyter lab\n\n# ...\n\n# To access the server, open this file in a browser:\n# file:///home/jovyan/.local/share/jupyter/runtime/jpserver-7-open.html\n# Or copy and paste one of these URLs:\n# http://eca4aa01751c:8888/lab?token=d4ac9278f5f5388e88097a3a8ebbe9401be206cfa0b83099\n# http://127.0.0.1:8888/lab?token=d4ac9278f5f5388e88097a3a8ebbe9401be206cfa0b83099\n```\n\nPressing `Ctrl-C` twice shuts down the Server but leaves the container intact on disk for later restart or permanent deletion using commands like the following:\n\n```bash\n# list containers\ndocker ps --all\n# CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\n# eca4aa01751c quay.io/jupyter/scipy-notebook:2025-12-31 \"tini -g -- start-noโ€ฆ\" About a minute ago Exited (0) 5 seconds ago silly_panini\n\n# start the stopped container\ndocker start --attach -i eca4aa01751c\n# Entered start.sh with args: jupyter lab\n# ...\n\n# remove the stopped container\ndocker rm eca4aa01751c\n# eca4aa01751c\n```\n\n### Example 2\n\nThis command pulls the `jupyter/r-notebook` image tagged `2025-12-31` from Quay.io if it is not already present on the local host.\nIt then starts a container running a Jupyter Server and exposes the server on host port 10000.\nThe server logs appear in the terminal and include a URL to the Server but with the internal container port (8888) instead of the correct host port (10000).\n\n```bash\ndocker run -it --rm -p 10000:8888 -v \"${PWD}\":/home/jovyan/work quay.io/jupyter/r-notebook:2025-12-31\n```\n\nPressing `Ctrl-C` twice shuts down the Server and immediately destroys the Docker container.\nNew files and changes in `~/work` in the container will be preserved.\nAny other changes made in the container will be lost.\n\n```{note}\nBy default, [jupyter's root_dir](https://jupyter-server.readthedocs.io/en/latest/other/full-config.html) is `/home/jovyan`.\nSo, new notebooks will be saved there, unless you change the directory in the file browser.\n\nTo change the default directory, you will need to specify `ServerApp.root_dir` by adding this line to the previous command: `start-notebook.py --ServerApp.root_dir=/home/jovyan/work`.\n```\n\n### Example 3\n\nThis command pulls the `jupyter/all-spark-notebook` image currently tagged `latest` from Quay.io if an image tagged `latest` is not already present on the local host.\nIt then starts a container named `notebook` running a JupyterLab server and exposes the server on a randomly selected port.\n\n```bash\ndocker run --detach -P --name notebook quay.io/jupyter/all-spark-notebook\n```\n\nwhere:\n\n- `--detach`: will run the container in detached mode\n\nYou can also use the following docker commands to see the port and Jupyter Server token:\n\n```bash\n# get the random host port assigned to the container port 8888\ndocker port notebook 8888\n# 0.0.0.0:49153\n# :::49153\n\n# get the notebook token from the logs\ndocker logs --tail 3 notebook\n # Or copy and paste one of these URLs:\n # http://878f1a9b4dfa:8888/lab?token=d336fa63c03f064ff15ce7b269cab95b2095786cf9ab2ba3\n # or http://127.0.0.1:8888/lab?token=d336fa63c03f064ff15ce7b269cab95b2095786cf9ab2ba3\n```\n\nTogether, the URL to visit on the host machine to access the server, in this case, is .\n\nThe container runs in the background until stopped and/or removed by additional Docker commands:\n\n```bash\n# stop the container\ndocker stop notebook\n# notebook\n\n# remove the container permanently\ndocker rm notebook\n# notebook\n```\n\n## Using the Podman CLI\n\nAn alternative to using the Docker CLI is to use the Podman CLI.\nPodman is mostly compatible with Docker.\n\n### Podman example\n\nIf we use Podman instead of Docker in the situation given in _Example 2_, it will look like this:\n\nThe example makes use of rootless Podman; in other words, the Podman command is run from a regular user account.\nIn a Bash shell, set the shell variables _uid_ and _gid_ to the UID and GID of the user _jovyan_ in the container.\n\n```bash\nuid=1000\ngid=100\n```\n\nSet the shell variables _subuidSize_ and _subgidSize_ to the number of subordinate UIDs and GIDs, respectively.\n\n```bash\nsubuidSize=$(( $(podman info --format \"{{ range .Host.IDMappings.UIDMap }}+{{.Size }}{{end }}\" ) - 1 ))\nsubgidSize=$(( $(podman info --format \"{{ range .Host.IDMappings.GIDMap }}+{{.Size }}{{end }}\" ) - 1 ))\n```\n\nThis command pulls the `quay.io/jupyter/r-notebook` image tagged `2025-12-31` from Quay.io if it is not already present on the local host.\nIt then starts a container running a Jupyter Server with the JupyterLab frontend and exposes the server on host port 10000.\nThe server logs appear in the terminal and include a URL to the server but with the internal container port (8888) instead of the correct host port (10000).\n\n```bash\npodman run -it --rm -p 10000:8888 \\\n -v \"${PWD}\":/home/jovyan/work --user $uid:$gid \\\n --uidmap $uid:0:1 --uidmap 0:1:$uid --uidmap $(($uid+1)):$(($uid+1)):$(($subuidSize-$uid)) \\\n --gidmap $gid:0:1 --gidmap 0:1:$gid --gidmap $(($gid+1)):$(($gid+1)):$(($subgidSize-$gid)) \\\n quay.io/jupyter/r-notebook:2025-12-31\n```\n\n```{warning}\nThe `podman run` options `--uidmap` and `--gidmap` can be used to map the container user _jovyan_ to the regular user on the host when running rootless Podman.\nThe same Podman command should not be run with sudo (i.e. running rootful Podman)\nbecause then the mapping would map the container user _jovyan_ to the root user on the host.\nIt's a good security practice to run programs with as few privileges as possible.\n```\n\n```{note}\nThe `podman run` command in the example above maps all subuids and subgids of the user into the container.\nThat works fine but is actually more than needed.\nThe `podman run` option `--userns=auto` will, for instance, not be possible to use as long as there are no unused subuids and subgids available.\nThe example could be improved by investigating more in detail which UIDs and GIDs need to be available in the container and then only map them.\n```\n\nPressing `Ctrl-C` twice shuts down the Server and immediately destroys the Docker container.\nNew files and changes in `~/work` in the container will be preserved.\nAny other changes made in the container will be lost.\n\n## Using Binder\n\nA [Binder](https://mybinder.org/) is a service that allows you to create and share custom computing environments for projects in version control.\nYou can use any of the Jupyter Docker Stacks images as a basis for a Binder-compatible Dockerfile.\nSee the\n[docker-stacks example](https://mybinder.readthedocs.io/en/latest/examples/sample_repos.html#using-a-docker-image-from-the-jupyter-docker-stacks-repository) and\n[Using a Dockerfile](https://mybinder.readthedocs.io/en/latest/tutorials/dockerfile.html) section in the\n[Binder documentation](https://mybinder.readthedocs.io/en/latest/index.html) for instructions.\n\n## Using JupyterHub\n\nYou can configure JupyterHub to launch Docker containers from the Jupyter Docker Stacks images.\nIf you've been following the [Zero to JupyterHub with Kubernetes](https://z2jh.jupyter.org/en/latest/) guide,\nsee the [Use an existing Docker image](https://z2jh.jupyter.org/en/latest/jupyterhub/customizing/user-environment.html#choose-and-use-an-existing-docker-image) section for details.\nIf you have a custom JupyterHub deployment, see the [Picking or building a Docker image](https://jupyterhub-dockerspawner.readthedocs.io/en/latest/docker-image.html)\ninstructions for the [dockerspawner](https://github.com/jupyterhub/dockerspawner) instead.\n\n## Using Other Tools and Services\n\nYou can use the Jupyter Docker Stacks with any Docker-compatible technology\n(e.g., [Docker Compose](https://docs.docker.com/compose/), [docker-py](https://github.com/docker/docker-py), or your favorite cloud container service).\nSee the documentation of the tool, library, or service for details about how to reference, configure, and launch containers from these images.\n" }, { "path": "docs/using/selecting.md", "content": "# Selecting an Image\n\n- [Core Stacks](#core-stacks)\n- [Image Relationships](#image-relationships)\n- [Community Stacks](#community-stacks)\n\n> **Pull vs Build: Quick Guidance**\n>\n> Images are published on **Quay.io** registry and most users should start by pulling an existing image that closely matches their needs.\n> Pulling a pre-built image is fast and suitable for common use cases such as running notebooks, teaching, or standard data science workflows.\n>\n> Building images locally is recommended only when additional customization is required, for example:\n>\n> - adding system-level packages that cannot be installed at runtime,\n> - preinstalling large or complex language-specific libraries,\n> - creating organization-specific images or heavy customization for CI.\n>\n> If unsure, try an existing Quay.io image first; if it falls short, follow the custom images guide to build a tailored image.\n\nUsing one of the Jupyter Docker Stacks requires two choices:\n\n1. Which Docker image you wish to use\n2. How you wish to start Docker containers from that image\n\nThis section provides details about the first.\n\n## Core Stacks\n\nThe Jupyter team maintains a set of Docker image definitions in the GitHub repository.\nThe following sections describe these images, including their contents, relationships, and versioning strategy.\n\n### jupyter/docker-stacks-foundation\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/docker-stacks-foundation) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/docker-stacks-foundation/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/docker-stacks-foundation?tab=tags)\n\n`jupyter/docker-stacks-foundation` is a small image supporting a majority of [options common across all core stacks](common.md).\nIt is the basis for all other stacks on which Jupyter-related applications can be built\n(e.g., kernel-based containers, [nbclient](https://github.com/jupyter/nbclient) applications, etc.).\nAs such, it does not contain application-level software like JupyterLab, Jupyter Notebook, or JupyterHub.\n\nIt contains:\n\n- Package managers\n - [conda](https://github.com/conda/conda): \"cross-platform, language-agnostic binary package manager\".\n - [mamba](https://github.com/mamba-org/mamba): \"reimplementation of the conda package manager in C++\". We use this package manager by default when installing packages.\n- Unprivileged user `jovyan` (`uid=1000`, configurable, [see options in the common features section](./common.md) of this documentation) in group `users` (`gid=100`)\n with ownership over the `/home/jovyan` and `/opt/conda` paths\n- `tini` and a `start.sh` script as the container entry point - useful for running alternative commands in the container as applications are added (e.g. `ipython`, `jupyter kernelgateway`, `jupyter lab`)\n- A `run-hooks.sh` script, which can source/run files in a given directory\n- Options for a passwordless sudo\n- Common system libraries like `bzip2`, `ca-certificates`, `locales`\n- `wget` to download external files\n- No preinstalled scientific computing packages\n\n### jupyter/base-notebook\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/base-notebook) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/base-notebook/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/base-notebook?tab=tags)\n\n`jupyter/base-notebook` adds base Jupyter Applications like JupyterLab, Jupyter Notebook, JupyterHub, and NBClassic\nand serves as the basis for all other stacks besides `jupyter/docker-stacks-foundation`.\n\nIt contains:\n\n- Everything in `jupyter/docker-stacks-foundation`\n- Minimally functional Server (e.g., no LaTeX support for saving notebooks as PDFs)\n- `notebook`, `jupyterhub-singleuser`, and `jupyterlab` packages\n- A `start-notebook.py` script as the default command\n- A `start-singleuser.py` script useful for launching containers in JupyterHub\n- Options for a self-signed HTTPS certificate\n\n```{warning}\n`jupyter/base-notebook` also contains `start-notebook.sh` and `start-singleuser.sh` files to maintain backward compatibility.\nExternal config that explicitly refers to those files should instead\nupdate to refer to `start-notebook.py` and `start-singleuser.py`.\nThe shim `.sh` files will be removed at some future date.\n```\n\n### jupyter/minimal-notebook\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/minimal-notebook) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/minimal-notebook/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/minimal-notebook?tab=tags)\n\n`jupyter/minimal-notebook` adds command-line tools useful when working in Jupyter applications.\n\nIt contains:\n\n- Everything in `jupyter/base-notebook`\n- Common useful utilities like\n [curl](https://curl.se),\n [git](https://git-scm.com/),\n [nano](https://www.nano-editor.org/) (actually `nano-tiny`),\n [tzdata](https://www.iana.org/time-zones),\n [unzip](https://code.launchpad.net/ubuntu/+source/unzip),\n and [vi](https://www.vim.org) (actually `vim-tiny`),\n- [TeX Live](https://www.tug.org/texlive/) for notebook document conversion\n\n### jupyter/r-notebook\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/r-notebook) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/r-notebook/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/r-notebook?tab=tags)\n\n`jupyter/r-notebook` includes popular packages from the R ecosystem listed below:\n\n- Everything in `jupyter/minimal-notebook` and its ancestor images\n- The [R](https://www.r-project.org/) interpreter and base environment\n- [IRKernel](https://irkernel.github.io/) to support R code in Jupyter notebooks\n- [tidyverse](https://tidyverse.org/)\n packages from [conda-forge](https://conda-forge.org/feedstock-outputs/index.html)\n- [caret](https://topepo.github.io/caret/index.html),\n [crayon](https://cran.r-project.org/web/packages/crayon/index.html),\n [devtools](https://cran.r-project.org/web/packages/devtools/index.html),\n [forecast](https://cran.r-project.org/web/packages/forecast/index.html),\n [hexbin](https://cran.r-project.org/web/packages/hexbin/index.html),\n [htmltools](https://cran.r-project.org/web/packages/htmltools/index.html),\n [htmlwidgets](https://www.htmlwidgets.org),\n [nycflights13](https://cran.r-project.org/web/packages/nycflights13/index.html),\n [randomforest](https://cran.r-project.org/web/packages/randomForest/index.html),\n [rcurl](https://cran.r-project.org/web/packages/RCurl/index.html),\n [rmarkdown](https://rmarkdown.rstudio.com),\n [rodbc](https://cran.r-project.org/web/packages/RODBC/index.html),\n [rsqlite](https://cran.r-project.org/web/packages/RSQLite/index.html),\n [shiny](https://shiny.posit.co),\n [tidymodels](https://www.tidymodels.org/),\n [unixodbc](https://www.unixodbc.org)\n packages from [conda-forge](https://conda-forge.org/feedstock-outputs/index.html)\n\n### jupyter/julia-notebook\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/julia-notebook) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/julia-notebook/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/julia-notebook?tab=tags)\n\n`jupyter/julia-notebook` includes popular packages from the Julia ecosystem listed below:\n\n- Everything in `jupyter/minimal-notebook` and its ancestor images\n- The [Julia](https://julialang.org/) compiler and base environment\n- [IJulia](https://github.com/JuliaLang/IJulia.jl) to support Julia code in Jupyter notebook\n- [Pluto.jl](https://plutojl.org/) reactive Julia notebook interface, made accessible with [jupyter-pluto-proxy](https://github.com/yuvipanda/jupyter-pluto-proxy)\n- [HDF5](https://github.com/JuliaIO/HDF5.jl) package\n\n### jupyter/scipy-notebook\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/scipy-notebook) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/scipy-notebook/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/scipy-notebook?tab=tags)\n\n`jupyter/scipy-notebook` includes popular packages from the scientific Python ecosystem.\n\n- Everything in `jupyter/minimal-notebook` and its ancestor images\n- [altair](https://altair-viz.github.io),\n [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/),\n [bokeh](https://docs.bokeh.org/en/latest/),\n [bottleneck](https://bottleneck.readthedocs.io/en/latest/),\n [cloudpickle](https://github.com/cloudpipe/cloudpickle),\n [conda-forge::blas=\\*=openblas](https://www.openblas.net),\n [cython](https://cython.org),\n [dask](https://www.dask.org/),\n [dill](https://pypi.org/project/dill/),\n [h5py](https://www.h5py.org),\n [jupyterlab-git](https://github.com/jupyterlab/jupyterlab-git),\n [matplotlib-base](https://matplotlib.org/),\n [numba](https://numba.pydata.org/),\n [numexpr](https://github.com/pydata/numexpr),\n [openpyxl](https://openpyxl.readthedocs.io/en/stable/),\n [pandas](https://pandas.pydata.org/),\n [patsy](https://patsy.readthedocs.io/en/latest/),\n [protobuf](https://protobuf.dev/getting-started/pythontutorial/),\n [pytables](https://www.pytables.org/),\n [scikit-image](https://scikit-image.org),\n [scikit-learn](https://scikit-learn.org/stable/),\n [scipy](https://scipy.org/),\n [seaborn](https://seaborn.pydata.org/),\n [sqlalchemy](https://www.sqlalchemy.org/),\n [statsmodel](https://www.statsmodels.org/stable/index.html),\n [sympy](https://www.sympy.org/en/index.html),\n [widgetsnbextension](https://ipywidgets.readthedocs.io/en/latest/user_install.html#installing-in-classic-jupyter-notebook),\n [xlrd](https://www.python-excel.org)\n packages\n- [ipympl](https://github.com/matplotlib/ipympl) and\n [ipywidgets](https://ipywidgets.readthedocs.io/en/stable/)\n for interactive visualizations and plots in Python notebooks\n\n### jupyter/tensorflow-notebook\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/tensorflow-notebook) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/tensorflow-notebook/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/tensorflow-notebook?tab=tags)\n\n`jupyter/tensorflow-notebook` includes popular Python deep learning libraries.\n\n- Everything in `jupyter/scipy-notebook` and its ancestor images\n- [TensorFlow](https://www.tensorflow.org/) machine learning library\n- [Jupyter Server Proxy](https://jupyter-server-proxy.readthedocs.io/en/latest/) to support [TensorBoard](https://www.tensorflow.org/tensorboard)\n\n### jupyter/pytorch-notebook\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/pytorch-notebook) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/pytorch-notebook/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/pytorch-notebook?tab=tags)\n\n`jupyter/pytorch-notebook` includes popular Python deep learning libraries.\n\n- Everything in `jupyter/scipy-notebook` and its ancestor images\n- [pytorch](https://pytorch.org/) machine learning library\n\n### jupyter/datascience-notebook\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/datascience-notebook) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/datascience-notebook/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/datascience-notebook?tab=tags)\n\n`jupyter/datascience-notebook` includes libraries for data analysis from the Python, R, and Julia communities.\n\n- Everything in the `jupyter/scipy-notebook`, `jupyter/r-notebook`, and `jupyter/julia-notebook` images and their ancestor\n images\n- [rpy2](https://rpy2.github.io/doc/latest/html/index.html) package\n\n### jupyter/pyspark-notebook\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/pyspark-notebook) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/pyspark-notebook/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/pyspark-notebook?tab=tags)\n\n`jupyter/pyspark-notebook` includes Python support for Apache Spark.\n\n- Everything in `jupyter/scipy-notebook` and its ancestor images\n- [Apache Spark](https://spark.apache.org/) with Hadoop binaries\n- [grpcio-status](https://github.com/grpc/grpc/tree/master/src/python/grpcio_status)\n- [grpcio](https://grpc.io/docs/languages/python/quickstart/)\n- [pyarrow](https://arrow.apache.org/docs/python/)\n\n### jupyter/all-spark-notebook\n\n[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/images/all-spark-notebook) |\n[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/all-spark-notebook/Dockerfile) |\n[Quay.io image tags](https://quay.io/repository/jupyter/all-spark-notebook?tab=tags)\n\n`jupyter/all-spark-notebook` includes Python and R support for Apache Spark.\n\n- Everything in `jupyter/pyspark-notebook` and its ancestor images\n- [IRKernel](https://irkernel.github.io/) to support R code in Jupyter notebooks\n- [rcurl](https://cran.r-project.org/web/packages/RCurl/index.html),\n [sparklyr](https://spark.posit.co),\n [ggplot2](https://ggplot2.tidyverse.org)\n packages\n\n### CUDA enabled variants\n\nWe provide CUDA accelerated versions of the `pytorch-notebook` and `tensorflow-notebook` images.\nPrepend a CUDA prefix (versioned prefix like `cuda12-` for `pytorch-notebook` or just `cuda-` for `tensorflow-notebook`) to the image tag\nto allow PyTorch or TensorFlow operations to use compatible NVIDIA GPUs for accelerated computation.\nWe only build `pytorch-notebook` for the last two major versions of CUDA.\nThe `tensorflow-notebook` image only supports the latest CUDA version listed in the [officially tested build configurations](https://www.tensorflow.org/install/source#gpu).\n\nFor example, you could use the image `quay.io/jupyter/pytorch-notebook:cuda12-python-3.11.8` or `quay.io/jupyter/tensorflow-notebook:cuda-latest`.\n\n### Image Relationships\n\nThe following diagram depicts the build dependency tree of the core images. (i.e., the `FROM` statements in their Dockerfiles).\nAny given image inherits the complete content of all ancestor images pointing to it.\n\n[![Image inheritance\ndiagram](../images/inherit.svg)](http://interactive.blockdiag.com/?compression=deflate&src=eJyFj8FKxDAQhu_7FKEnRYKsnmTRJ9ibe1yQaTp1x2ZnSjJxqeK7mywopFB6Cnzz_X_4Wy9u6AjezffGmA57SF7femGN9IXm2TztMpdAyApKwhmNEjQA6W6TT01qE2s68s3-8GoupKd8J1YT0CNEvG2MfTFd_gWDjQpuiLaXxN21rZQv3UquzQ2WRbEVGYpcgWKciekMvpLmrHjR0ThVVk2K0yhylNB7ufzzI9-51IH5hEDA2iwkx0kluNM8tn24vz6Pf_m4UJAnQ6bIDlenhFXjI3mClbHjFEcIQ2XNWfHAezs3f34BzoTGIw)\n\n### Builds\n\nEvery Monday and whenever a pull request is merged, images are rebuilt and pushed to [the public container registry](https://quay.io/organization/jupyter).\n\n### Versioning via image tags\n\nWhenever a docker image is pushed to the container registry, it is tagged with:\n\n- the `latest` tag\n- a 12-character git commit SHA like `1ffe43816ba9`\n- a date formatted like `2023-01-30`\n- OS version like `ubuntu-22.04`\n- a set of software version tags like `python-3.10.8` and `lab-3.5.3`\n\n```{warning}\n- Tags before `2022-07-05` were sometimes incorrect.\n Please, do not rely on them.\n- Single-platform images have either `aarch64-` or `x86_64-` tag prefixes, for example, `quay.io/jupyter/base-notebook:aarch64-python-3.11.6`\n```\n\nFor stability and reproducibility, you should either reference a date formatted\ntag from a date before the current date (in UTC) or a git commit SHA older\nthan the latest git commit SHA in the default branch of the\n[jupyter/docker-stacks GitHub repository](https://github.com/jupyter/docker-stacks/).\n\n## Community Stacks\n\nThe core stacks are but a tiny sample of what's possible when combining Jupyter with other technologies.\nWe encourage members of the Jupyter community to create their own stacks based on the core images and link them below.\nSee the [contributing guide](../contributing/stacks.md) for information about how to create your own Jupyter Docker Stack.\n\n| Flavor | Binder | Description |\n| -------------- | ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------- |\n| [csharp] | [![bb]][csharp_b] | More than 200 Jupyter Notebooks with example **C#** code |\n| [education] | [![bb]][education_b] | **`nbgrader`** and `RISE` on top of the `datascience-notebook` image |\n| [ihaskell] | [![bb]][ihaskell_b] | Based on [**IHaskell**][ihaskell_project]. Includes popular packages and example notebooks |\n| [java] | [![bb]][java_b] | [**IJava**][ijava] kernel on top of the `minimal-notebook` image |\n| [sage] | [![bb]][sage_b] | [**sagemath**][sagemath] kernel on top of the `minimal-notebook` image |\n| [cgspatial] | [![bb]][cgspatial_b] | Major **geospatial** Python & R libraries on top of the `datascience-notebook` image |\n| [kotlin] | [![bb]][kotlin_b] | [**Kotlin** kernel for Jupyter/IPython][kotlin_kernel] on top of the `base-notebook` image |\n| [transformers] | [![bb]][transformers_b] | [**Transformers**][transformers_lib] and NLP libraries such as `Tensorflow`, `Keras`, `Jax` and `PyTorch` |\n| [scraper] | [![bb]][scraper_b] | **Scraper** tools (`selenium`, `chromedriver`, `beatifulsoup4`, `requests`) on `minimal-notebook` image |\n| [almond] | [![bb]][almond_b] | Scala kernel for Jupyter using **Almond** on top of the `base-notebook` image |\n| [lisp-stat] | [![bb]][lisp-stat_b] | Common Lisp statistical computing environment on top of the `minimal-notebook` image |\n| [sequencing] | [![bb]][sequencing_b] | Collection for bioinformatics sequencing data analysis, covering bulk RNA-seq, single-cell RNA-seq, spatial sequencing, and multi-omics |\n\n[bb]: https://static.mybinder.org/badge_logo.svg\n[csharp]: https://github.com/tlinnet/csharp-notebook\n[csharp_b]: https://mybinder.org/v2/gh/tlinnet/csharp-notebook/master\n[education]: https://github.com/umsi-mads/education-notebook\n[education_b]: https://mybinder.org/v2/gh/umsi-mads/education-notebook/master\n[ihaskell]: https://github.com/IHaskell/ihaskell-notebook\n[ihaskell_b]: https://mybinder.org/v2/gh/jamesdbrock/learn-you-a-haskell-notebook/master?urlpath=lab/tree/ihaskell_examples/ihaskell/IHaskell.ipynb\n[ihaskell_project]: https://github.com/IHaskell/IHaskell\n[java]: https://github.com/jbindinga/java-notebook\n[java_b]: https://mybinder.org/v2/gh/jbindinga/java-notebook/master\n[ijava]: https://github.com/SpencerPark/IJava\n[sage]: https://github.com/sharpTrick/sage-notebook\n[sage_b]: https://mybinder.org/v2/gh/sharpTrick/sage-notebook/master\n[sagemath]: https://www.sagemath.org\n[cgspatial]: https://github.com/SCiO-systems/cgspatial-notebook\n[cgspatial_b]: https://mybinder.org/v2/gh/SCiO-systems/cgspatial-notebook/master\n[kotlin]: https://github.com/knonm/kotlin-notebook\n[kotlin_b]: https://mybinder.org/v2/gh/knonm/kotlin-notebook/main\n[kotlin_kernel]: https://github.com/Kotlin/kotlin-jupyter\n[transformers]: https://github.com/ToluClassics/transformers_notebook\n[transformers_b]: https://mybinder.org/v2/gh/ToluClassics/transformers_notebook/main\n[transformers_lib]: https://huggingface.co/docs/transformers/index\n[scraper]: https://github.com/rgriffogoes/scraper-notebook\n[scraper_b]: https://mybinder.org/v2/gh/rgriffogoes/scraper-notebook/main\n[almond]: https://almond.sh\n[almond_b]: https://mybinder.org/v2/gh/almond-sh/examples/master?urlpath=lab%2Ftree%2Fnotebooks%2Findex.ipynb\n[lisp-stat]: https://lisp-stat.dev\n[lisp-stat_b]: https://mybinder.org/v2/gh/Lisp-Stat/IPS9/HEAD?urlpath=%2Fdoc%2Ftree%2Findex.ipynb\n[sequencing]: https://github.com/huchlab/sequencing-docker-stacks\n[sequencing_b]: https://mybinder.org/v2/gh/huchlab/sequencing-docker-stacks/main?urlpath=lab%2Ftree%2FREADME.ipynb\n\n### Other GPU-accelerated notebooks\n\n| Flavor | Description |\n| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |\n| [GPU-Jupyter][gpu] | Power of your NVIDIA GPU and GPU calculations using Tensorflow and Pytorch in collaborative notebooks. This is done by generating a Dockerfile that consists of the **nvidia/cuda** base image, the well-maintained **docker-stacks** that is integrated as a submodule, and GPU-able libraries like **Tensorflow**, **Keras** and **PyTorch** on top of it. |\n| [myLab TH Lรผbeck Images][gpu_thl] | Images based on the **jupyter/docker-stacks**, built and maintained at the [myLab TH Lรผbeck][gpu_mylab] using build scripts similar to iot-salzburg. Several images include GPU libraries. |\n| [PRP-GPU][prp_gpu] | PRP (Pacific Research Platform) maintained [registry][prp_reg] for jupyter stack based on NVIDIA CUDA-enabled image. Added the PRP image with Pytorch and some other Python packages and GUI Desktop notebook based on . |\n| [b-data][b-data] | GPU accelerated, multi-arch (`linux/amd64`, `linux/arm64/v8`) Docker images for [R][r_cuda], [Python][python_cuda] , [MAX][max_cuda] and [Julia][julia_cuda]. Derived from nvidia/cuda `devel`-flavored images. With [code-server][code-server] next to JupyterLab. Just Python โ€“ no [Conda][conda] / [Mamba][mamba]. |\n\n[gpu]: https://github.com/iot-salzburg/gpu-jupyter\n[gpu_thl]: https://hub.docker.com/r/hanseware/jlab-images\n[gpu_mylab]: https://mylab.th-luebeck.de\n[prp_gpu]: https://gitlab.nrp-nautilus.io/prp/jupyter-stack/-/tree/prp\n[prp_reg]: https://gitlab.nrp-nautilus.io/prp/jupyter-stack/container_registry\n[b-data]: https://github.com/b-data\n[r_cuda]: https://github.com/b-data/jupyterlab-r-docker-stack/blob/main/CUDA.md\n[python_cuda]: https://github.com/b-data/jupyterlab-python-docker-stack/blob/main/CUDA.md\n[max_cuda]: https://github.com/b-data/jupyterlab-mojo-docker-stack/blob/main/CUDA.md\n[julia_cuda]: https://github.com/b-data/jupyterlab-julia-docker-stack/blob/main/CUDA.md\n[code-server]: https://github.com/coder/code-server\n[conda]: https://github.com/conda/conda\n[mamba]: https://github.com/mamba-org/mamba\n" }, { "path": "docs/using/specifics.md", "content": "# Image Specifics\n\nThis page provides details about features specific to one or more images.\n\n## Apache Sparkโ„ข\n\n### Specific Docker Image Options\n\n- `-p 4040:4040` - The `jupyter/pyspark-notebook` and `jupyter/all-spark-notebook` images open\n [SparkUI (Spark Monitoring and Instrumentation UI)](https://spark.apache.org/docs/latest/monitoring.html) at default port `4040`,\n this option maps the `4040` port inside the docker container to the `4040` port on the host machine.\n\n ```{note}\n Every new spark context that is created is put onto an incrementing port (i.e. 4040, 4041, 4042, etc.), and it might be necessary to open multiple ports.\n ```\n\n For example, `docker run --detach -p 8888:8888 -p 4040:4040 -p 4041:4041 quay.io/jupyter/pyspark-notebook`.\n\n#### IPython low-level output capture and forward\n\nSpark images (`pyspark-notebook` and `all-spark-notebook`) have been configured to disable IPython low-level output capture and forward system-wide.\nThe rationale behind this choice is that Spark logs can be verbose, especially at startup when Ivy is used to load additional jars.\nThose logs are still available but only in the container's logs.\n\nIf you want to make them appear in the notebook, you can overwrite the configuration in a user-level IPython kernel profile.\nTo do that, you have to uncomment the following line in your `~/.ipython/profile_default/ipython_kernel_config.py` and restart the kernel.\n\n```python\nc.IPKernelApp.capture_fd_output = True\n```\n\nIf you have no IPython profile, you can initiate a fresh one by running the following command.\n\n```bash\nipython profile create\n# [ProfileCreate] Generating default config file: '/home/jovyan/.ipython/profile_default/ipython_config.py'\n# [ProfileCreate] Generating default config file: '/home/jovyan/.ipython/profile_default/ipython_kernel_config.py'\n```\n\n### Build an Image with a Different Version of Spark\n\nYou can build a `pyspark-notebook` image with a different `Spark` version by overriding the default value of the following arguments at build time.\n`all-spark-notebook` is inherited from `pyspark-notebook`, so you have to first build `pyspark-notebook` and then `all-spark-notebook` to get the same version in `all-spark-notebook`.\n\n- Spark distribution is defined by the combination of Spark, Hadoop, and Scala versions,\n see [Download Apache Spark](https://spark.apache.org/downloads.html) and the [archive repo](https://archive.apache.org/dist/spark/) for more information.\n - `openjdk_version`: The version of the OpenJDK (JRE headless) distribution (`17` by default).\n - This version needs to match the version supported by the Spark distribution used above.\n - See [Spark Overview](https://spark.apache.org/docs/latest/#downloading) and [Ubuntu packages](https://packages.ubuntu.com/search?keywords=openjdk).\n - `spark_version` (optional): The Spark version to install, for example `3.5.0`.\n If not specified (this is the default), latest Spark will be installed.\n Note: to support Python 3.12, we currently install Spark v4 preview versions: .\n - `hadoop_version`: The Hadoop version (`3` by default).\n Note, that _Spark < 3.3_ require to specify `major.minor` Hadoop version (i.e. `3.2`).\n - `scala_version` (optional): The Scala version, for example `2.13` (not specified by default).\n Starting with _Spark >= 3.2_, the distribution file might contain the Scala version.\n - `spark_download_url`: URL to use for Spark downloads.\n You may need to use url if you want to download old Spark versions.\n\nFor example, here is how to build a `pyspark-notebook` image with Spark `3.2.0`, Hadoop `3.2`, and OpenJDK `11`.\n\n```{warning}\nThis recipe is not tested and might be broken.\n```\n\n```bash\n# From the root of the project\n# Build the image with different arguments\ndocker build --rm --force-rm \\\n -t my-pyspark-notebook ./images/pyspark-notebook \\\n --build-arg openjdk_version=11 \\\n --build-arg spark_version=3.2.0 \\\n --build-arg hadoop_version=3.2 \\\n --build-arg spark_download_url=\"https://archive.apache.org/dist/spark/\"\n\n# Check the newly built image\ndocker run -it --rm my-pyspark-notebook pyspark --version\n\n# Welcome to\n# ____ __\n# / __/__ ___ _____/ /__\n# _\\ \\/ _ \\/ _ `/ __/ '_/\n# /___/ .__/\\_,_/_/ /_/\\_\\ version 3.2.0\n# /_/\n\n# Using Scala version 2.12.15, OpenJDK 64-Bit Server VM, 11.0.21\n# Branch HEAD\n# Compiled by user ubuntu on 2021-10-06T12:46:30Z\n# Revision 5d45a415f3a29898d92380380cfd82bfc7f579ea\n# Url https://github.com/apache/spark\n# Type --help for more information.\n```\n\n### Usage Examples\n\nThe `jupyter/pyspark-notebook` and `jupyter/all-spark-notebook` images support the use of [Apache Spark](https://spark.apache.org/) in Python and R notebooks.\nThe following sections provide some examples of how to get started using them.\n\n#### Using Spark Local Mode\n\nSpark **local mode** is useful for experimentation on small data when you do not have a Spark cluster available.\n\n```{warning}\nIn these examples, Spark spawns all the main execution components in the same single JVM.\nYou can read additional info about local mode [here](https://books.japila.pl/apache-spark-internals/local/).\nIf you want to use all the CPU, one of the simplest ways is to set up a [Spark Standalone Cluster](https://spark.apache.org/docs/latest/spark-standalone.html).\n```\n\n##### Local Mode in Python\n\nIn a Python notebook.\n\n```python\nfrom pyspark.sql import SparkSession\n\n# Spark session & context\nspark = SparkSession.builder.master(\"local\").getOrCreate()\nsc = spark.sparkContext\n\n# Sum of the first 100 whole numbers\nrdd = sc.parallelize(range(100 + 1))\nrdd.sum()\n# 5050\n```\n\n##### Local Mode in R\n\nIn an R notebook with [SparkR][sparkr].\n\n```R\nlibrary(SparkR)\n\n# Spark session & context\nsc <- sparkR.session(\"local\")\n\n# Sum of the first 100 whole numbers\nsdf <- createDataFrame(list(1:100))\ndapplyCollect(sdf,\n function(x)\n { x <- sum(x)}\n )\n# 5050\n```\n\nIn an R notebook with [sparklyr][sparklyr].\n\n```R\nlibrary(sparklyr)\n\n# Spark configuration\nconf <- spark_config()\n# Set the catalog implementation in-memory\nconf$spark.sql.catalogImplementation <- \"in-memory\"\n\n# Spark session & context\nsc <- spark_connect(master = \"local\", config = conf)\n\n# Sum of the first 100 whole numbers\nsdf_len(sc, 100, repartition = 1) %>%\n spark_apply(function(e) sum(e))\n# 5050\n```\n\n#### Connecting to a Spark Cluster in Standalone Mode\n\nConnection to Spark Cluster on **[Standalone Mode](https://spark.apache.org/docs/latest/spark-standalone.html)** requires the following set of steps:\n\n0. Verify that the docker image (check the Dockerfile) and the Spark Cluster, which is being\n deployed, run the same version of Spark.\n1. [Deploy Spark in Standalone Mode](https://spark.apache.org/docs/latest/spark-standalone.html).\n2. Run the Docker container with `--net=host` in a location that is network-addressable by all of\n your Spark workers.\n (This is a [Spark networking requirement](https://spark.apache.org/docs/latest/cluster-overview.html#components).)\n\n ```{note}\n When using `--net=host`, you must also use the flags `--pid=host -e TINI_SUBREAPER=true`. See for details._\n ```\n\n**Note**: In the following examples, we are using the Spark master URL `spark://master:7077` which shall be replaced by the URL of the Spark master.\n\n##### Standalone Mode in Python\n\nThe **same Python version** needs to be used on the notebook (where the driver is located) and on the Spark workers.\nThe Python version used on the driver and worker side can be adjusted by setting the environment variables `PYSPARK_PYTHON` and/or `PYSPARK_DRIVER_PYTHON`,\nsee [Spark Configuration][spark-conf] for more information.\n\n```python\nfrom pyspark.sql import SparkSession\n\n# Spark session & context\nspark = SparkSession.builder.master(\"spark://master:7077\").getOrCreate()\nsc = spark.sparkContext\n\n# Sum of the first 100 whole numbers\nrdd = sc.parallelize(range(100 + 1))\nrdd.sum()\n# 5050\n```\n\n##### Standalone Mode in R\n\nIn an R notebook with [SparkR][sparkr].\n\n```R\nlibrary(SparkR)\n\n# Spark session & context\nsc <- sparkR.session(\"spark://master:7077\")\n\n# Sum of the first 100 whole numbers\nsdf <- createDataFrame(list(1:100))\ndapplyCollect(sdf,\n function(x)\n { x <- sum(x)}\n )\n# 5050\n```\n\nIn an R notebook with [sparklyr][sparklyr].\n\n```R\nlibrary(sparklyr)\n\n# Spark session & context\n# Spark configuration\nconf <- spark_config()\n# Set the catalog implementation in-memory\nconf$spark.sql.catalogImplementation <- \"in-memory\"\nsc <- spark_connect(master = \"spark://master:7077\", config = conf)\n\n# Sum of the first 100 whole numbers\nsdf_len(sc, 100, repartition = 1) %>%\n spark_apply(function(e) sum(e))\n# 5050\n```\n\n### Define Spark Dependencies\n\n```{note}\nThis example is given for [Elasticsearch](https://www.elastic.co/docs/reference/elasticsearch-hadoop/installation).\n```\n\nSpark dependencies can be declared thanks to the `spark.jars.packages` property\n(see [Spark Configuration](https://spark.apache.org/docs/latest/configuration.html#runtime-environment) for more information).\n\nThey can be defined as a comma-separated list of Maven coordinates at the creation of the Spark session.\n\n```python\nfrom pyspark.sql import SparkSession\n\nspark = (\n SparkSession.builder.appName(\"elasticsearch\")\n .config(\n \"spark.jars.packages\", \"org.elasticsearch:elasticsearch-spark-30_2.12:7.13.0\"\n )\n .getOrCreate()\n)\n```\n\nDependencies can also be defined in the `spark-defaults.conf`.\nHowever, it has to be done by `root`, so it should only be considered to build custom images.\n\n```dockerfile\nUSER root\nRUN echo \"spark.jars.packages org.elasticsearch:elasticsearch-spark-30_2.12:7.13.0\" >> \"${SPARK_HOME}/conf/spark-defaults.conf\"\nUSER ${NB_UID}\n```\n\nJars will be downloaded dynamically at the creation of the Spark session and stored by default in `${HOME}/.ivy2/jars` (can be changed by setting `spark.jars.ivy`).\n\n## Tensorflow\n\nThe `jupyter/tensorflow-notebook` image supports the use of\n[Tensorflow](https://www.tensorflow.org/) in a single machine or distributed mode.\n\n### Single Machine Mode\n\n```python\nimport tensorflow as tf\n\nhello = tf.Variable(\"Hello World!\")\n\nsess = tf.Session()\ninit = tf.global_variables_initializer()\n\nsess.run(init)\nsess.run(hello)\n```\n\n### Distributed Mode\n\n```python\nimport tensorflow as tf\n\nhello = tf.Variable(\"Hello Distributed World!\")\n\nserver = tf.train.Server.create_local_server()\nsess = tf.Session(server.target)\ninit = tf.global_variables_initializer()\n\nsess.run(init)\nsess.run(hello)\n```\n\n[sparkr]: https://spark.apache.org/docs/latest/sparkr.html\n[sparklyr]: https://spark.posit.co\n[spark-conf]: https://spark.apache.org/docs/latest/configuration.html\n" }, { "path": "docs/using/troubleshooting.md", "content": "# Troubleshooting Common Problems\n\nWhen troubleshooting, you may see unexpected behaviors or receive an error message.\nThis section provides advice on how to identify and fix some of the most commonly encountered issues.\n\nMost of the `docker run` flags used in this document are explained in detail in the\n[Common Features, Docker Options section](common.md#docker-options) of the documentation.\n\n## Permission denied when mounting volumes\n\nIf you are running a Docker container while mounting a local volume or host directory using the `-v` flag like so:\n\n```bash\ndocker run -it --rm \\\n -p 8888:8888 \\\n -v : \\\n quay.io/jupyter/minimal-notebook:latest\n```\n\nyou might face permissions issues when trying to access the mounted volume:\n\n```bash\n# assuming we mounted the volume in /home/jovyan/stagingarea\n# root is the owner of the mounted volume\nls -ld ~/stagingarea/\n# drwxr-xr-x 2 root root 4096 Feb 1 12:55 stagingarea/\n\ntouch stagingarea/kale.txt\n# touch: cannot touch 'stagingarea/kale.txt': Permission denied\n```\n\nIn this case, the user of the container (`jovyan`) and the owner of the mounted volume (`root`)\nhave different permission levels and ownership over the container's directories and mounts.\nThe following sections cover a few of these scenarios and how to fix them.\n\n**Some things to try:**\n\n1. **Change ownership of the volume mount**\n\n You can change the ownership of the volume mount using the `chown` command.\n In the case of the docker-stacks images, you can set the `CHOWN_EXTRA` and `CHOWN_EXTRA_OPTS` environment variables.\n\n For example, to change the ownership of the volume mount to the `jovyan` user (non-privileged default user in the Docker images):\n\n ```bash\n # running in detached mode - can also be run in interactive mode\n docker run --detach \\\n -v : \\\n -p 8888:8888 \\\n --user root \\\n -e CHOWN_EXTRA=\"\" \\\n -e CHOWN_EXTRA_OPTS=\"-R\" \\\n quay.io/jupyter/minimal-notebook\n ```\n\n where:\n - `CHOWN_EXTRA=,`: will change the ownership and group of the specified container directory (non-recursive by default).\n You need to provide full paths starting with `/`.\n - `CHOWN_EXTRA_OPTS=\"-R\"`: will recursively change the ownership and group of the directory specified in `CHOWN_EXTRA`.\n - `--user root`: you **must** run the container with the root user to change ownership at runtime.\n\n Now accessing the mount should work as expected:\n\n ```bash\n # assuming we mounted the volume in /home/jovyan/stagingarea\n ls -ld ~/stagingarea\n # drwxr-xr-x 2 jovyan users 4096 Feb 1 12:55 stagingarea/\n\n touch stagingarea/kale.txt\n # jovyan is now the owner of /home/jovyan/stagingarea\n # ls -la ~/stagingarea/\n # -rw-r--r-- 1 jovyan users 0 Feb 1 14:41 kale.txt\n ```\n\n ```{admonition} Additional notes\n - If you are mounting your volume inside the `/home/` directory, you can use the `-e CHOWN_HOME=yes` and `CHOWN_HOME_OPTS=\"-R\"` flags\n instead of the `-e CHOWN_EXTRA` and `-e CHOWN_EXTRA_OPTS` in the example above.\n - This solution should work in most cases where you have created a docker volume\n (i.e. using the [`docker volume create --name ` command](https://docs.docker.com/engine/storage/volumes/#create-and-manage-volumes)) and mounted it using the `-v` flag in `docker run`.\n ```\n\n2. **Matching the container's UID/GID with the host's**\n\n Docker handles mounting host directories differently from mounting volumes, even though the syntax is essentially the same (i.e. `-v`).\n\n When you initialize a Docker container using the `-v`flag, the host directories are bind-mounted directly into the container.\n Therefore, the permissions and ownership are copied over and will be **the same** as the ones in your local host\n (including user ids) which may result in permissions errors when trying to access directories or create/modify files inside.\n\n Suppose your local user has a `UID` and `GID` of `1234` and `5678`, respectively.\n To fix the UID discrepancies between your local directories and the container's directories,\n you can run the container with an explicit `NB_UID` and `NB_GID` to match that of the local user:\n\n ```bash\n docker run -it --rm \\\n --user root \\\n -p 8888:8888 \\\n -e NB_UID=1234 \\\n -e NB_GID=5678 \\\n -v \"${PWD}\"/test:/home/jovyan/work \\\n quay.io/jupyter/minimal-notebook:latest\n\n # you should see an output similar to this\n # Update jovyan's UID:GID to 1234:5678\n # Running as jovyan: bash\n ```\n\n where:\n - `NB_UID` and `NB_GID` should match the local user's UID and GID.\n - You **must** use `--user root` to ensure that the `UID` and `GID` are updated at runtime.\n\n````{admonition} Additional notes\n- The caveat with this approach is that since these changes are applied at runtime,\n you will need to re-run the same command with the appropriate flags and environment variables\n if you need to recreate the container (i.e. after removing/destroying it).\n - If you pass a numeric UID, it **must** be in the range of 0-2147483647\n - This approach only updates the UID and GID of the **existing `jovyan` user** instead of creating a new user.\n From the above example:\n ```bash\n id\n # uid=1234(jovyan) gid=5678(jovyan) groups=5678(jovyan),100(users)\n ```\n````\n\n## Permission issues after changing the UID/GID and USER in the container\n\nIf you have also **created a new user**, you might be experiencing any of the following issues:\n\n- the `root` user is the owner of `/home` or a mounted volume\n- when starting the container, you get an error such as `Failed to change ownership of the home directory.`\n- getting permission denied when trying to `conda install` packages\n\n**Some things to try:**\n\n1. **Ensure the new user has ownership of `/home` and volume mounts**\n\n For example, say you want to create a user `callisto` with a `GID` and `UID` of `1234`.\n You will have to add the following flags to the docker run command:\n\n ```bash\n docker run -it --rm \\\n -p 8888:8888 \\\n --user root \\\n -e NB_USER=callisto \\\n -e NB_UID=1234 \\\n -e NB_GID=1234 \\\n -e CHOWN_HOME=yes \\\n -e CHOWN_HOME_OPTS=\"-R\" \\\n -w \"/home/callisto\" \\\n -v \"${PWD}\"/test:/home/callisto/work \\\n quay.io/jupyter/minimal-notebook\n\n # Updated the jovyan user:\n # - username: jovyan -> callisto\n # - home dir: /home/jovyan -> /home/callisto\n # Update callisto UID:GID to 1234:1234\n # Attempting to copy /home/jovyan to /home/callisto...\n # Success!\n # Ensuring /home/callisto is owned by 1234:1234\n # Running as callisto: bash\n ```\n\n where:\n - `-e NB_USER=callisto`: will create a new user `callisto` and automatically add it to the `users` group (does not delete jovyan)\n - `-e NB_UID=1234` and `-e NB_GID=1234`: will set the `UID` and `GID` of the new user (`callisto`) to `1234`\n - `-e CHOWN_HOME_OPTS=\"-R\"` and `-e CHOWN_HOME=yes`: ensure that the new user is the owner of the `/home` directory and subdirectories\n (setting `CHOWN_HOME_OPTS=\"-R` will ensure this change is applied recursively)\n - `-w \"/home/callisto\"` sets the working directory to be the new user's home\n\n ```{admonition} Additional notes\n In the example above, the `-v` flag is used to mount the local volume onto the new user's `/home` directory.\n\n However, if you are mounting a volume elsewhere,\n you also need to use the `-e CHOWN_EXTRA=` flag to avoid any permission issues\n (see the section [Permission denied when mounting volumes](#permission-denied-when-mounting-volumes) on this page).\n ```\n\n2. **Dynamically assign the user ID and GID**\n\n The above case ensures that the `/home` directory is owned by a newly created user with a specific `UID` and `GID`,\n but if you want to assign the `UID` and `GID` of the new user dynamically,\n you can make the following adjustments:\n\n ```bash\n docker run -it --rm \\\n -p 8888:8888 \\\n --user root \\\n -e NB_USER=callisto \\\n -e NB_UID=\"$(id -u)\" \\\n -e NB_GID=\"$(id -g)\" \\\n -e CHOWN_HOME=yes \\\n -e CHOWN_HOME_OPTS=\"-R\" \\\n -w \"/home/callisto\" \\\n -v \"${PWD}\"/test:/home/callisto/work \\\n quay.io/jupyter/minimal-notebook\n ```\n\n where:\n - `\"$(id -u)\"` and `\"$(id -g)\"` will dynamically assign the `UID` and `GID` of the user executing the `docker run` command to the new user (`callisto`)\n\n## Additional tips and troubleshooting commands for permission-related errors\n\n- Pass absolute paths to the `-v` flag:\n\n ```bash\n -v \"${PWD}\"/:/home/jovyan/work\n ```\n\n This example uses the syntax `${PWD}`, which is replaced with the full path to the current directory at runtime.\n The destination path should also be an absolute path starting with a `/` such as `/home/jovyan/work`.\n\n- You might want to consider using the Docker native `--user ` and `--group-add users` flags instead of `-e NB_UID` and `-e NB_GID`:\n\n ```bash\n # note this will use the same UID from\n # the user calling the command, thus matching the local host\n\n docker run -it --rm \\\n -p 8888:8888 \\\n --user \"$(id -u)\" --group-add users \\\n -v :/home/jovyan/work quay.io/jupyter/datascience-notebook\n ```\n\n This command will launch the container with a specific user UID and add that user to the `users` group\n to modify the files in the default `/home` and `/opt/conda` directories.\n Further avoiding issues when trying to `conda install` additional packages.\n\n- Use `docker inspect ` and look for the [`Mounts` section](https://docs.docker.com/engine/storage/volumes/#start-a-container-with-a-volume)\n to verify that the volume was created and mounted accordingly:\n\n ```json\n {\n \"Mounts\": [\n {\n \"Type\": \"volume\",\n \"Name\": \"my-vol\",\n \"Source\": \"/var/lib/docker/volumes/stagingarea/_data\",\n \"Destination\": \"/home/jovyan/stagingarea\",\n \"Driver\": \"local\",\n \"Mode\": \"z\",\n \"RW\": true,\n \"Propagation\": \"\"\n }\n ]\n }\n ```\n\n## Problems installing conda packages from specific channels\n\nBy default, the docker-stacks images have the conda channels priority set to `strict`.\nThis may cause problems when trying to install packages from a channel with lower priority.\n\n```bash\nconda config --show | grep priority\n# channel_priority: strict\n\n# to see its meaning\nconda config --describe channel_priority\n\n# checking the current channels\nconda config --show default_channels\n# default_channels:\n# - https://repo.anaconda.com/pkgs/main\n# - https://repo.anaconda.com/pkgs/r\n```\n\n**Installing packages from alternative channels:**\n\nYou can install packages from other conda channels (e.g. `bioconda`) by disabling the `channel_priority` setting:\n\n```bash\n# install by disabling channel priority at the command level\nconda install --no-channel-priority -c bioconda bioconductor-geoquery\n```\n\nAdditional details are provided in the [Using Alternative Channels](../using/common.md#using-alternative-channels) section of the [Common Features](common.md) page.\n\n## Tokens are being rejected\n\nIf you are a regular user of VSCode and the Jupyter extension,\nyou might experience either of these issues when using any of the docker-stacks images:\n\n- when clicking on the URL displayed on your command line logs, you face a \"This site cannot be reached\" page on your web browser\n- using the produced token and/or URL results in an \"Invalid credentials\" error on the Jupyter \"Token authentication is enabled\" page\n\n ```bash\n # example log output from the docker run command\n\n # [...]\n # Or copy and paste one of these URLs:\n # http://3d4cf3809e3f:8888/?token=996426e890f8dc22fa6835a44442b6026cba02ee61fee6a2\n # or http://127.0.0.1:8888/?token=996426e890f8dc22fa6835a44442b6026cba02ee61fee6a2\n ```\n\n**Some things to try:**\n\n1. **Find lingering Jupyter processes in the background**\n\n The first thing you want to try is to check that no other Jupyter processes are running in the background:\n\n ```bash\n ps aux | grep jupyter\n ```\n\n If there are existing processes running, you can kill them with:\n\n ```bash\n # example output from the above command\n # my-user 3412 ... --daemon-module=vscode_datascience_helpers.jupyter_daemon\n\n # using the pid from the above log\n kill 3412\n ```\n\n2. **Turn off Jupyter auto-start in VSCode**\n\n Alternatively - you might want to ensure that the `Jupyter: Disable Jupyter Auto Start` setting is turned on to avoid this issue in the future.\n\n You can achieve this from the `Settings > Jupyter` menu in VScode:\n\n ![VSCode Settings UI - Jupyter: Disable Jupyter Auto Start checkbox checked](../_static/using/troubleshooting/vscode-jupyter-settings.png)\n\n3. **Route container to unused local port**\n\n Instead of mapping Docker port `8888` to local port `8888`, map to another unused local port.\n You can see an example of mapping to local port `8001`:\n\n ```bash\n docker run -it --rm -p 8001:8888 quay.io/jupyter/datascience-notebook\n ```\n\n When the terminal provides the link to access Jupyter: ,\n change the default port value of `8888` in the URL to the port value mapped with the `docker run` command.\n\n In this example, we use 8001, so the edited link would be: .\n\n Note: Port mapping for Jupyter has other applications outside of Docker.\n For example, it can be used to allow multiple Jupyter instances when using SSH to control cloud devices.\n" }, { "path": "examples/README.md", "content": "# Examples\n\nThese examples are not tested and might not work.\nPlease, send PRs if you start using these examples and see some issues.\n" }, { "path": "examples/docker-compose/README.md", "content": "# Docker Compose example\n\nThis example demonstrate how to deploy docker-stack notebook containers to any Docker Machine-controlled host using Docker Compose.\n\n## Prerequisites\n\n- [Docker Engine](https://docs.docker.com/engine/) 1.10.0+\n- [Docker Machine](https://docs.docker.com/machine/) 0.6.0+\n- [Docker Compose](https://docs.docker.com/compose/) 1.6.0+\n\nSee the [installation instructions](https://docs.docker.com/engine/installation/) for your environment.\n\n## Quickstart\n\nBuild and run a `jupyter/minimal-notebook` image on a VirtualBox VM on local desktop.\n\n```bash\n# create a Docker Machine-controlled VirtualBox VM\nbin/vbox.sh mymachine\n\n# activate the docker machine\neval \"$(docker-machine env mymachine)\"\n\n# build the notebook image on the machine\nnotebook/build.sh\n\n# bring up the notebook container\nnotebook/up.sh\n```\n\nTo stop and remove the container:\n\n```bash\nnotebook/down.sh\n```\n\n## FAQ\n\n### How do I specify which docker-stack notebook image to deploy?\n\nYou can customize the docker-stack notebook image to deploy by modifying the `notebook/Dockerfile`.\nFor example, you can build and deploy a `jupyter/all-spark-notebook` by modifying the Dockerfile like so:\n\n```dockerfile\nFROM quay.io/jupyter/all-spark-notebook\n# Your RUN commands and so on\n```\n\nOnce you modify the Dockerfile, don't forget to rebuild the image.\n\n```bash\n# activate the docker machine\neval \"$(docker-machine env mymachine)\"\n\nnotebook/build.sh\n```\n\n### Can I run multiple notebook containers on the same VM?\n\nYes. Set environment variables to specify unique names and ports when running the `up.sh` command.\n\n```bash\nNAME=my-notebook PORT=9000 notebook/up.sh\nNAME=your-notebook PORT=9001 notebook/up.sh\n```\n\nTo stop and remove the containers:\n\n```bash\nNAME=my-notebook notebook/down.sh\nNAME=your-notebook notebook/down.sh\n```\n\n### Where are my notebooks stored?\n\nThe `up.sh` creates a Docker volume named after the notebook container with a `-work` suffix, e.g., `my-notebook-work`.\n\n### Can multiple notebook containers share the same notebook volume?\n\nYes. Set the `WORK_VOLUME` environment variable to the same value for each notebook.\n\n```bash\nNAME=my-notebook PORT=9000 WORK_VOLUME=our-work notebook/up.sh\nNAME=your-notebook PORT=9001 WORK_VOLUME=our-work notebook/up.sh\n```\n\n### How do I run over HTTPS?\n\nTo run the Jupyter Server with a self-signed certificate, pass the `--secure` option to the `up.sh` script.\nYou must also provide a password, which will be used to secure the Jupyter Server.\nYou can specify the password by setting the `PASSWORD` environment variable, or by passing it to the `up.sh` script.\n\n```bash\nPASSWORD=a_secret notebook/up.sh --secure\n\n# or\nnotebook/up.sh --secure --password a_secret\n```\n\n### Can I use Let's Encrypt certificate chains?\n\nSure. If you want to secure access to publicly addressable notebook containers, you can generate a free certificate using the [Let's Encrypt](https://letsencrypt.org) service.\n\nThis example includes the `bin/letsencrypt.sh` script, which runs the `letsencrypt` client to create a full-chain certificate and private key, and stores them in a Docker volume.\n\n```{note}\nThe script hard codes several `letsencrypt` options, one of which automatically agrees to the Let's Encrypt Terms of Service.\n```\n\nThe following command will create a certificate chain and store it in a Docker volume named `mydomain-secrets`.\n\n```bash\nFQDN=host.mydomain.com EMAIL=myemail@somewhere.com \\\n SECRETS_VOLUME=mydomain-secrets \\\n bin/letsencrypt.sh\n```\n\nNow run `up.sh` with the `--letsencrypt` option.\nYou must also provide the name of the secrets volume and a password.\n\n```bash\nPASSWORD=a_secret SECRETS_VOLUME=mydomain-secrets notebook/up.sh --letsencrypt\n\n# or\nnotebook/up.sh --letsencrypt --password a_secret --secrets mydomain-secrets\n```\n\nBe aware that Let's Encrypt has a pretty [low rate limit per domain](https://community.letsencrypt.org/t/public-beta-rate-limits/4772/3) at the moment.\nYou can avoid exhausting your limit by testing against the Let's Encrypt staging servers.\nTo hit their staging servers, set the environment variable `CERT_SERVER=--staging`.\n\n```bash\nFQDN=host.mydomain.com EMAIL=myemail@somewhere.com \\\n CERT_SERVER=--staging \\\n bin/letsencrypt.sh\n```\n\nAlso, be aware that Let's Encrypt certificates are short-lived (90 days).\nIf you need them for a longer period of time, you'll need to manually set up a cron job to run the renewal steps.\n(You can reuse the command above.)\n\n### Can I deploy to any Docker Machine host?\n\nYes, you should be able to deploy to any Docker Machine-controlled host.\nTo make it easier to get up and running, this example includes scripts to provision Docker Machines to VirtualBox and IBM SoftLayer, but more scripts are welcome!\n\nTo create a Docker machine using a VirtualBox VM on local desktop:\n\n```bash\nbin/vbox.sh mymachine\n```\n\nTo create a Docker machine using a virtual device on IBM SoftLayer:\n\n```bash\nexport SOFTLAYER_USER=my_softlayer_username\nexport SOFTLAYER_API_KEY=my_softlayer_api_key\nexport SOFTLAYER_DOMAIN=my.domain\n\n# Create virtual device\nbin/softlayer.sh myhost\n\n# Add DNS entry (SoftLayer DNS zone must exist for SOFTLAYER_DOMAIN)\nbin/sl-dns.sh myhost\n```\n\n## Troubleshooting\n\n### Unable to connect to VirtualBox VM on Mac OS X when using Cisco VPN client\n\nThe Cisco VPN client blocks access to IP addresses that it does not know about, and may block access to a new VM if it is created while the Cisco VPN client is running.\n\n1. Stop Cisco VPN client. (It does not allow modifications to route table).\n2. Run `ifconfig` to list `vboxnet` virtual network devices.\n3. Run `sudo route -nv add -net 192.168.99 -interface vboxnetX`, where X is the number of the virtual device assigned to the VirtualBox VM.\n4. Start Cisco VPN client.\n" }, { "path": "examples/docker-compose/bin/letsencrypt.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# Use https://letsencrypt.org to create a certificate for a single domain\n# and store it in a Docker volume.\n\nset -e\n\n# Get domain and email from environment\n[ -z \"${FQDN}\" ] && \\\n echo \"ERROR: Must set FQDN environment variable\" && \\\n exit 1\n\n[ -z \"${EMAIL}\" ] && \\\n echo \"ERROR: Must set EMAIL environment variable\" && \\\n exit 1\n\n# letsencrypt certificate server type (default is production).\n# Set `CERT_SERVER=--staging` for staging.\n: \"${CERT_SERVER=''}\"\n\n# Create Docker volume to contain the cert\n: \"${SECRETS_VOLUME:=my-notebook-secrets}\"\ndocker volume create --name \"${SECRETS_VOLUME}\" 1>/dev/null\n# Generate the cert and save it to the Docker volume\ndocker run -it --rm \\\n -p 80:80 \\\n -v \"${SECRETS_VOLUME}\":/etc/letsencrypt \\\n quay.io/letsencrypt/letsencrypt:latest \\\n certonly \\\n --non-interactive \\\n --keep-until-expiring \\\n --standalone \\\n --standalone-supported-challenges http-01 \\\n --agree-tos \\\n --domain \"${FQDN}\" \\\n --email \"${EMAIL}\" \\\n \"${CERT_SERVER}\"\n\n# Set permissions so nobody can read the cert and key.\n# Also symlink the certs into the root of the /etc/letsencrypt\n# directory so that the FQDN doesn't have to be known later.\ndocker run -it --rm \\\n -v \"${SECRETS_VOLUME}\":/etc/letsencrypt \\\n ubuntu \\\n bash -c \"ln -s /etc/letsencrypt/live/${FQDN}/* /etc/letsencrypt/ && \\\n find /etc/letsencrypt -type d -exec chmod 755 {} +\"\n" }, { "path": "examples/docker-compose/bin/sl-dns.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nset -e\n\n# User must have slcli installed\nwhich slcli > /dev/null || (echo \"SoftLayer cli not found (pip install softlayer)\"; exit 1)\n\nUSAGE=\"Usage: $(basename \"${0}\") machine_name [domain]\"\nE_BADARGS=85\n\n# Machine name is first command line arg\nMACHINE_NAME=\"${1}\" && [ -z \"${MACHINE_NAME}\" ] && echo \"${USAGE}\" && exit ${E_BADARGS}\n\n# Use SOFTLAYER_DOMAIN env var if domain name not set as second arg\nDOMAIN=\"${2:-$SOFTLAYER_DOMAIN}\" && [ -z \"${DOMAIN}\" ] && \\\n echo \"Must specify domain or set SOFTLAYER_DOMAIN environment variable\" && \\\n echo \"${USAGE}\" && exit ${E_BADARGS}\n\nIP=$(docker-machine ip \"${MACHINE_NAME}\")\n\nslcli dns record-add \"${DOMAIN}\" \"${MACHINE_NAME}\" A \"${IP}\"\n" }, { "path": "examples/docker-compose/bin/softlayer.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# Set default SoftLayer VM settings\n: \"${SOFTLAYER_CPU:=4}\"\nexport SOFTLAYER_CPU\n: \"${SOFTLAYER_DISK_SIZE:=100}\"\nexport SOFTLAYER_DISK_SIZE\n: \"${SOFTLAYER_MEMORY:=4096}\"\nexport SOFTLAYER_MEMORY\n: \"${SOFTLAYER_REGION:=wdc01}\"\nexport SOFTLAYER_REGION\n\ndocker-machine create --driver softlayer \"$@\"\n" }, { "path": "examples/docker-compose/bin/vbox.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# Set reasonable default VM settings\n: \"${VIRTUALBOX_CPUS:=4}\"\nexport VIRTUALBOX_CPUS\n: \"${VIRTUALBOX_MEMORY_SIZE:=4096}\"\nexport VIRTUALBOX_MEMORY_SIZE\n\ndocker-machine create --driver virtualbox \"$@\"\n" }, { "path": "examples/docker-compose/notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# Pick your favorite docker-stacks image\nFROM quay.io/jupyter/minimal-notebook\n\nUSER root\n\n# Add permanent apt-get installs and other root commands here\n# e.g., RUN apt-get install --yes --no-install-recommends npm nodejs\n\nUSER ${NB_UID}\n\n# Switch back to jovyan to avoid accidental container runs as root\n# Add permanent mamba/pip/conda installs, data files, other user libs here\n# e.g., RUN pip install --no-cache-dir flake8\n" }, { "path": "examples/docker-compose/notebook/build.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nDIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n\n# Setup environment\n# shellcheck source=examples/docker-compose/notebook/env.sh\nsource \"${DIR}/env.sh\"\n\n# Build the notebook image\ndocker-compose -f \"${DIR}/notebook.yml\" build\n" }, { "path": "examples/docker-compose/notebook/down.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nDIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n\n# Setup environment\n# shellcheck source=examples/docker-compose/notebook/env.sh\nsource \"${DIR}/env.sh\"\n\n# Bring down the notebook container, using container name as project name\ndocker-compose -f \"${DIR}/notebook.yml\" -p \"${NAME}\" down\n" }, { "path": "examples/docker-compose/notebook/env.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# Set default values for environment variables required by notebook compose\n# configuration file.\n\n# Container name\n: \"${NAME:=my-notebook}\"\nexport NAME\n\n# Exposed container port\n: \"${PORT:=80}\"\nexport PORT\n\n# Container work volume name\n: \"${WORK_VOLUME:=${NAME}-work}\"\nexport WORK_VOLUME\n\n# Container secrets volume name\n: \"${SECRETS_VOLUME:=${NAME}-secrets}\"\nexport SECRETS_VOLUME\n" }, { "path": "examples/docker-compose/notebook/letsencrypt-notebook.yml", "content": "---\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nversion: \"2\"\n\nservices:\n notebook:\n build: .\n image: my-notebook\n container_name: ${NAME}\n volumes:\n - \"work:/home/jovyan/work\"\n - \"secrets:/etc/letsencrypt\"\n ports:\n - \"${PORT}:8888\"\n environment:\n USE_HTTPS: \"yes\"\n PASSWORD: ${PASSWORD}\n command: >\n start-notebook.py\n --ServerApp.certfile=/etc/letsencrypt/fullchain.pem\n --ServerApp.keyfile=/etc/letsencrypt/privkey.pem\n\nvolumes:\n work:\n external:\n name: ${WORK_VOLUME}\n secrets:\n external:\n name: ${SECRETS_VOLUME}\n" }, { "path": "examples/docker-compose/notebook/notebook.yml", "content": "---\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nversion: \"2\"\n\nservices:\n notebook:\n build: .\n image: my-notebook\n container_name: ${NAME}\n volumes:\n - \"work:/home/jovyan/work\"\n ports:\n - \"${PORT}:8888\"\n\nvolumes:\n work:\n external:\n name: ${WORK_VOLUME}\n" }, { "path": "examples/docker-compose/notebook/secure-notebook.yml", "content": "---\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nversion: \"2\"\n\nservices:\n notebook:\n build: .\n image: my-notebook\n container_name: ${NAME}\n volumes:\n - \"work:/home/jovyan/work\"\n ports:\n - \"${PORT}:8888\"\n environment:\n USE_HTTPS: \"yes\"\n PASSWORD: ${PASSWORD}\n\nvolumes:\n work:\n external:\n name: ${WORK_VOLUME}\n" }, { "path": "examples/docker-compose/notebook/up.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nset -e\n\nDIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n\nUSAGE=\"Usage: $(basename \"${0}\") [--secure | --letsencrypt] [--password PASSWORD] [--secrets SECRETS_VOLUME]\"\n\n# Parse args to determine security settings\nSECURE=${SECURE:=no}\nLETSENCRYPT=${LETSENCRYPT:=no}\nwhile [[ $# -gt 0 ]]; do\nkey=\"${1}\"\ncase \"${key}\" in\n --secure)\n SECURE=yes\n ;;\n --letsencrypt)\n LETSENCRYPT=yes\n ;;\n --secrets)\n SECRETS_VOLUME=\"${2}\"\n shift # past argument\n ;;\n --password)\n PASSWORD=\"${2}\"\n export PASSWORD\n shift # past argument\n ;;\n *) # unknown option\n ;;\nesac\nshift # past argument or value\ndone\n\nif [[ \"${LETSENCRYPT}\" == yes || \"${SECURE}\" == yes ]]; then\n if [ -z \"${PASSWORD:+x}\" ]; then\n echo \"ERROR: Must set PASSWORD if running in secure mode\"\n echo \"${USAGE}\"\n exit 1\n fi\n if [ \"${LETSENCRYPT}\" == yes ]; then\n CONFIG=letsencrypt-notebook.yml\n if [ -z \"${SECRETS_VOLUME:+x}\" ]; then\n echo \"ERROR: Must set SECRETS_VOLUME if running in letsencrypt mode\"\n echo \"${USAGE}\"\n exit 1\n fi\n else\n CONFIG=secure-notebook.yml\n fi\n export PORT=${PORT:=443}\nelse\n CONFIG=notebook.yml\n export PORT=${PORT:=80}\nfi\n\n# Setup environment\n# shellcheck source=examples/docker-compose/notebook/env.sh\nsource \"${DIR}/env.sh\"\n\n# Create a Docker volume to store notebooks\ndocker volume create --name \"${WORK_VOLUME}\"\n\n# Bring up a notebook container, using container name as project name\necho \"Bringing up notebook '${NAME}'\"\ndocker-compose -f \"${DIR}/${CONFIG}\" -p \"${NAME}\" up -d\n\nIP=$(docker-machine ip \"$(docker-machine active)\")\necho \"Notebook ${NAME} listening on ${IP}:${PORT}\"\n" }, { "path": "examples/make-deploy/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# Pick your favorite docker-stacks image\nFROM quay.io/jupyter/minimal-notebook\n\nUSER root\n\n# Add permanent apt-get installs and other root commands here\n# e.g., RUN apt-get install --yes --no-install-recommends npm nodejs\n\nUSER ${NB_UID}\n\n# Switch back to jovyan to avoid accidental container runs as root\n# Add permanent mamba/pip/conda installs, data files, other user libs here\n# e.g., RUN pip install --no-cache-dir flake8\n" }, { "path": "examples/make-deploy/Makefile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n.PHONY: help check image notebook\n\nIMAGE:=my-notebook\n\n# Common, extensible docker run command\ndefine RUN_NOTEBOOK\n@docker volume create --name $(WORK_VOLUME) > /dev/null\n-@docker rm --force $(NAME) 2> /dev/null\n@docker run --detach -p $(PORT):8888 \\\n\t--name $(NAME) \\\n\t-v $(WORK_VOLUME):/home/jovyan/work \\\n\t$(DOCKER_ARGS) \\\n\t$(IMAGE) bash -c \"$(PRE_CMD) chown jovyan /home/jovyan/work && start-notebook.py $(ARGS)\" > /dev/null\n@echo \"DONE: Notebook '$(NAME)' listening on $$(docker-machine ip $$(docker-machine active)):$(PORT)\"\nendef\n\nhelp:\n\t@cat README.md\n\ncheck:\n\t@which docker-machine > /dev/null || (echo \"ERROR: docker-machine not found (brew install docker-machine)\"; exit 1)\n\t@which docker > /dev/null || (echo \"ERROR: docker not found (brew install docker)\"; exit 1)\n\t@docker | grep volume > /dev/null || (echo \"ERROR: docker 1.9.0+ required\"; exit 1)\n\nimage: DOCKER_ARGS?=\nimage:\n\t@docker build --rm $(DOCKER_ARGS) --tag $(IMAGE) .\n\nnotebook: PORT?=80\nnotebook: NAME?=notebook\nnotebook: WORK_VOLUME?=$(NAME)-data\nnotebook: check\n\t$(RUN_NOTEBOOK)\n\n# docker-machine drivers\ninclude virtualbox.makefile\ninclude softlayer.makefile\n\n# Preset notebook configurations\ninclude self-signed.makefile\ninclude letsencrypt.makefile\n" }, { "path": "examples/make-deploy/README.md", "content": "# Make deploy example\n\nThis folder contains a Makefile and a set of supporting files demonstrating how to run a docker-stack notebook container on a docker-machine controlled host.\n\n## Prerequisites\n\n- make 3.81+\n - Ubuntu users: Be aware of [make 3.81 defect 483086](https://bugs.launchpad.net/ubuntu/+source/make-dfsg/+bug/483086) which exists in 14.04 LTS but is fixed in 15.04+\n- docker-machine 0.5.0+\n- docker 1.9.0+\n\n## Quickstart\n\nTo show what's possible, here's how to run the `jupyter/minimal-notebook` on a brand-new local virtualbox.\n\n```bash\n# create a new VM\nmake virtualbox-vm NAME=dev\n# make the new VM the active docker machine\neval $(docker-machine env dev)\n# pull a docker stack and build a local image from it\nmake image\n# start a Server in a container\nmake notebook\n```\n\nThe last command will log the IP address and port to visit in your browser.\n\n## FAQ\n\n### Can I run multiple notebook containers on the same VM?\n\nYes. Specify a unique name and port on the `make notebook` command.\n\n```bash\nmake notebook NAME=my-notebook PORT=9000\nmake notebook NAME=your-notebook PORT=9001\n```\n\n### Can multiple notebook containers share their notebook directory?\n\nYes.\n\n```bash\nmake notebook NAME=my-notebook PORT=9000 WORK_VOLUME=our-work\nmake notebook NAME=your-notebook PORT=9001 WORK_VOLUME=our-work\n```\n\n### How do I run over HTTPS?\n\nInstead of `make notebook`, run `make self-signed-notebook PASSWORD=your_desired_password`.\nThis target gives you a notebook with a self-signed certificate.\n\n### That self-signed certificate is a pain. Let's Encrypt?\n\nYes. Please.\n\n```bash\nmake letsencrypt FQDN=host.mydomain.com EMAIL=myemail@somewhere.com\nmake letsencrypt-notebook\n```\n\nThe first command creates a Docker volume named after the notebook container with a `-secrets` suffix.\nIt then runs the `letsencrypt` client with a slew of options (one of which has you automatically agreeing to the Let's Encrypt Terms of Service, see the Makefile).\nThe second command mounts the secrets volume and configures Jupyter to use the full-chain certificate and private key.\n\nBe aware: Let's Encrypt has a pretty [low rate limit per domain](https://community.letsencrypt.org/t/public-beta-rate-limits/4772/3) at the moment.\nYou can avoid exhausting your limit by testing against the Let's Encrypt staging servers.\nTo hit their staging servers, set the environment variable `CERT_SERVER=--staging`.\n\n```bash\nmake letsencrypt FQDN=host.mydomain.com EMAIL=myemail@somewhere.com CERT_SERVER=--staging\n```\n\nAlso, keep in mind Let's Encrypt certificates are short-lived: 90 days at the moment.\nYou'll need to manually set up a cron job to run the renewal steps at the moment.\n(You can reuse the first command above.)\n\n### My pip/conda/apt-get installs disappear every time I restart the container. Can I make them permanent?\n\n```bash\n# add your pip, conda, apt-get, etc. permanent features to the Dockerfile where\n# indicated by the comments in the Dockerfile\nvi Dockerfile\nmake image\nmake notebook\n```\n\n### How do I upgrade my Docker container?\n\n```bash\nmake image DOCKER_ARGS=--pull\nmake notebook\n```\n\nThe first line pulls the latest version of the Docker image used in the local Dockerfile.\nThen it rebuilds the local Docker image containing any customizations you may have added to it.\nThe second line kills your currently running notebook container, and starts a fresh one using the new image.\n\n### Can I run on another VM provider other than VirtualBox?\n\nYes. As an example, there's a `softlayer.makefile` included in this repo as an example.\nYou would use it like so:\n\n```bash\nmake softlayer-vm NAME=myhost \\\n SOFTLAYER_DOMAIN=your_desired_domain \\\n SOFTLAYER_USER=your_user_id \\\n SOFTLAYER_API_KEY=your_api_key\neval $(docker-machine env myhost)\n# optional, creates a real DNS entry for the VM using the machine name as the hostname\nmake softlayer-dns SOFTLAYER_DOMAIN=your_desired_domain\nmake image\nmake notebook\n```\n\nIf you'd like to add support for another docker-machine driver, use the `softlayer.makefile` as a template.\n\n### Where are my notebooks stored?\n\n`make notebook` creates a Docker volume named after the notebook container with a `-data` suffix.\n\n### Uh ... make?\n\nYes, sorry Windows users. It got the job done for a simple example.\nWe can certainly accept other deployment mechanism examples in the parent folder or in other repos.\n\n### Are there any other options?\n\nYes indeed. `cat` the Makefiles and look at the target parameters.\n" }, { "path": "examples/make-deploy/letsencrypt.makefile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# BE CAREFUL when using Docker engine <1.10 because running a container with\n# `--rm` option while mounting a docker volume may wipe out the volume.\n# See issue: https://github.com/moby/moby/issues/17907\n\n# Use letsencrypt production server by default to get a real cert.\n# Use CERT_SERVER=--staging to hit the staging server (not a real cert).\n\nletsencrypt: NAME?=notebook\nletsencrypt: SECRETS_VOLUME?=$(NAME)-secrets\nletsencrypt: TMP_CONTAINER?=$(NAME)-tmp\nletsencrypt: CERT_SERVER?=\nletsencrypt:\n\t@test -n \"$(FQDN)\" || \\\n\t\t(echo \"ERROR: FQDN not defined or blank\"; exit 1)\n\t@test -n \"$(EMAIL)\" || \\\n\t\t(echo \"ERROR: EMAIL not defined or blank\"; exit 1)\n\t@docker volume create --name $(SECRETS_VOLUME) > /dev/null\n\t@docker run -it -p 80:80 \\\n\t\t--name=$(TMP_CONTAINER) \\\n\t\t-v $(SECRETS_VOLUME):/etc/letsencrypt \\\n\t\tquay.io/letsencrypt/letsencrypt:latest \\\n\t\tcertonly \\\n\t\t$(CERT_SERVER) \\\n\t\t--keep-until-expiring \\\n\t\t--standalone \\\n\t\t--standalone-supported-challenges http-01 \\\n\t\t--agree-tos \\\n\t\t--domain '$(FQDN)' \\\n\t\t--email '$(EMAIL)'; \\\n\t\tdocker rm --force $(TMP_CONTAINER) > /dev/null\n# The letsencrypt image has an entrypoint, so we use the notebook image\n# instead so we can run arbitrary commands.\n# Here we set the permissions so nobody can read the cert and key.\n# We also symlink the certs into the root of the /etc/letsencrypt\n# directory so that the FQDN doesn't have to be known later.\n\t@docker run -it \\\n\t\t--name=$(TMP_CONTAINER) \\\n\t\t-v $(SECRETS_VOLUME):/etc/letsencrypt \\\n\t\t$(NOTEBOOK_IMAGE) \\\n\t\tbash -c \"ln -s /etc/letsencrypt/live/$(FQDN)/* /etc/letsencrypt/ && \\\n\t\t\tfind /etc/letsencrypt -type d -exec chmod 755 {} +\"; \\\n\t\t\tdocker rm --force $(TMP_CONTAINER) > /dev/null\n\nletsencrypt-notebook: PORT?=443\nletsencrypt-notebook: NAME?=notebook\nletsencrypt-notebook: WORK_VOLUME?=$(NAME)-data\nletsencrypt-notebook: SECRETS_VOLUME?=$(NAME)-secrets\nletsencrypt-notebook: DOCKER_ARGS:=-e USE_HTTPS=yes \\\n\t-e PASSWORD=$(PASSWORD) \\\n\t-v $(SECRETS_VOLUME):/etc/letsencrypt\nletsencrypt-notebook: ARGS:=\\\n\t--ServerApp.certfile=/etc/letsencrypt/fullchain.pem \\\n\t--ServerApp.keyfile=/etc/letsencrypt/privkey.pem\nletsencrypt-notebook: check\n\t@test -n \"$(PASSWORD)\" || \\\n\t\t(echo \"ERROR: PASSWORD not defined or blank\"; exit 1)\n\t$(RUN_NOTEBOOK)\n" }, { "path": "examples/make-deploy/self-signed.makefile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nself-signed-notebook: PORT?=443\nself-signed-notebook: NAME?=notebook\nself-signed-notebook: WORK_VOLUME?=$(NAME)-data\nself-signed-notebook: DOCKER_ARGS:=-e USE_HTTPS=yes \\\n\t-e PASSWORD=$(PASSWORD)\nself-signed-notebook: check\n\t@test -n \"$(PASSWORD)\" || \\\n\t\t(echo \"ERROR: PASSWORD not defined or blank\"; exit 1)\n\t$(RUN_NOTEBOOK)\n" }, { "path": "examples/make-deploy/softlayer.makefile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nsoftlayer-vm: export SOFTLAYER_CPU?=4\nsoftlayer-vm: export SOFTLAYER_DISK_SIZE?=100\nsoftlayer-vm: export SOFTLAYER_MEMORY?=4096\nsoftlayer-vm: export SOFTLAYER_REGION?=wdc01\nsoftlayer-vm: check\n\t@test -n \"$(NAME)\" || \\\n\t\t(echo \"ERROR: NAME not defined (make help)\"; exit 1)\n\t@test -n \"$(SOFTLAYER_API_KEY)\" || \\\n\t\t(echo \"ERROR: SOFTLAYER_API_KEY not defined (make help)\"; exit 1)\n\t@test -n \"$(SOFTLAYER_USER)\" || \\\n\t\t(echo \"ERROR: SOFTLAYER_USER not defined (make help)\"; exit 1)\n\t@test -n \"$(SOFTLAYER_DOMAIN)\" || \\\n\t\t(echo \"ERROR: SOFTLAYER_DOMAIN not defined (make help)\"; exit 1)\n\t@docker-machine create -d softlayer $(NAME)\n\t@echo \"DONE: Docker host '$(NAME)' up at $$(docker-machine ip $(NAME))\"\n\nsoftlayer-dns: HOST_NAME:=$$(docker-machine active)\nsoftlayer-dns: IP:=$$(docker-machine ip $(HOST_NAME))\nsoftlayer-dns: check\n\t@which slcli > /dev/null || (echo \"softlayer cli not found (pip install softlayer)\"; exit 1)\n\t@test -n \"$(SOFTLAYER_DOMAIN)\" || \\\n\t\t(echo \"ERROR: SOFTLAYER_DOMAIN not defined (make help)\"; exit 1)\n\t@slcli dns record-add $(SOFTLAYER_DOMAIN) $(HOST_NAME) A $(IP)\n" }, { "path": "examples/make-deploy/virtualbox.makefile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nvirtualbox-vm: export VIRTUALBOX_CPU_COUNT?=4\nvirtualbox-vm: export VIRTUALBOX_DISK_SIZE?=100000\nvirtualbox-vm: export VIRTUALBOX_MEMORY_SIZE?=4096\nvirtualbox-vm: check\n\t@test -n \"$(NAME)\" || \\\n\t\t(echo \"ERROR: NAME not defined (make help)\"; exit 1)\n\t@docker-machine create -d virtualbox $(NAME)\n" }, { "path": "examples/openshift/README.md", "content": "# OpenShift example\n\nThis example provides templates for deploying the Jupyter Project docker-stacks images to OpenShift.\n\n## Prerequisites\n\nAny OpenShift 3 environment.\nThe templates were tested with OpenShift 3.7.\nIt is believed they should work with at least OpenShift 3.6 or later.\n\nDo be aware that the Jupyter Project docker-stacks images are very large.\nThe OpenShift environment you are using must provide sufficient quota on the per-user space for images and the file system for running containers.\nIf the quota is too small, the pulling of the images to a node in the OpenShift cluster when deploying them, will fail due to lack of space.\nEven if the image is able to run, if the quota is only just larger than the space required for the image, you will not be able to install many packages into the container before running out of space.\n\nOpenShift Online, the public hosted version of OpenShift from Red Hat has a quota of only 3GB for the image and container file system.\nAs a result, only the `minimal-notebook` can be started and there is little space remaining to install additional packages.\nAlthough OpenShift Online is suitable for demonstrating these templates work, what you can do in that environment will be limited due to the size of the images.\n\nIf you want to experiment with using Jupyter Notebooks in an OpenShift environment, you should instead use [Minishift](https://www.openshift.org/minishift/).\nMinishift provides you the ability to run OpenShift in a virtual machine on your own local computer.\n\n## Loading the Templates\n\nTo load the templates, login to OpenShift from the command line and run:\n\n```bash\noc create -f https://raw.githubusercontent.com/jupyter-on-openshift/docker-stacks/master/examples/openshift/templates.json\n```\n\nThis should create the `jupyter-notebook` template\n\nThe template can be used from the command line using the `oc new-app` command, or from the OpenShift web console by selecting _Add to Project_.\nThis `README` is only going to explain deploying from the command line.\n\n## Deploying a Notebook\n\nTo deploy a notebook from the command line using the template, run:\n\n```bash\noc new-app --template jupyter-notebook\n```\n\nThe output will be similar to:\n\n```lang-none\n--> Deploying template \"jupyter/jupyter-notebook\" to project jupyter\n\n Jupyter Notebook\n ---------\n Template for deploying Jupyter Notebook images.\n\n * With parameters:\n * APPLICATION_NAME=notebook\n * NOTEBOOK_IMAGE=docker.io/jupyter/minimal-notebook:latest\n * NOTEBOOK_PASSWORD=ded4d7cada554aa48e0db612e1ed1080 # generated\n\n--> Creating resources ...\n configmap \"notebook-cfg\" created\n deploymentconfig \"notebook\" created\n route \"notebook\" created\n service \"notebook\" created\n--> Success\n Access your application via route 'notebook-jupyter.b9ad.pro-us-east-1.openshiftapps.com'\n Run 'oc status' to view your app.\n```\n\nWhen no template parameters are provided, the name of the deployed notebook will be `notebook`.\nThe image used will be:\n\n```lang-none\ndocker.io/jupyter/minimal-notebook:latest\n```\n\nA password you can use when accessing the notebook will be auto generated and is displayed in the output from running `oc new-app`.\n\nTo see the hostname for accessing the notebook run:\n\n```bash\noc get routes\n```\n\nThe output will be similar to:\n\n```lang-none\nNAME HOST/PORT PATH SERVICES PORT TERMINATION WILDCARD\nnotebook notebook-jupyter.abcd.pro-us-east-1.openshiftapps.com notebook 8888-tcp edge/Redirect None\n```\n\nA secure route will be used to expose the notebook outside the OpenShift cluster, so in this case the URL would be:\n\n```lang-none\nhttps://notebook-jupyter.abcd.pro-us-east-1.openshiftapps.com/\n```\n\nWhen prompted, enter the password for the notebook.\n\n## Passing Template Parameters\n\nTo override the name for the notebook, the image used, and the password, you can pass template parameters using the `--param` option.\n\n```bash\noc new-app --template jupyter-notebook \\\n --param APPLICATION_NAME=mynotebook \\\n --param NOTEBOOK_IMAGE=docker.io/jupyter/scipy-notebook:latest \\\n --param NOTEBOOK_PASSWORD=mypassword\n```\n\nYou can deploy any of the Jupyter Project docker-stacks images.\n\nIf you don't care what version of the image is used, add the `:latest` tag at the end of the image name, otherwise use the hash corresponding to the image version you want to use.\n\n## Deleting the Notebook Instance\n\nTo delete the notebook instance, run `oc delete` using a label selector for the application name.\n\n```bash\noc delete all,configmap --selector app=mynotebook\n```\n\n## Adding Persistent Storage\n\nYou can upload notebooks and other files using the web interface of the notebook.\nAny uploaded files or changes you make to them will be lost when the notebook instance is restarted.\nIf you want to save your work, you need to add persistent storage to the notebook.\nTo add persistent storage run:\n\n```bash\noc set volume dc/mynotebook --add \\\n --type=pvc --claim-size=1Gi --claim-mode=ReadWriteOnce \\\n --claim-name mynotebook-data --name data \\\n --mount-path /home/jovyan\n```\n\nWhen you have deleted the notebook instance, if using a persistent volume, you will need to delete it in a separate step.\n\n```bash\noc delete pvc/mynotebook-data\n```\n\n## Customizing the Configuration\n\nIf you want to set any custom configuration for the notebook, you can edit the config map created by the template.\n\n```bash\noc edit configmap/mynotebook-cfg\n```\n\nThe `data` field of the config map contains Python code used as the `jupyter_server_config.py` file.\n\nIf you are using a persistent volume, you can also create a configuration file at:\n\n```lang-none\n/home/jovyan/.jupyter/jupyter_server_config.py\n```\n\nThis will be merged at the end of the configuration from the config map.\n\nBecause the configuration is Python code, ensure any indenting is correct.\nAny errors in the configuration file will cause the notebook to fail when starting.\n\nIf the error is in the config map, edit it again to fix it and trigger a new deployment if necessary by running:\n\n```bash\noc rollout latest dc/mynotebook\n```\n\nIf you make an error in the configuration file stored in the persistent volume, you will need to scale down the notebook, so it isn't running.\n\n```bash\noc scale dc/mynotebook --replicas 0\n```\n\nThen run:\n\n```bash\noc debug dc/mynotebook\n```\n\nto run the notebook in debug mode.\nThis will provide you with an interactive terminal session inside a running container, but the notebook will not have been started.\nEdit the configuration file in the volume to fix any errors and exit the terminal session.\n\nStart up the notebook again.\n\n```bash\noc scale dc/mynotebook --replicas 1\n```\n\n## Changing the Notebook Password\n\nThe password for the notebook is supplied as a template parameter, or if not supplied will be automatically generated by the template.\nIt will be passed into the container through an environment variable.\n\nIf you want to change the password, you can do so by editing the environment variable on the deployment configuration.\n\n```bash\noc set env dc/mynotebook JUPYTER_NOTEBOOK_PASSWORD=mypassword\n```\n\nThis will trigger a new deployment so ensure you have downloaded any work if not using a persistent volume.\n\nIf using a persistent volume, you could instead set up a password in the file `/home/jovyan/.jupyter/jupyter_server_config.py` as per guidelines in .\n\n## Deploying from a Custom Image\n\nIf you want to deploy a custom variant of the Jupyter Project docker-stacks images, you can replace the image name with that of your own.\nIf the image is not stored on Docker Hub, but some other public image registry, prefix the name of the image with the image registry host details.\n\nIf the image is in your OpenShift project, because you imported the image into OpenShift, or used the docker build strategy of OpenShift to build a derived custom image,\nyou can use the name of the image stream for the image name, including any image tag if necessary.\n\nThis can be illustrated by first importing an image into the OpenShift project.\n\n```bash\noc import-image docker.io/jupyter/datascience-notebook:latest --confirm\n```\n\nThen deploy it using the name of the image stream created.\n\n```bash\noc new-app --template jupyter-notebook \\\n --param APPLICATION_NAME=mynotebook \\\n --param NOTEBOOK_IMAGE=datascience-notebook \\\n --param NOTEBOOK_PASSWORD=mypassword\n```\n\nImporting an image into OpenShift before deploying it means that when a notebook is started, the image need only be pulled from the internal OpenShift image registry rather than Docker Hub for each deployment.\nBecause the images are so large, this can speed up deployments when the image hasn't previously been deployed to a node in the OpenShift cluster.\n" }, { "path": "examples/openshift/templates.json", "content": "{\n \"kind\": \"Template\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"jupyter-notebook\",\n \"annotations\": {\n \"openshift.io/display-name\": \"Jupyter Notebook\",\n \"description\": \"Template for deploying Jupyter Notebook images.\",\n \"iconClass\": \"icon-python\",\n \"tags\": \"python,jupyter\"\n }\n },\n \"parameters\": [\n {\n \"name\": \"APPLICATION_NAME\",\n \"value\": \"notebook\",\n \"required\": true\n },\n {\n \"name\": \"NOTEBOOK_IMAGE\",\n \"value\": \"docker.io/jupyter/minimal-notebook:latest\",\n \"required\": true\n },\n {\n \"name\": \"NOTEBOOK_PASSWORD\",\n \"from\": \"[a-f0-9]{32}\",\n \"generate\": \"expression\"\n }\n ],\n \"objects\": [\n {\n \"kind\": \"ConfigMap\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"${APPLICATION_NAME}-cfg\",\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\"\n }\n },\n \"data\": {\n \"jupyter_server_config.py\": \"import os\\n\\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\\n\\nif password:\\n from jupyter_server.auth import passwd\\n c.ServerApp.password = passwd(password)\\n del password\\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\\n\\nimage_config_file = '/home/jovyan/.jupyter/jupyter_server_config.py'\\n\\nif os.path.exists(image_config_file):\\n with open(image_config_file) as fp:\\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\\n\"\n }\n },\n {\n \"kind\": \"DeploymentConfig\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"${APPLICATION_NAME}\",\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\"\n }\n },\n \"spec\": {\n \"strategy\": {\n \"type\": \"Recreate\"\n },\n \"triggers\": [\n {\n \"type\": \"ConfigChange\"\n }\n ],\n \"replicas\": 1,\n \"selector\": {\n \"app\": \"${APPLICATION_NAME}\",\n \"deploymentconfig\": \"${APPLICATION_NAME}\"\n },\n \"template\": {\n \"metadata\": {\n \"annotations\": {\n \"alpha.image.policy.openshift.io/resolve-names\": \"*\"\n },\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\",\n \"deploymentconfig\": \"${APPLICATION_NAME}\"\n }\n },\n \"spec\": {\n \"containers\": [\n {\n \"name\": \"jupyter-notebook\",\n \"image\": \"${NOTEBOOK_IMAGE}\",\n \"command\": [\n \"start-notebook.py\",\n \"--config=/etc/jupyter/openshift/jupyter_server_config.py\",\n \"--no-browser\",\n \"--ip=0.0.0.0\"\n ],\n \"ports\": [\n {\n \"containerPort\": 8888,\n \"protocol\": \"TCP\"\n }\n ],\n \"env\": [\n {\n \"name\": \"JUPYTER_NOTEBOOK_PASSWORD\",\n \"value\": \"${NOTEBOOK_PASSWORD}\"\n }\n ],\n \"volumeMounts\": [\n {\n \"mountPath\": \"/etc/jupyter/openshift\",\n \"name\": \"configs\"\n }\n ]\n }\n ],\n \"automountServiceAccountToken\": false,\n \"securityContext\": {\n \"supplementalGroups\": [100]\n },\n \"volumes\": [\n {\n \"configMap\": {\n \"name\": \"${APPLICATION_NAME}-cfg\"\n },\n \"name\": \"configs\"\n }\n ]\n }\n }\n }\n },\n {\n \"kind\": \"Route\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"${APPLICATION_NAME}\",\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\"\n }\n },\n \"spec\": {\n \"host\": \"\",\n \"to\": {\n \"kind\": \"Service\",\n \"name\": \"${APPLICATION_NAME}\",\n \"weight\": 100\n },\n \"port\": {\n \"targetPort\": \"8888-tcp\"\n },\n \"tls\": {\n \"termination\": \"edge\",\n \"insecureEdgeTerminationPolicy\": \"Redirect\"\n }\n }\n },\n {\n \"kind\": \"Service\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"${APPLICATION_NAME}\",\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\"\n }\n },\n \"spec\": {\n \"ports\": [\n {\n \"name\": \"8888-tcp\",\n \"protocol\": \"TCP\",\n \"port\": 8888,\n \"targetPort\": 8888\n }\n ],\n \"selector\": {\n \"app\": \"${APPLICATION_NAME}\",\n \"deploymentconfig\": \"${APPLICATION_NAME}\"\n },\n \"type\": \"ClusterIP\"\n }\n }\n ]\n}\n" }, { "path": "examples/source-to-image/README.md", "content": "# Custom Jupyter Notebook images\n\nThis example provides scripts for building custom Jupyter Notebook images containing notebooks, data files, and with Python packages required by the notebooks already installed.\nThe scripts provided work with the Source-to-Image tool, and you can create the images from the command line on your own computer.\nTemplates are also provided to enable running builds in OpenShift, as well as deploying the resulting image to OpenShift to make it available.\n\nThe build scripts, when used with the Source-to-Image tool, provide similar capabilities to `repo2docker`.\nWhen builds are run under OpenShift with the supplied templates, it provides similar capabilities to `mybinder.org`,\nbut where notebook instances are deployed in your existing OpenShift project and JupyterHub is not required.\n\nFor separate examples of using JupyterHub with OpenShift, see the project:\n\n- \n\n## Source-to-Image Project\n\nSource-to-Image (S2I) is an open source project which provides a tool for creating container images.\nIt works by taking a base image, injecting additional source code or files into a running container created from the base image,\nand running a builder script in the container to process the source code or files to prepare the new image.\n\nDetails on the S2I tool, and executable binaries for Linux, macOS and Windows, can be found on GitHub at:\n\n- \n\nThe tool is standalone, and can be used on any system which provides a docker daemon for running containers.\nTo provide an end-to-end capability to build and deploy applications in containers, support for S2I is also integrated into container platforms such as OpenShift.\n\n## Getting Started with S2I\n\nAs an example of how S2I can be used to create a custom image with a bundled set of notebooks, run:\n\n```bash\ns2i build \\\n --scripts-url https://raw.githubusercontent.com/jupyter/docker-stacks/main/examples/source-to-image \\\n --context-dir docs/source/examples/Notebook \\\n https://github.com/jupyter/notebook \\\n docker.io/jupyter/minimal-notebook:latest \\\n notebook-examples\n```\n\nThis example command will pull down the Git repository \nand build the image `notebook-examples` using the files contained in the `docs/source/examples/Notebook` directory of that Git repository.\nThe base image which the files will be combined with is `docker.io/jupyter/minimal-notebook:latest`, but you can specify any of the Jupyter Project `docker-stacks` images as the base image.\n\nThe resulting image from running the command can be seen by running `docker images` command:\n\n```bash\ndocker images\n# REPOSITORY TAG IMAGE ID CREATED SIZE\n# notebook-examples latest f5899ed1241d 2 minutes ago 2.59GB\n```\n\nYou can now run the image.\n\n```bash\ndocker run --rm -p 8888:8888 notebook-examples\n```\n\nOpen your browser on the URL displayed, and you will find the notebooks from the Git repository and can work with them.\n\n## The S2I Builder Scripts\n\nNormally when using S2I, the base image would be S2I enabled and contain the builder scripts needed to prepare the image and define how the application in the image should be run.\nAs the Jupyter Project `docker-stacks` images are not S2I enabled (although they could be),\nin the above example the `--scripts-url` option has been used to specify that the example builder scripts contained in this directory of this Git repository should be used.\n\nUsing the `--scripts-url` option, the builder scripts can be hosted on any HTTP server,\nor you could also use builder scripts local to your computer file using an appropriate `file://` format URI argument to `--scripts-url`.\n\nThe builder scripts in this directory of this repository are `assemble` and `run` and are provided as examples of what can be done.\nYou can use the scripts as is, or create your own.\n\nThe supplied `assemble` script performs a few key steps.\n\nThe first steps copy files into the location they need to be when the image is run, from the directory where they are initially placed by the `s2i` command.\n\n```bash\ncp -Rf /tmp/src/. \"/home/${NB_USER}\"\n\nrm -rf /tmp/src\n```\n\nThe next steps are:\n\n```bash\nif [ -f \"/home/${NB_USER}/environment.yml\" ]; then\n mamba env update --name root --file \"/home/${NB_USER}/environment.yml\"\n mamba clean --all -f -y\nelse\n if [ -f \"/home/${NB_USER}/requirements.txt\" ]; then\n pip --no-cache-dir install -r \"/home/${NB_USER}/requirements.txt\"\n fi\nfi\n```\n\nThis determines whether a `environment.yml` or `requirements.txt` file exists with the files and if so, runs the appropriate package management tool to install any Python packages listed in those files.\n\nThis means that so long as a set of notebook files provides one of these files listing what Python packages they need,\nthose packages will be automatically installed into the image, so they are available when the image is run.\n\nA final step is:\n\n```bash\nfix-permissions \"${CONDA_DIR}\"\nfix-permissions \"/home/${NB_USER}\"\n```\n\nThis fixes up permissions on any new files created by the build.\nThis is necessary to ensure that when the image is run, you can still install additional files.\nThis is important for when an image is run in `sudo` mode, or it is hosted in a more secure container platform such as Kubernetes/OpenShift where it will be run as a set user ID that isn't known in advance.\n\nAs long as you preserve the first and last set of steps, you can do whatever you want in the `assemble` script to install packages, create files etc.\nDo be aware though that S2I builds do not run as `root` and so you cannot install additional system packages.\nIf you need to install additional system packages, use a `Dockerfile` and normal `docker build` to first create a new custom base image from the Jupyter Project `docker-stacks` images,\nwith the extra system packages, and then use that image with the S2I build to combine your notebooks and have Python packages installed.\n\nThe `run` script in this directory is very simple and just runs the notebook application.\n\n```bash\nexec start-notebook.py \"$@\"\n```\n\n## Integration with OpenShift\n\nThe OpenShift platform provides integrated support for S2I type builds.\nTemplates are provided for using the S2I build mechanism with the scripts in this directory.\nTo load the templates run:\n\n```bash\noc create -f https://raw.githubusercontent.com/jupyter/docker-stacks/main/examples/source-to-image/templates.json\n```\n\nThis will create the templates:\n\n```bash\njupyter-notebook-builder\njupyter-notebook-quickstart\n```\n\nThe templates can be used from the OpenShift web console or command line.\nThis `README` is only going to explain deploying from the command line.\n\nTo use the OpenShift command line to build into an image, and deploy, the set of notebooks used above, run:\n\n```bash\noc new-app --template jupyter-notebook-quickstart \\\n --param APPLICATION_NAME=notebook-examples \\\n --param GIT_REPOSITORY_URL=https://github.com/jupyter/notebook \\\n --param CONTEXT_DIR=docs/source/examples/Notebook \\\n --param BUILDER_IMAGE=docker.io/jupyter/minimal-notebook:latest \\\n --param NOTEBOOK_PASSWORD=mypassword\n```\n\nYou can provide a password using the `NOTEBOOK_PASSWORD` parameter.\nIf you don't set that parameter, a password will be generated, with it being displayed by the `oc new-app` command.\n\nOnce the image has been built, it will be deployed.\nTo see the hostname for accessing the notebook, run `oc get routes`.\n\n```lang-none\nNAME HOST/PORT PATH SERVICES PORT TERMINATION WILDCARD\nnotebook-examples notebook-examples-jupyter.abcd.pro-us-east-1.openshiftapps.com notebook-examples 8888-tcp edge/Redirect None\n```\n\nAs the deployment will use a secure connection, the URL for accessing the notebook in this case would be .\n\nIf you only want to build an image but not deploy it, you can use the `jupyter-notebook-builder` template.\nYou can then deploy it using the `jupyter-notebook` template provided with the [openshift](../openshift) examples directory.\n\nSee the `openshift` examples directory for further information on customizing configuration for a Jupyter Notebook deployment and deleting a deployment.\n" }, { "path": "examples/source-to-image/assemble", "content": "#!/bin/bash\n\nset -x\n\nset -eo pipefail\n\n# Remove any 'environment.yml' or 'requirements.txt' files which may\n# have been carried over from the base image so we don't reinstall\n# packages which have already been installed. This could occur where\n# an S2I build was used to create a new base image with pre-installed\n# Python packages, with the new image then subsequently being used as a\n# S2I builder base image.\n\nrm -f \"/home/${NB_USER}/environment.yml\"\nrm -f \"/home/${NB_USER}/requirements.txt\"\n\n# Copy injected files to target directory.\n\ncp -Rf /tmp/src/. \"/home/${NB_USER}\"\n\nrm -rf /tmp/src\n\n# Install any Python modules. If we find an 'environment.yml' file we\n# assume we should use 'conda' to install packages. If 'requirements.txt'\n# use 'pip' instead.\n\nif [ -f \"/home/${NB_USER}/environment.yml\" ]; then\n mamba env update --name root --file \"/home/${NB_USER}/environment.yml\"\n mamba clean --all -f -y\nelse\n if [ -f \"/home/${NB_USER}/requirements.txt\" ]; then\n pip --no-cache-dir install -r \"/home/${NB_USER}/requirements.txt\"\n fi\nfi\n\n# Fix up permissions on home directory and Python installation so that\n# everything is still writable by 'users' group.\n\nfix-permissions \"${CONDA_DIR}\"\nfix-permissions \"/home/${NB_USER}\"\n" }, { "path": "examples/source-to-image/run", "content": "#!/bin/bash\n\n# Start up the notebook instance.\n\nexec start-notebook.py \"$@\"\n" }, { "path": "examples/source-to-image/save-artifacts", "content": "#!/bin/bash\n\ntar cf - --files-from /dev/null\n" }, { "path": "examples/source-to-image/templates.json", "content": "{\n \"kind\": \"List\",\n \"apiVersion\": \"v1\",\n \"items\": [\n {\n \"kind\": \"Template\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"jupyter-notebook-builder\",\n \"annotations\": {\n \"openshift.io/display-name\": \"Jupyter Notebook Builder\",\n \"description\": \"Template for building Jupyter Notebook images with bundled notebooks and files.\",\n \"iconClass\": \"icon-python\",\n \"tags\": \"python,jupyter\"\n }\n },\n \"parameters\": [\n {\n \"name\": \"IMAGE_NAME\",\n \"value\": \"notebook\",\n \"required\": true\n },\n {\n \"name\": \"BUILDER_IMAGE\",\n \"value\": \"docker.io/jupyter/minimal-notebook:latest\",\n \"required\": true\n },\n {\n \"name\": \"BUILDER_SCRIPTS\",\n \"value\": \"https://raw.githubusercontent.com/jupyter/docker-stacks/main/examples/source-to-image\",\n \"required\": true\n },\n {\n \"name\": \"GIT_REPOSITORY_URL\",\n \"value\": \"\",\n \"required\": true\n },\n {\n \"name\": \"GIT_REFERENCE\",\n \"value\": \"main\",\n \"required\": true\n },\n {\n \"name\": \"CONTEXT_DIR\",\n \"value\": \"\",\n \"required\": false\n }\n ],\n \"objects\": [\n {\n \"apiVersion\": \"v1\",\n \"kind\": \"ImageStream\",\n \"metadata\": {\n \"name\": \"${IMAGE_NAME}\",\n \"labels\": {\n \"app\": \"${IMAGE_NAME}\"\n }\n }\n },\n {\n \"apiVersion\": \"v1\",\n \"kind\": \"BuildConfig\",\n \"metadata\": {\n \"name\": \"${IMAGE_NAME}\",\n \"labels\": {\n \"app\": \"${IMAGE_NAME}\"\n }\n },\n \"spec\": {\n \"output\": {\n \"to\": {\n \"kind\": \"ImageStreamTag\",\n \"name\": \"${IMAGE_NAME}:latest\"\n }\n },\n \"resources\": {\n \"limits\": {\n \"memory\": \"1Gi\"\n }\n },\n \"source\": {\n \"type\": \"Git\",\n \"git\": {\n \"uri\": \"${GIT_REPOSITORY_URL}\",\n \"ref\": \"${GIT_REFERENCE}\"\n },\n \"contextDir\": \"${CONTEXT_DIR}\"\n },\n \"strategy\": {\n \"type\": \"Source\",\n \"sourceStrategy\": {\n \"from\": {\n \"kind\": \"DockerImage\",\n \"name\": \"${BUILDER_IMAGE}\"\n },\n \"scripts\": \"${BUILDER_SCRIPTS}\"\n }\n },\n \"triggers\": [\n {\n \"type\": \"ConfigChange\"\n }\n ]\n }\n }\n ]\n },\n {\n \"kind\": \"Template\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"jupyter-notebook-quickstart\",\n \"annotations\": {\n \"openshift.io/display-name\": \"Jupyter Notebook QuickStart\",\n \"description\": \"Template for deploying Jupyter Notebook images with bundled notebooks and files.\",\n \"iconClass\": \"icon-python\",\n \"tags\": \"python,jupyter\"\n }\n },\n \"parameters\": [\n {\n \"name\": \"APPLICATION_NAME\",\n \"value\": \"notebook\",\n \"required\": true\n },\n {\n \"name\": \"BUILDER_IMAGE\",\n \"value\": \"docker.io/jupyter/minimal-notebook:latest\",\n \"required\": true\n },\n {\n \"name\": \"BUILDER_SCRIPTS\",\n \"value\": \"https://raw.githubusercontent.com/jupyter/docker-stacks/main/examples/source-to-image\",\n \"required\": true\n },\n {\n \"name\": \"GIT_REPOSITORY_URL\",\n \"value\": \"\",\n \"required\": true\n },\n {\n \"name\": \"GIT_REFERENCE\",\n \"value\": \"main\",\n \"required\": true\n },\n {\n \"name\": \"CONTEXT_DIR\",\n \"value\": \"\",\n \"required\": false\n },\n {\n \"name\": \"NOTEBOOK_PASSWORD\",\n \"from\": \"[a-f0-9]{32}\",\n \"generate\": \"expression\"\n }\n ],\n \"objects\": [\n {\n \"apiVersion\": \"v1\",\n \"kind\": \"ImageStream\",\n \"metadata\": {\n \"name\": \"${APPLICATION_NAME}\",\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\"\n }\n }\n },\n {\n \"apiVersion\": \"v1\",\n \"kind\": \"BuildConfig\",\n \"metadata\": {\n \"name\": \"${APPLICATION_NAME}\",\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\"\n }\n },\n \"spec\": {\n \"output\": {\n \"to\": {\n \"kind\": \"ImageStreamTag\",\n \"name\": \"${APPLICATION_NAME}:latest\"\n }\n },\n \"resources\": {\n \"limits\": {\n \"memory\": \"1Gi\"\n }\n },\n \"source\": {\n \"type\": \"Git\",\n \"git\": {\n \"uri\": \"${GIT_REPOSITORY_URL}\",\n \"ref\": \"${GIT_REFERENCE}\"\n },\n \"contextDir\": \"${CONTEXT_DIR}\"\n },\n \"strategy\": {\n \"type\": \"Source\",\n \"sourceStrategy\": {\n \"from\": {\n \"kind\": \"DockerImage\",\n \"name\": \"${BUILDER_IMAGE}\"\n },\n \"scripts\": \"${BUILDER_SCRIPTS}\"\n }\n },\n \"triggers\": [\n {\n \"type\": \"ConfigChange\"\n }\n ]\n }\n },\n {\n \"kind\": \"ConfigMap\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"${APPLICATION_NAME}-cfg\",\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\"\n }\n },\n \"data\": {\n \"jupyter_server_config.py\": \"import os\\n\\npassword = os.environ.get('JUPYTER_NOTEBOOK_PASSWORD')\\n\\nif password:\\n from jupyter_server.auth import passwd\\n c.ServerApp.password = passwd(password)\\n del password\\n del os.environ['JUPYTER_NOTEBOOK_PASSWORD']\\n\\nimage_config_file = '/home/jovyan/.jupyter/jupyter_server_config.py'\\n\\nif os.path.exists(image_config_file):\\n with open(image_config_file) as fp:\\n exec(compile(fp.read(), image_config_file, 'exec'), globals())\\n\"\n }\n },\n {\n \"kind\": \"DeploymentConfig\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"${APPLICATION_NAME}\",\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\"\n }\n },\n \"spec\": {\n \"strategy\": {\n \"type\": \"Recreate\"\n },\n \"triggers\": [\n {\n \"type\": \"ConfigChange\"\n },\n {\n \"type\": \"ImageChange\",\n \"imageChangeParams\": {\n \"automatic\": true,\n \"containerNames\": [\"jupyter-notebook\"],\n \"from\": {\n \"kind\": \"ImageStreamTag\",\n \"name\": \"${APPLICATION_NAME}:latest\"\n }\n }\n }\n ],\n \"replicas\": 1,\n \"selector\": {\n \"app\": \"${APPLICATION_NAME}\",\n \"deploymentconfig\": \"${APPLICATION_NAME}\"\n },\n \"template\": {\n \"metadata\": {\n \"annotations\": {\n \"alpha.image.policy.openshift.io/resolve-names\": \"*\"\n },\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\",\n \"deploymentconfig\": \"${APPLICATION_NAME}\"\n }\n },\n \"spec\": {\n \"containers\": [\n {\n \"name\": \"jupyter-notebook\",\n \"image\": \"${APPLICATION_NAME}:latest\",\n \"command\": [\n \"start-notebook.py\",\n \"--config=/etc/jupyter/openshift/jupyter_server_config.py\",\n \"--no-browser\",\n \"--ip=0.0.0.0\"\n ],\n \"ports\": [\n {\n \"containerPort\": 8888,\n \"protocol\": \"TCP\"\n }\n ],\n \"env\": [\n {\n \"name\": \"JUPYTER_NOTEBOOK_PASSWORD\",\n \"value\": \"${NOTEBOOK_PASSWORD}\"\n }\n ],\n \"volumeMounts\": [\n {\n \"mountPath\": \"/etc/jupyter/openshift\",\n \"name\": \"configs\"\n }\n ]\n }\n ],\n \"automountServiceAccountToken\": false,\n \"securityContext\": {\n \"supplementalGroups\": [100]\n },\n \"volumes\": [\n {\n \"configMap\": {\n \"name\": \"${APPLICATION_NAME}-cfg\"\n },\n \"name\": \"configs\"\n }\n ]\n }\n }\n }\n },\n {\n \"kind\": \"Route\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"${APPLICATION_NAME}\",\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\"\n }\n },\n \"spec\": {\n \"host\": \"\",\n \"to\": {\n \"kind\": \"Service\",\n \"name\": \"${APPLICATION_NAME}\",\n \"weight\": 100\n },\n \"port\": {\n \"targetPort\": \"8888-tcp\"\n },\n \"tls\": {\n \"termination\": \"edge\",\n \"insecureEdgeTerminationPolicy\": \"Redirect\"\n }\n }\n },\n {\n \"kind\": \"Service\",\n \"apiVersion\": \"v1\",\n \"metadata\": {\n \"name\": \"${APPLICATION_NAME}\",\n \"labels\": {\n \"app\": \"${APPLICATION_NAME}\"\n }\n },\n \"spec\": {\n \"ports\": [\n {\n \"name\": \"8888-tcp\",\n \"protocol\": \"TCP\",\n \"port\": 8888,\n \"targetPort\": 8888\n }\n ],\n \"selector\": {\n \"app\": \"${APPLICATION_NAME}\",\n \"deploymentconfig\": \"${APPLICATION_NAME}\"\n },\n \"type\": \"ClusterIP\"\n }\n }\n ]\n }\n ]\n}\n" }, { "path": "images/all-spark-notebook/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/all-spark-notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/pyspark-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\n# RSpark config\nENV R_LIBS_USER=\"${SPARK_HOME}/R/lib\"\nRUN fix-permissions \"${R_LIBS_USER}\"\n\n# R pre-requisites\nRUN apt-get update --yes && \\\n apt-get install --yes --no-install-recommends \\\n fonts-dejavu \\\n gfortran \\\n gcc && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\nUSER ${NB_UID}\n\n# R packages including IRKernel which gets installed globally.\nRUN mamba install --yes \\\n 'r-base' \\\n 'r-ggplot2' \\\n 'r-irkernel' \\\n 'r-rcurl' \\\n 'r-sparklyr' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "images/all-spark-notebook/README.md", "content": "# Jupyter Notebook Python, R, Spark Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/all-spark-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-all-spark-notebook)\n- [Image Specifics :: Apache Spark](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/specifics.html#apache-spark)\n" }, { "path": "images/base-notebook/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/base-notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/docker-stacks-foundation\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\n# Install all OS dependencies for the Server that starts\n# but lacks all features (e.g., download as all possible file formats)\nRUN apt-get update --yes && \\\n apt-get install --yes --no-install-recommends \\\n # - Add necessary fonts for matplotlib/seaborn\n # See https://github.com/jupyter/docker-stacks/pull/380 for details\n fonts-liberation \\\n # - `pandoc` is used to convert notebooks to html files\n # it's not present in the aarch64 Ubuntu image, so we install it here\n pandoc \\\n # - `run-one` - a wrapper script that runs no more\n # than one unique instance of some command with a unique set of arguments,\n # we use `run-one-constantly` to support the `RESTARTABLE` option\n run-one && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\nUSER ${NB_UID}\n\n# Install JupyterHub, JupyterLab, NBClassic and Jupyter Notebook\n# Generate a Jupyter Server config\n# Cleanup temporary files\n# Correct permissions\n# Do all this in a single RUN command to avoid duplicating all of the\n# files across image layers when the permissions change\nWORKDIR /tmp\nRUN mamba install --yes \\\n 'jupyterhub-singleuser' \\\n 'jupyterlab' \\\n 'nbclassic' \\\n # Sometimes, when the new version of `jupyterlab` is released, latest `notebook` might not support it for some time\n # Old versions of `notebook` (=7.2.2' && \\\n jupyter server --generate-config && \\\n mamba clean --all -f -y && \\\n jupyter lab clean && \\\n rm -rf \"/home/${NB_USER}/.cache/yarn\" && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\nENV JUPYTER_PORT=8888\nEXPOSE $JUPYTER_PORT\n\n# Configure container startup\nCMD [\"start-notebook.py\"]\n\n# Copy local files as late as possible to avoid cache busting\nCOPY start-notebook.py start-notebook.sh start-singleuser.py start-singleuser.sh /usr/local/bin/\nCOPY jupyter_server_config.py docker_healthcheck.py /etc/jupyter/\n\n# Fix permissions on /etc/jupyter as root\nUSER root\nRUN fix-permissions /etc/jupyter/\n\n# HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck\n# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server`, and `retro` jupyter commands\n# https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799\nHEALTHCHECK --interval=3s --timeout=1s --start-period=3s --retries=3 \\\n CMD /etc/jupyter/docker_healthcheck.py || exit 1\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\n# Switch back to jovyan to avoid accidental container runs as root\nUSER ${NB_UID}\n\nWORKDIR \"${HOME}\"\n" }, { "path": "images/base-notebook/README.md", "content": "# Base Jupyter Notebook Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/base-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-base-notebook)\n" }, { "path": "images/base-notebook/docker_healthcheck.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport json\nimport os\nimport subprocess\nfrom pathlib import Path\n\nimport requests\n\n# Several operations below deliberately don't check for possible errors\n# As this is a health check, it should succeed or raise an exception on error\n\n# Docker runs health checks using an exec\n# It uses the default user configured when running the image: root for the case of a custom NB_USER or jovyan for the case of the default image user.\n# We manually change HOME to make `jupyter --runtime-dir` report a correct path\n# More information: \nresult = subprocess.run(\n [\"jupyter\", \"--runtime-dir\"],\n check=True,\n capture_output=True,\n text=True,\n env=dict(os.environ) | {\"HOME\": \"/home/\" + os.environ[\"NB_USER\"]},\n)\nruntime_dir = Path(result.stdout.rstrip())\n\njson_file = next(runtime_dir.glob(\"*server-*.json\"))\n\nurl = json.loads(json_file.read_bytes())[\"url\"]\nurl = url + \"api\"\n\nproxies = {\n \"http\": \"\",\n \"https\": \"\",\n}\n\nr = requests.get(url, proxies=proxies, verify=False) # request without SSL verification\nr.raise_for_status()\nprint(r.content)\n" }, { "path": "images/base-notebook/jupyter_server_config.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n# mypy: ignore-errors\nimport os\nimport stat\nimport subprocess\nfrom pathlib import Path\n\nfrom jupyter_core.paths import jupyter_data_dir\n\nc = get_config() # noqa: F821\n# Listen on all interfaces (ipv4 and ipv6)\nc.ServerApp.ip = \"\"\nc.ServerApp.open_browser = False\n\n# to output both image/svg+xml and application/pdf plot formats in the notebook file\nc.InlineBackend.figure_formats = {\"png\", \"jpeg\", \"svg\", \"pdf\"}\n\n# https://github.com/jupyter/notebook/issues/3130\nc.FileContentsManager.delete_to_trash = False\n\n# Generate a self-signed certificate\nOPENSSL_CONFIG = \"\"\"\\\n[req]\ndistinguished_name = req_distinguished_name\n[req_distinguished_name]\n\"\"\"\nif \"GEN_CERT\" in os.environ:\n dir_name = Path(jupyter_data_dir())\n dir_name.mkdir(parents=True, exist_ok=True)\n pem_file = dir_name / \"notebook.pem\"\n\n # Generate an openssl.cnf file to set the distinguished name\n cnf_file = Path(os.getenv(\"CONDA_DIR\", \"/usr/lib\")) / \"ssl/openssl.cnf\"\n if not cnf_file.exists():\n cnf_file.write_text(OPENSSL_CONFIG)\n\n # Generate a certificate if one doesn't exist on a disk\n subprocess.check_call(\n [\n \"openssl\",\n \"req\",\n \"-new\",\n \"-newkey=rsa:2048\",\n \"-days=365\",\n \"-nodes\",\n \"-x509\",\n \"-subj=/C=XX/ST=XX/L=XX/O=generated/CN=generated\",\n f\"-keyout={pem_file}\",\n f\"-out={pem_file}\",\n ]\n )\n # Restrict access to the file\n pem_file.chmod(stat.S_IRUSR | stat.S_IWUSR)\n c.ServerApp.certfile = str(pem_file)\n\n# Change default umask for all subprocesses of the Server if set in the environment\nif \"NB_UMASK\" in os.environ:\n os.umask(int(os.environ[\"NB_UMASK\"], 8))\n" }, { "path": "images/base-notebook/start-notebook.py", "content": "#!/usr/bin/env python\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport os\nimport shlex\nimport sys\n\n# If we are in a JupyterHub, we pass on to `start-singleuser.py` instead so it does the right thing\nif \"JUPYTERHUB_API_TOKEN\" in os.environ:\n print(\n \"WARNING: using start-singleuser.py instead of start-notebook.py to start a server associated with JupyterHub.\",\n flush=True,\n )\n command = [\"/usr/local/bin/start-singleuser.py\"] + sys.argv[1:]\n os.execvp(command[0], command)\n\n\n# Entrypoint is start.sh\ncommand = []\n\n# If we want to survive restarts, launch the command using `run-one-constantly`\nif os.environ.get(\"RESTARTABLE\") == \"yes\":\n command.append(\"run-one-constantly\")\n\n# We always launch a jupyter subcommand from this script\ncommand.append(\"jupyter\")\n\n# Launch the configured subcommand.\n# Note that this should be a single string, so we don't split it.\n# We default to `lab`.\njupyter_command = os.environ.get(\"DOCKER_STACKS_JUPYTER_CMD\", \"lab\")\ncommand.append(jupyter_command)\n\n# Append any optional NOTEBOOK_ARGS we were passed in.\n# This is supposed to be multiple args passed on to the notebook command,\n# so we split it correctly with shlex\nif \"NOTEBOOK_ARGS\" in os.environ:\n command += shlex.split(os.environ[\"NOTEBOOK_ARGS\"])\n\n# Pass through any other args we were passed on the command line\ncommand += sys.argv[1:]\n\n# Execute the command!\nprint(\"Executing: \" + \" \".join(command), flush=True)\nos.execvp(command[0], command)\n" }, { "path": "images/base-notebook/start-notebook.sh", "content": "#!/bin/bash\n# Shim to emit warning and call start-notebook.py\necho \"WARNING: Use start-notebook.py instead\"\n\nexec /usr/local/bin/start-notebook.py \"$@\"\n" }, { "path": "images/base-notebook/start-singleuser.py", "content": "#!/usr/bin/env python\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport os\nimport shlex\nimport sys\n\n# Entrypoint is start.sh\ncommand = [\"jupyterhub-singleuser\"]\n\n# JupyterHub singleuser arguments are set using environment variables\n\n# Append any optional NOTEBOOK_ARGS we were passed in.\n# This is supposed to be multiple args passed on to the notebook command,\n# so we split it correctly with shlex\nif \"NOTEBOOK_ARGS\" in os.environ:\n command += shlex.split(os.environ[\"NOTEBOOK_ARGS\"])\n\n# Pass any other args we have been passed through\ncommand += sys.argv[1:]\n\n# Execute the command!\nprint(\"Executing: \" + \" \".join(command), flush=True)\nos.execvp(command[0], command)\n" }, { "path": "images/base-notebook/start-singleuser.sh", "content": "#!/bin/bash\n# Shim to emit warning and call start-singleuser.py\necho \"WARNING: Use start-singleuser.py instead\"\n\nexec /usr/local/bin/start-singleuser.py \"$@\"\n" }, { "path": "images/datascience-notebook/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/datascience-notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\n# R pre-requisites\nRUN apt-get update --yes && \\\n apt-get install --yes --no-install-recommends \\\n fonts-dejavu \\\n gfortran \\\n gcc && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# Julia dependencies\n# install Julia packages in /opt/julia instead of ${HOME}\nENV JULIA_DEPOT_PATH=/opt/julia \\\n JULIA_PKGDIR=/opt/julia\n\n# Setup Julia\nRUN /opt/setup-scripts/setup_julia.py\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\n# hadolint ignore=DL3059\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\nUSER ${NB_UID}\n\n# Setup IJulia kernel & other packages\nRUN /opt/setup-scripts/setup-julia-packages.bash\n\n# R packages including IRKernel which gets installed globally.\n# r-e1071: dependency of the caret R package\nRUN mamba install --yes \\\n 'r-base' \\\n 'r-caret' \\\n 'r-crayon' \\\n 'r-devtools' \\\n 'r-e1071' \\\n 'r-forecast' \\\n 'r-hexbin' \\\n 'r-htmltools' \\\n 'r-htmlwidgets' \\\n 'r-irkernel' \\\n 'r-nycflights13' \\\n 'r-randomforest' \\\n 'r-rcurl' \\\n 'r-rmarkdown' \\\n 'r-rodbc' \\\n 'r-rsqlite' \\\n 'r-shiny' \\\n 'r-tidymodels' \\\n 'r-tidyverse' \\\n 'rpy2' \\\n 'unixodbc' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "images/datascience-notebook/README.md", "content": "# Jupyter Notebook Data Science Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/datascience-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-datascience-notebook)\n" }, { "path": "images/docker-stacks-foundation/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/docker-stacks-foundation/10activate-conda-env.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# This registers the initialization code for the conda shell code\n# It also activates default environment in the end, so we don't need to activate it manually\n# Documentation: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html\neval \"$(conda shell.bash hook)\"\n" }, { "path": "images/docker-stacks-foundation/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# Ubuntu 24.04 (noble)\n# https://hub.docker.com/_/ubuntu/tags?page=1&name=noble\nARG ROOT_IMAGE=ubuntu:24.04\n\nFROM $ROOT_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\nARG NB_USER=\"jovyan\"\nARG NB_UID=\"1000\"\nARG NB_GID=\"100\"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\n# Install all OS dependencies for the Server that starts\n# but lacks all features (e.g., download as all possible file formats)\nENV DEBIAN_FRONTEND=noninteractive\nRUN apt-get update --yes && \\\n # - `apt-get upgrade` is run to patch known vulnerabilities in system packages\n # as the Ubuntu base image is rebuilt too seldom sometimes (less than once a month)\n apt-get upgrade --yes && \\\n apt-get install --yes --no-install-recommends \\\n # - bzip2 is necessary to extract the micromamba executable.\n bzip2 \\\n ca-certificates \\\n locales \\\n # - `netbase` provides /etc/{protocols,rpc,services}, part of POSIX\n # and required by various C functions like getservbyname and getprotobyname\n # https://github.com/jupyter/docker-stacks/pull/2129\n netbase \\\n sudo \\\n # - `tini` is installed as a helpful container entrypoint,\n # that reaps zombie processes and such of the actual executable we want to start\n # See https://github.com/krallin/tini#why-tini for details\n tini \\\n wget && \\\n apt-get clean && rm -rf /var/lib/apt/lists/* && \\\n echo \"en_US.UTF-8 UTF-8\" > /etc/locale.gen && \\\n echo \"C.UTF-8 UTF-8\" >> /etc/locale.gen && \\\n locale-gen\n\n# Configure environment\nENV CONDA_DIR=/opt/conda \\\n SHELL=/bin/bash \\\n NB_USER=\"${NB_USER}\" \\\n NB_UID=${NB_UID} \\\n NB_GID=${NB_GID} \\\n LC_ALL=C.UTF-8 \\\n LANG=C.UTF-8 \\\n LANGUAGE=C.UTF-8\nENV PATH=\"${CONDA_DIR}/bin:${PATH}\" \\\n HOME=\"/home/${NB_USER}\"\n\n# Copy a script that we will use to correct permissions after running certain commands\nCOPY fix-permissions /usr/local/bin/fix-permissions\nRUN chmod a+rx /usr/local/bin/fix-permissions\n\n# Enable prompt color in the skeleton .bashrc before creating the default NB_USER\n# hadolint ignore=SC2016\nRUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \\\n # More information in: https://github.com/jupyter/docker-stacks/pull/2047\n # and docs: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html\n echo 'eval \"$(conda shell.bash hook)\"' >> /etc/skel/.bashrc\n\n# Delete existing user with UID=\"${NB_UID}\" if it exists\n# hadolint ignore=SC2046\nRUN if grep -q \"${NB_UID}\" /etc/passwd; then \\\n userdel --remove $(id -un \"${NB_UID}\"); \\\n fi\n\n# Create \"${NB_USER}\" user (`jovyan` by default) with UID=\"${NB_UID}\" (`1000` by default) and in the 'users' group\n# and make sure these dirs are writable by the `users` group.\nRUN echo \"auth requisite pam_deny.so\" >> /etc/pam.d/su && \\\n sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \\\n sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \\\n useradd --no-log-init --create-home --shell /bin/bash --uid \"${NB_UID}\" --no-user-group \"${NB_USER}\" && \\\n mkdir -p \"${CONDA_DIR}\" && \\\n chown \"${NB_USER}:${NB_GID}\" \"${CONDA_DIR}\" && \\\n chmod g+w /etc/passwd && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\nUSER ${NB_UID}\n\n# Pin the Python version here, or set it to \"default\"\nARG PYTHON_VERSION=3.13\n\n# Setup work directory for backward-compatibility\nRUN mkdir \"/home/${NB_USER}/work\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# Download and install Micromamba, and initialize the Conda prefix.\n# \n# Similar projects using Micromamba:\n# - Micromamba-Docker: \n# - repo2docker: \n# Install Python, Mamba, and jupyter_core\n# Cleanup temporary files and remove Micromamba\n# Correct permissions\n# Do all this in a single RUN command to avoid duplicating all of the\n# files across image layers when the permissions change\nCOPY --chown=\"${NB_UID}:${NB_GID}\" initial-condarc \"${CONDA_DIR}/.condarc\"\nWORKDIR /tmp\nRUN set -x && \\\n arch=$(uname -m) && \\\n if [ \"${arch}\" = \"x86_64\" ]; then \\\n # Should be simpler, see \n arch=\"64\"; \\\n fi && \\\n # https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html#linux-and-macos\n wget --progress=dot:giga -O - \\\n \"https://micro.mamba.pm/api/micromamba/linux-${arch}/latest\" | tar -xvj bin/micromamba && \\\n PYTHON_SPECIFIER=\"python=${PYTHON_VERSION}\" && \\\n if [[ \"${PYTHON_VERSION}\" == \"default\" ]]; then PYTHON_SPECIFIER=\"python\"; fi && \\\n # Install the packages\n ./bin/micromamba install \\\n --root-prefix=\"${CONDA_DIR}\" \\\n --prefix=\"${CONDA_DIR}\" \\\n --yes \\\n 'jupyter_core' \\\n 'conda' \\\n 'mamba' \\\n \"${PYTHON_SPECIFIER}\" && \\\n rm -rf /tmp/bin/ && \\\n # Pin major.minor version of python\n # https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#preventing-packages-from-updating-pinning\n mamba list --full-name 'python' | awk 'END{sub(\"[^.]*$\", \"*\", $2); print $1 \" \" $2}' >> \"${CONDA_DIR}/conda-meta/pinned\" && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# Copy local files as late as possible to avoid cache busting\nCOPY run-hooks.sh start.sh /usr/local/bin/\n\n# Configure container entrypoint\nENTRYPOINT [\"tini\", \"-g\", \"--\", \"start.sh\"]\n\nUSER root\n\n# Create dirs for startup hooks\nRUN mkdir /usr/local/bin/start-notebook.d && \\\n mkdir /usr/local/bin/before-notebook.d\n\nCOPY 10activate-conda-env.sh /usr/local/bin/before-notebook.d/\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\n# Switch back to jovyan to avoid accidental container runs as root\nUSER ${NB_UID}\n\nWORKDIR \"${HOME}\"\n" }, { "path": "images/docker-stacks-foundation/README.md", "content": "# Foundation Jupyter Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/docker-stacks-foundation](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-docker-stacks-foundation)\n" }, { "path": "images/docker-stacks-foundation/fix-permissions", "content": "#!/bin/bash\n# Set permissions on a directory\n# After any installation, if a directory needs to be (human) user-writable, run this script on it.\n# It will make everything in the directory owned by the group ${NB_GID} and writable by that group.\n# Deployments that want to set a specific user id can preserve permissions\n# by adding the `--group-add users` line to `docker run`.\n\n# Uses find to avoid touching files that already have the right permissions,\n# which would cause a massive image explosion\n\n# Right permissions are:\n# group=${NB_GID}\n# AND permissions include group rwX (directory-execute)\n# AND directories have setuid,setgid bits set\n\nset -e\n\nfor d in \"$@\"; do\n find \"${d}\" \\\n ! \\( \\\n -group \"${NB_GID}\" \\\n -a -perm -g+rwX \\\n \\) \\\n -exec chgrp \"${NB_GID}\" -- {} \\+ \\\n -exec chmod g+rwX -- {} \\+\n # setuid, setgid *on directories only*\n find \"${d}\" \\\n \\( \\\n -type d \\\n -a ! -perm -6000 \\\n \\) \\\n -exec chmod +6000 -- {} \\+\ndone\n" }, { "path": "images/docker-stacks-foundation/initial-condarc", "content": "# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html\n\nauto_update_conda: false\nshow_channel_urls: true\nchannels:\n - conda-forge\n" }, { "path": "images/docker-stacks-foundation/run-hooks.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# identical _log to start.sh\n# only used when _not_ sourced from start.sh (i.e. unittests)\nif ! declare -F _log > /dev/null; then\n _log () {\n if [[ \"$*\" == \"ERROR:\"* ]] || [[ \"$*\" == \"WARNING:\"* ]] || [[ \"${JUPYTER_DOCKER_STACKS_QUIET}\" == \"\" ]]; then\n echo \"$@\" >&2\n fi\n }\nfi\n\n# The run-hooks.sh script looks for *.sh scripts to source\n# and executable files to run within a passed directory\n\nif [ \"$#\" -ne 1 ]; then\n _log \"ERROR: Should pass exactly one directory\"\n return 1\nfi\n\nif [[ ! -d \"${1}\" ]]; then\n _log \"ERROR: Directory ${1} doesn't exist or is not a directory\"\n return 1\nfi\n\n_log \"Running hooks in: ${1} as uid: $(id -u) gid: $(id -g)\"\nfor f in \"${1}/\"*; do\n # Handling a case when the directory is empty\n [ -e \"${f}\" ] || continue\n case \"${f}\" in\n *.sh)\n _log \"Sourcing shell script: ${f}\"\n # shellcheck disable=SC1090\n source \"${f}\"\n # shellcheck disable=SC2181\n if [ $? -ne 0 ]; then\n _log \"ERROR: ${f} has failed, continuing execution\"\n fi\n ;;\n *)\n if [ -x \"${f}\" ]; then\n _log \"Running executable: ${f}\"\n \"${f}\"\n # shellcheck disable=SC2181\n if [ $? -ne 0 ]; then\n _log \"ERROR: ${f} has failed, continuing execution\"\n fi\n else\n _log \"Ignoring non-executable: ${f}\"\n fi\n ;;\n esac\ndone\n_log \"Done running hooks in: ${1}\"\n" }, { "path": "images/docker-stacks-foundation/start.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nset -e\n\n# The _log function is used for everything this script wants to log.\n# It will always log errors and warnings but can be silenced for other messages\n# by setting the JUPYTER_DOCKER_STACKS_QUIET environment variable.\n_log () {\n if [[ \"$*\" == \"ERROR:\"* ]] || [[ \"$*\" == \"WARNING:\"* ]] || [[ \"${JUPYTER_DOCKER_STACKS_QUIET}\" == \"\" ]]; then\n echo \"$@\" >&2\n fi\n}\n_log \"Entered start.sh with args:\" \"$@\"\n\n# A helper function to unset env vars listed in the value of the env var\n# JUPYTER_ENV_VARS_TO_UNSET.\nunset_explicit_env_vars () {\n if [ -n \"${JUPYTER_ENV_VARS_TO_UNSET}\" ]; then\n for env_var_to_unset in $(echo \"${JUPYTER_ENV_VARS_TO_UNSET}\" | tr ',' ' '); do\n _log \"Unset ${env_var_to_unset} due to JUPYTER_ENV_VARS_TO_UNSET\"\n unset \"${env_var_to_unset}\"\n done\n unset JUPYTER_ENV_VARS_TO_UNSET\n fi\n}\n\n\n# Default to starting bash if no command was specified\nif [ $# -eq 0 ]; then\n cmd=( \"bash\" )\nelse\n cmd=( \"$@\" )\nfi\n\n# Backwards compatibility: `start.sh` is executed by default in ENTRYPOINT\n# so it should no longer be specified in CMD\nif [ \"${_START_SH_EXECUTED}\" = \"1\" ]; then\n _log \"WARNING: start.sh is the default ENTRYPOINT, do not include it in CMD\"\n _log \"Executing the command:\" \"${cmd[@]}\"\n exec \"${cmd[@]}\"\nelse\n export _START_SH_EXECUTED=1\nfi\n\n\n# NOTE: This hook will run as the user the container was started with!\n# shellcheck source=images/docker-stacks-foundation/run-hooks.sh\nsource /usr/local/bin/run-hooks.sh /usr/local/bin/start-notebook.d\n\n# If the container started as the root user, then we have permission to refit\n# the jovyan user, and ensure file permissions, grant sudo rights, and such\n# things before we run the command passed to start.sh as the desired user\n# (NB_USER).\n#\nif [ \"$(id -u)\" == 0 ]; then\n # Environment variables:\n # - NB_USER: the desired username and associated home folder\n # - NB_UID: the desired user id\n # - NB_GID: a group id we want our user to belong to\n # - NB_GROUP: a group name we want for the group\n # - GRANT_SUDO: a boolean (\"1\" or \"yes\") to grant the user sudo rights\n # - CHOWN_HOME: a boolean (\"1\" or \"yes\") to chown the user's home folder\n # - CHOWN_EXTRA: a comma-separated list of paths to chown\n # - CHOWN_HOME_OPTS / CHOWN_EXTRA_OPTS: arguments to the chown commands\n\n # Refit the jovyan user to the desired user (NB_USER)\n if id jovyan &> /dev/null; then\n if ! usermod --home \"/home/${NB_USER}\" --login \"${NB_USER}\" jovyan 2>&1 | grep \"no changes\" > /dev/null; then\n _log \"Updated the jovyan user:\"\n _log \"- username: jovyan -> ${NB_USER}\"\n _log \"- home dir: /home/jovyan -> /home/${NB_USER}\"\n fi\n elif ! id -u \"${NB_USER}\" &> /dev/null; then\n _log \"ERROR: Neither the jovyan user nor '${NB_USER}' exists. This could be the result of stopping and starting, the container with a different NB_USER environment variable.\"\n exit 1\n fi\n # Ensure the desired user (NB_USER) gets its desired user id (NB_UID) and is\n # a member of the desired group (NB_GROUP, NB_GID)\n if [ \"${NB_UID}\" != \"$(id -u \"${NB_USER}\")\" ] || [ \"${NB_GID}\" != \"$(id -g \"${NB_USER}\")\" ]; then\n _log \"Update ${NB_USER}'s UID:GID to ${NB_UID}:${NB_GID}\"\n # Ensure the desired group's existence\n if [ \"${NB_GID}\" != \"$(id -g \"${NB_USER}\")\" ]; then\n groupadd --force --gid \"${NB_GID}\" --non-unique \"${NB_GROUP:-${NB_USER}}\"\n fi\n # Recreate the desired user as we want it\n userdel \"${NB_USER}\"\n useradd --no-log-init --home \"/home/${NB_USER}\" --shell /bin/bash --uid \"${NB_UID}\" --gid \"${NB_GID}\" --groups 100 \"${NB_USER}\"\n fi\n # Update the home directory if the desired user (NB_USER) is root and the\n # desired user id (NB_UID) is 0 and the desired group id (NB_GID) is 0.\n if [ \"${NB_USER}\" = \"root\" ] && [ \"${NB_UID}\" = \"$(id -u \"${NB_USER}\")\" ] && [ \"${NB_GID}\" = \"$(id -g \"${NB_USER}\")\" ]; then\n sed -i \"s|/root|/home/root|g\" /etc/passwd\n # Do not preserve ownership in rootless mode\n CP_OPTS=\"-a --no-preserve=ownership\"\n fi\n\n # Move or symlink the jovyan home directory to the desired user's home\n # directory if it doesn't already exist, and update the current working\n # directory to the new location if needed.\n if [[ \"${NB_USER}\" != \"jovyan\" ]]; then\n if [[ ! -e \"/home/${NB_USER}\" ]]; then\n _log \"Attempting to copy /home/jovyan to /home/${NB_USER}...\"\n mkdir \"/home/${NB_USER}\"\n # shellcheck disable=SC2086\n if cp ${CP_OPTS:--a} /home/jovyan/. \"/home/${NB_USER}/\"; then\n _log \"Success!\"\n else\n _log \"Failed to copy data from /home/jovyan to /home/${NB_USER}!\"\n _log \"Attempting to symlink /home/jovyan to /home/${NB_USER}...\"\n if ln -s /home/jovyan \"/home/${NB_USER}\"; then\n _log \"Success creating symlink!\"\n else\n _log \"ERROR: Failed copy data from /home/jovyan to /home/${NB_USER} or to create symlink!\"\n exit 1\n fi\n fi\n fi\n # Ensure the current working directory is updated to the new path\n if [[ \"${PWD}/\" == \"/home/jovyan/\"* ]]; then\n new_wd=\"/home/${NB_USER}/${PWD:13}\"\n _log \"Changing working directory to ${new_wd}\"\n cd \"${new_wd}\"\n fi\n fi\n\n # Optionally ensure the desired user gets filesystem ownership of its home\n # folder and/or additional folders\n if [[ \"${CHOWN_HOME}\" == \"1\" || \"${CHOWN_HOME}\" == \"yes\" ]]; then\n _log \"Ensuring /home/${NB_USER} is owned by ${NB_UID}:${NB_GID} ${CHOWN_HOME_OPTS:+(chown options: ${CHOWN_HOME_OPTS})}\"\n # shellcheck disable=SC2086\n chown ${CHOWN_HOME_OPTS} \"${NB_UID}:${NB_GID}\" \"/home/${NB_USER}\"\n fi\n if [ -n \"${CHOWN_EXTRA}\" ]; then\n for extra_dir in $(echo \"${CHOWN_EXTRA}\" | tr ',' ' '); do\n _log \"Ensuring ${extra_dir} is owned by ${NB_UID}:${NB_GID} ${CHOWN_EXTRA_OPTS:+(chown options: ${CHOWN_EXTRA_OPTS})}\"\n # shellcheck disable=SC2086\n chown ${CHOWN_EXTRA_OPTS} \"${NB_UID}:${NB_GID}\" \"${extra_dir}\"\n done\n fi\n\n # Prepend ${CONDA_DIR}/bin to sudo secure_path\n sed -r \"s#Defaults\\s+secure_path\\s*=\\s*\\\"?([^\\\"]+)\\\"?#Defaults secure_path=\\\"${CONDA_DIR}/bin:\\1\\\"#\" /etc/sudoers | grep secure_path > /etc/sudoers.d/path\n\n # Optionally grant passwordless sudo rights for the desired user\n if [[ \"${GRANT_SUDO}\" == \"1\" || \"${GRANT_SUDO}\" == \"yes\" ]]; then\n _log \"Granting ${NB_USER} passwordless sudo rights!\"\n echo \"${NB_USER} ALL=(ALL) NOPASSWD:ALL\" >> /etc/sudoers.d/added-by-start-script\n fi\n\n # NOTE: This hook is run as the root user!\n # shellcheck source=images/docker-stacks-foundation/run-hooks.sh\n source /usr/local/bin/run-hooks.sh /usr/local/bin/before-notebook.d\n unset_explicit_env_vars\n\n _log \"Running as ${NB_USER}:\" \"${cmd[@]}\"\n if [ \"${NB_USER}\" = \"root\" ] && [ \"${NB_UID}\" = \"$(id -u \"${NB_USER}\")\" ] && [ \"${NB_GID}\" = \"$(id -g \"${NB_USER}\")\" ]; then\n HOME=\"/home/root\" exec \"${cmd[@]}\"\n else\n exec sudo --preserve-env --set-home --user \"${NB_USER}\" \\\n LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}\" \\\n PATH=\"${PATH}\" \\\n PYTHONPATH=\"${PYTHONPATH:-}\" \\\n \"${cmd[@]}\"\n # Notes on how we ensure that the environment that this container is started\n # with is preserved (except vars listed in JUPYTER_ENV_VARS_TO_UNSET) when\n # we transition from running as root to running as NB_USER.\n #\n # - We use `sudo` to execute the command as NB_USER. What then\n # happens to the environment will be determined by configuration in\n # /etc/sudoers and /etc/sudoers.d/* as well as flags we pass to the sudo\n # command. The behavior can be inspected with `sudo -V` run as root.\n #\n # ref: `man sudo` https://linux.die.net/man/8/sudo\n # ref: `man sudoers` https://www.sudo.ws/docs/man/sudoers.man/\n #\n # - We use the `--preserve-env` flag to pass through most environment\n # variables, but understand that exceptions are caused by the sudoers\n # configuration: `env_delete` and `env_check`.\n #\n # - We use the `--set-home` flag to set the HOME variable appropriately.\n #\n # - To reduce the default list of variables deleted by sudo, we could have\n # used `env_delete` from /etc/sudoers. It has a higher priority than the\n # `--preserve-env` flag and the `env_keep` configuration.\n #\n # - We preserve LD_LIBRARY_PATH, PATH and PYTHONPATH explicitly. Note however that sudo\n # resolves `${cmd[@]}` using the \"secure_path\" variable we modified\n # above in /etc/sudoers.d/path. Thus PATH is irrelevant to how the above\n # sudo command resolves the path of `${cmd[@]}`. The PATH will be relevant\n # for resolving paths of any subprocesses spawned by `${cmd[@]}`.\n fi\n\n# The container didn't start as the root user, so we will have to act as the\n# user we started as.\nelse\n # Warn about misconfiguration of: granting sudo rights\n if [[ \"${GRANT_SUDO}\" == \"1\" || \"${GRANT_SUDO}\" == \"yes\" ]]; then\n _log \"WARNING: container must be started as root to grant sudo permissions!\"\n fi\n\n JOVYAN_UID=\"$(id -u jovyan 2>/dev/null)\" # The default UID for the jovyan user\n JOVYAN_GID=\"$(id -g jovyan 2>/dev/null)\" # The default GID for the jovyan user\n\n # Attempt to ensure the user uid we currently run as has a named entry in\n # the /etc/passwd file, as it avoids software crashing on hard assumptions\n # on such entry. Writing to the /etc/passwd was allowed for the root group\n # from the Dockerfile during the build.\n #\n # ref: https://github.com/jupyter/docker-stacks/issues/552\n if ! whoami &> /dev/null; then\n _log \"There is no entry in /etc/passwd for our UID=$(id -u). Attempting to fix...\"\n if [[ -w /etc/passwd ]]; then\n _log \"Renaming old jovyan user to nayvoj ($(id -u jovyan):$(id -g jovyan))\"\n\n # We cannot use \"sed --in-place\" since sed tries to create a temp file in\n # /etc/ and we may not have write access. Apply sed on our own temp file:\n sed --expression=\"s/^jovyan:/nayvoj:/\" /etc/passwd > /tmp/passwd\n echo \"${NB_USER}:x:$(id -u):$(id -g):,,,:/home/jovyan:/bin/bash\" >> /tmp/passwd\n cat /tmp/passwd > /etc/passwd\n rm /tmp/passwd\n\n _log \"Added new ${NB_USER} user ($(id -u):$(id -g)). Fixed UID!\"\n\n if [[ \"${NB_USER}\" != \"jovyan\" ]]; then\n _log \"WARNING: user is ${NB_USER} but home is /home/jovyan. You must run as root to rename the home directory!\"\n fi\n else\n _log \"WARNING: unable to fix missing /etc/passwd entry because we don't have write permission. Try setting gid=0 with \\\"--user=$(id -u):0\\\".\"\n fi\n fi\n\n # Warn about misconfiguration of: desired username, user id, or group id.\n # A misconfiguration occurs when the user modifies the default values of\n # NB_USER, NB_UID, or NB_GID, but we cannot update those values because we\n # are not root.\n if [[ \"${NB_USER}\" != \"jovyan\" && \"${NB_USER}\" != \"$(id -un)\" ]]; then\n _log \"WARNING: container must be started as root to change the desired user's name with NB_USER=\\\"${NB_USER}\\\"!\"\n fi\n if [[ \"${NB_UID}\" != \"${JOVYAN_UID}\" && \"${NB_UID}\" != \"$(id -u)\" ]]; then\n _log \"WARNING: container must be started as root to change the desired user's id with NB_UID=\\\"${NB_UID}\\\"!\"\n fi\n if [[ \"${NB_GID}\" != \"${JOVYAN_GID}\" && \"${NB_GID}\" != \"$(id -g)\" ]]; then\n _log \"WARNING: container must be started as root to change the desired user's group id with NB_GID=\\\"${NB_GID}\\\"!\"\n fi\n\n # Warn if the user isn't able to write files to ${HOME}\n if [[ ! -w /home/jovyan ]]; then\n _log \"WARNING: no write access to /home/jovyan. Try starting the container with group 'users' (100), e.g. using \\\"--group-add=users\\\".\"\n fi\n\n # NOTE: This hook is run as the user we started the container as!\n # shellcheck source=images/docker-stacks-foundation/run-hooks.sh\n source /usr/local/bin/run-hooks.sh /usr/local/bin/before-notebook.d\n unset_explicit_env_vars\n\n _log \"Executing the command:\" \"${cmd[@]}\"\n exec \"${cmd[@]}\"\nfi\n" }, { "path": "images/julia-notebook/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/julia-notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/minimal-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\n# Julia dependencies\n# install Julia packages in /opt/julia instead of ${HOME}\nENV JULIA_DEPOT_PATH=/opt/julia \\\n JULIA_PKGDIR=/opt/julia\n\n# Setup Julia\nRUN /opt/setup-scripts/setup_julia.py\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\n# hadolint ignore=DL3059\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\nUSER ${NB_UID}\n\n# Setup IJulia kernel & other packages\nRUN /opt/setup-scripts/setup-julia-packages.bash\n" }, { "path": "images/julia-notebook/README.md", "content": "# Jupyter Notebook Julia Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/julia-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-julia-notebook)\n" }, { "path": "images/minimal-notebook/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/minimal-notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/base-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\n# Install all OS dependencies for a fully functional Server\nRUN apt-get update --yes && \\\n apt-get install --yes --no-install-recommends \\\n # Common useful utilities\n curl \\\n git \\\n nano-tiny \\\n tzdata \\\n unzip \\\n vim-tiny \\\n # git-over-ssh\n openssh-client \\\n # `less` is needed to run help in R\n # see: https://github.com/jupyter/docker-stacks/issues/1588\n less \\\n # `nbconvert` dependencies\n # https://nbconvert.readthedocs.io/en/latest/install.html#installing-tex\n texlive-xetex \\\n texlive-fonts-recommended \\\n texlive-plain-generic \\\n # Enable clipboard on Linux host systems\n xclip && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# Create alternative for nano -> nano-tiny\nRUN update-alternatives --install /usr/bin/nano nano /bin/nano-tiny 10\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\n# hadolint ignore=DL3059\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\n# Switch back to jovyan to avoid accidental container runs as root\nUSER ${NB_UID}\n\n# Add an R mimetype option to specify how the plot returns from R to the browser\nCOPY --chown=${NB_UID}:${NB_GID} Rprofile.site /opt/conda/lib/R/etc/\n\n# Add setup scripts that may be used by downstream images or inherited images\nCOPY setup-scripts/ /opt/setup-scripts/\n" }, { "path": "images/minimal-notebook/README.md", "content": "# Minimal Jupyter Notebook Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/minimal-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-minimal-notebook)\n" }, { "path": "images/minimal-notebook/Rprofile.site", "content": "# Add R mimetype to specify how the plot returns from R to the browser.\n# https://notebook.community/andrie/jupyter-notebook-samples/Changing%20R%20plot%20options%20in%20Jupyter\n\noptions(jupyter.plot_mimetypes = c('text/plain', 'image/png', 'image/jpeg', 'image/svg+xml', 'application/pdf'))\n" }, { "path": "images/minimal-notebook/setup-scripts/activate_notebook_custom_env.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport json\nimport os\nimport sys\nfrom pathlib import Path\n\nenv_name = sys.argv[1]\nCONDA_DIR = os.environ[\"CONDA_DIR\"]\n\nfile = Path.home() / f\".local/share/jupyter/kernels/{env_name}/kernel.json\"\ncontent = json.loads(file.read_text())\ncontent[\"env\"] = {\n \"XML_CATALOG_FILES\": \"\",\n \"PATH\": f\"{CONDA_DIR}/envs/{env_name}/bin:$PATH\",\n \"CONDA_PREFIX\": f\"{CONDA_DIR}/envs/{env_name}\",\n \"CONDA_PROMPT_MODIFIER\": f\"({env_name}) \",\n \"CONDA_SHLVL\": \"2\",\n \"CONDA_DEFAULT_ENV\": env_name,\n \"CONDA_PREFIX_1\": CONDA_DIR,\n}\n\nfile.write_text(json.dumps(content, indent=1))\n" }, { "path": "images/minimal-notebook/setup-scripts/setup-julia-packages.bash", "content": "#!/bin/bash\nset -exuo pipefail\n# Requirements:\n# - Run as a non-root user\n# - The JULIA_PKGDIR environment variable is set\n# - Julia is already set up, with the setup_julia.py command\n\n\n# If we don't specify what CPUs the precompilation should be done for, it's\n# *only* done for the target of the host doing the compilation. When the\n# container runs on a host that's the same architecture, but a *different*\n# generation of CPU than what the build host was, the precompilation is useless\n# and Julia takes a long long time to start up. This specific multitarget comes\n# from https://github.com/JuliaCI/julia-buildkite/blob/9f354745a1f2bf31a5952462aa1ff2d869507cb8/utilities/build_envs.sh#L20-L82,\n# and may need to be updated as new CPU generations come out.\n# If the architecture the container runs on is different,\n# precompilation may still have to be re-done on first startup - but this\n# *should* catch most of the issues. See\n# https://github.com/jupyter/docker-stacks/issues/2015 for more information\nif [ \"$(uname -m)\" == \"x86_64\" ]; then\n # See https://github.com/JuliaCI/julia-buildkite/blob/9f354745a1f2bf31a5952462aa1ff2d869507cb8/utilities/build_envs.sh#L23\n # for an explanation of these options\n export JULIA_CPU_TARGET=\"generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1)\"\nelif [ \"$(uname -m)\" == \"aarch64\" ]; then\n # See https://github.com/JuliaCI/julia-buildkite/blob/9f354745a1f2bf31a5952462aa1ff2d869507cb8/utilities/build_envs.sh#L56\n # for an explanation of these options\n export JULIA_CPU_TARGET=\"generic;cortex-a57;thunderx2t99;carmel,clone_all;apple-m1,base(3);neoverse-512tvb,base(3)\"\nfi\n\n# Install base Julia packages\njulia -e '\nimport Pkg;\nPkg.update();\nPkg.add([\n \"HDF5\",\n \"IJulia\",\n \"Pluto\"\n]);\nPkg.precompile();\n'\n\n# Move the kernelspec out of ${HOME} to the system share location.\n# Avoids problems with runtime UID change not taking effect properly\n# on the .local folder in the jovyan home dir.\nmv \"${HOME}/.local/share/jupyter/kernels/julia\"* \"${CONDA_DIR}/share/jupyter/kernels/\"\nchmod -R go+rx \"${CONDA_DIR}/share/jupyter\"\nrm -rf \"${HOME}/.local\"\nfix-permissions \"${JULIA_PKGDIR}\" \"${CONDA_DIR}/share/jupyter\"\n\n# Install jupyter-pluto-proxy to get Pluto to work on JupyterHub\nmamba install --yes \\\n 'jupyter-pluto-proxy' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "images/minimal-notebook/setup-scripts/setup_julia.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# Requirements:\n# - Run as the root user\n# - The JULIA_PKGDIR environment variable is set\n\nimport logging\nimport os\nimport platform\nimport shutil\nimport subprocess\nfrom pathlib import Path\n\nimport requests\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef unify_aarch64(platform: str) -> str:\n \"\"\"\n Renames arm64->aarch64 to support local builds on aarch64 Macs\n \"\"\"\n return {\"arm64\": \"aarch64\"}.get(platform, platform)\n\n\ndef get_latest_julia_url() -> tuple[str, str]:\n \"\"\"\n Get the last stable version of Julia\n Based on: https://github.com/JuliaLang/www.julialang.org/issues/878#issuecomment-749234813\n \"\"\"\n LOGGER.info(\"Downloading Julia versions information\")\n versions = requests.get(\n \"https://julialang-s3.julialang.org/bin/versions.json\"\n ).json()\n stable_versions = {k: v for k, v in versions.items() if v[\"stable\"]}\n # Compare versions semantically\n latest_stable_version = max(\n stable_versions, key=lambda ver: [int(sub_ver) for sub_ver in ver.split(\".\")]\n )\n latest_version_files = stable_versions[latest_stable_version][\"files\"]\n triplet = unify_aarch64(platform.machine()) + \"-linux-gnu\"\n file_info = [vf for vf in latest_version_files if vf[\"triplet\"] == triplet][0]\n LOGGER.info(f\"Latest version: {file_info['version']} url: {file_info['url']}\")\n return file_info[\"url\"], file_info[\"version\"]\n\n\ndef download_julia(julia_url: str) -> None:\n \"\"\"\n Downloads and unpacks julia\n The resulting julia directory is \"/opt/julia-VERSION/\"\n \"\"\"\n LOGGER.info(\"Downloading and unpacking Julia\")\n tmp_file = Path(\"/tmp/julia.tar.gz\")\n subprocess.check_call(\n [\"curl\", \"--progress-bar\", \"--location\", \"--output\", tmp_file, julia_url]\n )\n shutil.unpack_archive(tmp_file, \"/opt/\")\n tmp_file.unlink()\n\n\ndef configure_julia(julia_version: str) -> None:\n \"\"\"\n Creates /usr/local/bin/julia symlink\n Make Julia aware of conda libraries\n Creates a directory for Julia user libraries\n \"\"\"\n LOGGER.info(\"Configuring Julia\")\n # Link Julia installed version to /usr/local/bin, so julia launches it\n subprocess.check_call(\n [\"ln\", \"-fs\", f\"/opt/julia-{julia_version}/bin/julia\", \"/usr/local/bin/julia\"]\n )\n\n # Tell Julia where conda libraries are\n Path(\"/etc/julia\").mkdir()\n Path(\"/etc/julia/juliarc.jl\").write_text(\n f'push!(Libdl.DL_LOAD_PATH, \"{os.environ[\"CONDA_DIR\"]}/lib\")\\n'\n )\n\n # Create JULIA_PKGDIR, where user libraries are installed\n JULIA_PKGDIR = Path(os.environ[\"JULIA_PKGDIR\"])\n JULIA_PKGDIR.mkdir()\n subprocess.check_call([\"chown\", os.environ[\"NB_USER\"], JULIA_PKGDIR])\n subprocess.check_call([\"fix-permissions\", JULIA_PKGDIR])\n\n\nif __name__ == \"__main__\":\n logging.basicConfig(level=logging.INFO)\n\n julia_url, julia_version = get_latest_julia_url()\n download_julia(julia_url=julia_url)\n configure_julia(julia_version=julia_version)\n" }, { "path": "images/pyspark-notebook/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/pyspark-notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\n# Spark dependencies\n# Default values can be overridden at build time\n# (ARGS are in lowercase to distinguish them from ENV)\nARG openjdk_version=\"17\"\n\nRUN apt-get update --yes && \\\n apt-get install --yes --no-install-recommends \\\n \"openjdk-${openjdk_version}-jre-headless\" \\\n ca-certificates-java && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# If spark_version is not set, latest Spark will be installed\nARG spark_version\nARG hadoop_version=\"3\"\n# If scala_version is not set, Spark without Scala will be installed\nARG scala_version\n# URL to use for Spark downloads\n# You need to use https://archive.apache.org/dist/spark/ website if you want to download old Spark versions\n# But it seems to be slower, that's why we use the recommended site for download\nARG spark_download_url=\"https://dlcdn.apache.org/spark/\"\n\nENV SPARK_HOME=/usr/local/spark\nENV PATH=\"${PATH}:${SPARK_HOME}/bin\"\nENV SPARK_OPTS=\"--driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info\"\n\nCOPY setup_spark.py /opt/setup-scripts/\n\n# Setup Spark\nRUN /opt/setup-scripts/setup_spark.py \\\n --spark-version=\"${spark_version}\" \\\n --hadoop-version=\"${hadoop_version}\" \\\n --scala-version=\"${scala_version}\" \\\n --spark-download-url=\"${spark_download_url}\"\n\n# Configure IPython system-wide\nCOPY ipython_kernel_config.py \"/etc/ipython/\"\nRUN fix-permissions \"/etc/ipython/\"\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\n# hadolint ignore=DL3059\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\nUSER ${NB_UID}\n\n# Install pyarrow\n# NOTE: It's important to ensure compatibility between Pandas versions.\n# The pandas version in this Dockerfile should match the version\n# on which the Pandas API for Spark is built.\n# To find the right version, check the pandas version being installed here:\n# https://github.com/apache/spark/blob//dev/infra/Dockerfile\nRUN mamba install --yes \\\n 'grpcio-status' \\\n 'grpcio' \\\n 'pandas=2.2.3' \\\n 'pyarrow' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\nWORKDIR \"${HOME}\"\nEXPOSE 4040\n" }, { "path": "images/pyspark-notebook/README.md", "content": "# Jupyter Notebook Python, Spark Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/pyspark-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-pyspark-notebook)\n- [Image Specifics :: Apache Spark](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/specifics.html#apache-spark)\n" }, { "path": "images/pyspark-notebook/ipython_kernel_config.py", "content": "# Configuration file for ipython-kernel.\n# See \n\n# With IPython >= 6.0.0, all outputs to stdout/stderr are captured.\n# It is the case for subprocesses and output of compiled libraries like Spark.\n# Those logs now both head to notebook logs and in notebooks outputs.\n# Logs are particularly verbose with Spark, that is why we turn them off through this flag.\n# \n\n# Attempt to capture and forward low-level output, e.g. produced by Extension libraries.\n# Default: True\n# type: ignore\nc.IPKernelApp.capture_fd_output = False # noqa: F821\n" }, { "path": "images/pyspark-notebook/setup_spark.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# Requirements:\n# - Run as the root user\n# - Required env variable: SPARK_HOME\n\nimport argparse\nimport logging\nimport os\nimport re\nimport subprocess\nfrom pathlib import Path\n\nimport requests\nfrom bs4 import BeautifulSoup\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef get_all_refs(url: str) -> list[str]:\n \"\"\"\n Get all the references for a given webpage\n \"\"\"\n resp = requests.get(url)\n soup = BeautifulSoup(resp.text, \"html.parser\")\n return [a[\"href\"] for a in soup.find_all(\"a\", href=True)]\n\n\ndef get_latest_spark_version() -> str:\n \"\"\"\n Returns the last version of Spark using spark archive\n \"\"\"\n LOGGER.info(\"Downloading Spark versions information\")\n all_refs = get_all_refs(\"https://archive.apache.org/dist/spark/\")\n LOGGER.info(f\"All refs: {all_refs}\")\n pattern = re.compile(r\"^spark-(\\d+\\.\\d+\\.\\d+)/$\")\n versions = [match.group(1) for ref in all_refs if (match := pattern.match(ref))]\n LOGGER.info(f\"Available versions: {versions}\")\n\n # Compare versions semantically\n def version_array(ver: str) -> tuple[int, int, int, str]:\n # 3.5.3 -> [3, 5, 3, \"\"]\n # 4.0.0-preview2 -> [4, 0, 0, \"preview2\"]\n arr = ver.split(\".\")\n assert len(arr) == 3, arr\n major, minor = int(arr[0]), int(arr[1])\n patch, _, preview = arr[2].partition(\"-\")\n return (major, minor, int(patch), preview)\n\n latest_version = max(versions, key=lambda ver: version_array(ver))\n LOGGER.info(f\"Latest version: {latest_version}\")\n return latest_version\n\n\ndef download_spark(\n *,\n spark_version: str,\n hadoop_version: str,\n scala_version: str,\n spark_download_url: Path,\n) -> str:\n \"\"\"\n Downloads and unpacks spark\n The resulting spark directory name is returned\n \"\"\"\n LOGGER.info(\"Downloading and unpacking Spark\")\n spark_dir_name = f\"spark-{spark_version}-bin-hadoop{hadoop_version}\"\n if scala_version:\n spark_dir_name += f\"-scala{scala_version}\"\n LOGGER.info(f\"Spark directory name: {spark_dir_name}\")\n spark_url = spark_download_url / f\"spark-{spark_version}\" / f\"{spark_dir_name}.tgz\"\n LOGGER.info(f\"Spark download URL: {spark_url}\")\n\n tmp_file = Path(\"/tmp/spark.tar.gz\")\n subprocess.check_call(\n [\"curl\", \"--progress-bar\", \"--location\", \"--output\", tmp_file, spark_url]\n )\n subprocess.check_call(\n [\n \"tar\",\n \"xzf\",\n tmp_file,\n \"-C\",\n \"/usr/local\",\n \"--owner\",\n \"root\",\n \"--group\",\n \"root\",\n \"--no-same-owner\",\n ]\n )\n tmp_file.unlink()\n return spark_dir_name\n\n\ndef configure_spark(spark_dir_name: str, spark_home: Path) -> None:\n \"\"\"\n Creates a ${SPARK_HOME} symlink to a versioned spark directory\n Creates a 10spark-config.sh symlink to source PYTHONPATH automatically\n \"\"\"\n LOGGER.info(\"Configuring Spark\")\n subprocess.check_call([\"ln\", \"-s\", f\"/usr/local/{spark_dir_name}\", spark_home])\n\n # Add a link in the before_notebook hook in order to source PYTHONPATH automatically\n CONFIG_SCRIPT = \"/usr/local/bin/before-notebook.d/10spark-config.sh\"\n subprocess.check_call(\n [\"ln\", \"-s\", spark_home / \"sbin/spark-config.sh\", CONFIG_SCRIPT]\n )\n\n\nif __name__ == \"__main__\":\n logging.basicConfig(level=logging.INFO)\n\n arg_parser = argparse.ArgumentParser()\n arg_parser.add_argument(\"--spark-version\", required=True)\n arg_parser.add_argument(\"--hadoop-version\", required=True)\n arg_parser.add_argument(\"--scala-version\", required=True)\n arg_parser.add_argument(\"--spark-download-url\", type=Path, required=True)\n args = arg_parser.parse_args()\n\n args.spark_version = args.spark_version or get_latest_spark_version()\n\n spark_dir_name = download_spark(\n spark_version=args.spark_version,\n hadoop_version=args.hadoop_version,\n scala_version=args.scala_version,\n spark_download_url=args.spark_download_url,\n )\n configure_spark(\n spark_dir_name=spark_dir_name, spark_home=Path(os.environ[\"SPARK_HOME\"])\n )\n" }, { "path": "images/pytorch-notebook/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/pytorch-notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\n# Install PyTorch with pip (https://pytorch.org/get-started/locally/)\n# hadolint ignore=DL3013\nRUN pip install --no-cache-dir --index-url 'https://download.pytorch.org/whl/cpu' \\\n 'torch' \\\n 'torchaudio' \\\n 'torchvision' && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "images/pytorch-notebook/README.md", "content": "# Jupyter Notebook PyTorch Deep Learning Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/pytorch-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-pytorch-notebook)\n" }, { "path": "images/pytorch-notebook/cuda12/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\n# Install PyTorch with pip (https://pytorch.org/get-started/locally/)\n# hadolint ignore=DL3013\nRUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu128' \\\n 'torch' \\\n 'torchaudio' \\\n 'torchvision' && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html#dockerfiles\nENV NVIDIA_VISIBLE_DEVICES=\"all\" \\\n NVIDIA_DRIVER_CAPABILITIES=\"compute,utility\"\n\n# Puts the nvidia-smi binary (system management interface) on path\n# with associated library files to execute it\nENV PATH=\"${PATH}:/usr/local/nvidia/bin\" \\\n LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}:/usr/local/nvidia/lib64\"\n" }, { "path": "images/pytorch-notebook/cuda13/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\n# Install PyTorch with pip (https://pytorch.org/get-started/locally/)\n# hadolint ignore=DL3013\nRUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu130' \\\n 'torch' \\\n 'torchaudio' \\\n 'torchvision' && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html#dockerfiles\nENV NVIDIA_VISIBLE_DEVICES=\"all\" \\\n NVIDIA_DRIVER_CAPABILITIES=\"compute,utility\"\n\n# Puts the nvidia-smi binary (system management interface) on path\n# with associated library files to execute it\nENV PATH=\"${PATH}:/usr/local/nvidia/bin\" \\\n LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}:/usr/local/nvidia/lib64\"\n" }, { "path": "images/r-notebook/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/r-notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/minimal-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\n# R pre-requisites\nRUN apt-get update --yes && \\\n apt-get install --yes --no-install-recommends \\\n fonts-dejavu \\\n unixodbc \\\n unixodbc-dev \\\n r-cran-rodbc \\\n gfortran \\\n gcc && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\nUSER ${NB_UID}\n\n# R packages including IRKernel which gets installed globally.\n# r-e1071: dependency of the caret R package\nRUN mamba install --yes \\\n 'r-base' \\\n 'r-caret' \\\n 'r-crayon' \\\n 'r-devtools' \\\n 'r-e1071' \\\n 'r-forecast' \\\n 'r-hexbin' \\\n 'r-htmltools' \\\n 'r-htmlwidgets' \\\n 'r-irkernel' \\\n 'r-nycflights13' \\\n 'r-randomforest' \\\n 'r-rcurl' \\\n 'r-rmarkdown' \\\n 'r-rodbc' \\\n 'r-rsqlite' \\\n 'r-shiny' \\\n 'r-tidymodels' \\\n 'r-tidyverse' \\\n 'unixodbc' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n" }, { "path": "images/r-notebook/README.md", "content": "# Jupyter Notebook R Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/r-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-r-notebook)\n" }, { "path": "images/scipy-notebook/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/scipy-notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/minimal-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\nUSER root\n\nRUN apt-get update --yes && \\\n apt-get install --yes --no-install-recommends \\\n # for cython: https://cython.readthedocs.io/en/latest/src/quickstart/install.html\n build-essential \\\n # for latex labels\n cm-super \\\n dvipng \\\n # for matplotlib anim\n ffmpeg && \\\n apt-get clean && rm -rf /var/lib/apt/lists/*\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\nUSER ${NB_UID}\n\n# Install Python 3 packages\nRUN mamba install --yes \\\n 'altair' \\\n 'beautifulsoup4' \\\n 'bokeh' \\\n 'bottleneck' \\\n 'cloudpickle' \\\n 'conda-forge::blas=*=openblas' \\\n 'cython' \\\n 'dask' \\\n 'dill' \\\n 'h5py' \\\n 'ipympl' \\\n 'ipywidgets' \\\n 'jupyterlab-git' \\\n 'matplotlib-base' \\\n 'numba' \\\n 'numexpr' \\\n 'openpyxl' \\\n 'pandas' \\\n 'patsy' \\\n 'protobuf' \\\n 'pytables' \\\n 'scikit-image' \\\n 'scikit-learn' \\\n 'scipy' \\\n 'seaborn' \\\n 'sqlalchemy' \\\n 'statsmodels' \\\n 'sympy' \\\n 'widgetsnbextension' \\\n 'xlrd' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# Import matplotlib the first time to build the font cache\nRUN MPLBACKEND=Agg python -c \"import matplotlib.pyplot\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# macOS Rosetta virtualization creates junk directory which gets owned by root further up.\n# It'll get re-created, but as USER runner after the next directive so hopefully should not cause permission issues.\n#\n# More info: https://github.com/jupyter/docker-stacks/issues/2296\nRUN rm -rf \"/home/${NB_USER}/.cache/\"\n\nUSER ${NB_UID}\n\nWORKDIR \"${HOME}\"\n" }, { "path": "images/scipy-notebook/README.md", "content": "# Jupyter Notebook Scientific Python Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/scipy-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-scipy-notebook)\n" }, { "path": "images/tensorflow-notebook/.dockerignore", "content": "# Documentation\nREADME.md\n" }, { "path": "images/tensorflow-notebook/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\n# Pin protobuf version for tensorflow 2.20 to avoid user warning.\n# The fix can be removed once tensorflow is built with newer version of protobuf:\n# https://github.com/tensorflow/tensorflow/issues/98980\nRUN mamba install --yes \\\n 'jupyter-server-proxy' \\\n 'protobuf>=5.28.3,<6' && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# Install tensorflow with pip, on x86_64 tensorflow-cpu\nRUN [[ $(uname -m) = x86_64 ]] && TF_POSTFIX=\"-cpu\" || TF_POSTFIX=\"\" && \\\n pip install --no-cache-dir \\\n \"tensorflow${TF_POSTFIX}\" && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\nCOPY --chown=\"${NB_UID}:${NB_GID}\" cuda/20tensorboard-proxy-env.sh /usr/local/bin/before-notebook.d/\n" }, { "path": "images/tensorflow-notebook/README.md", "content": "# Jupyter Notebook TensorFlow Deep Learning Stack\n\nGitHub Actions in the project builds and pushes this image to the Registry.\n\nPlease visit the project documentation site for help to use and contribute to this image and others.\n\n- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html)\n- [Selecting an Image :: Core Stacks :: jupyter/tensorflow-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-tensorflow-notebook)\n- [Image Specifics :: Tensorflow](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/specifics.html#tensorflow)\n" }, { "path": "images/tensorflow-notebook/cuda/20tensorboard-proxy-env.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nset -e\n\n# Initialize the TENSORBOARD_PROXY_URL with the appropriate path\n# to use jupyter-server-proxy.\n\nexport TENSORBOARD_PROXY_URL=\"${JUPYTERHUB_SERVICE_PREFIX:-/}proxy/%PORT%/\"\n" }, { "path": "images/tensorflow-notebook/cuda/Dockerfile", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nARG REGISTRY=quay.io\nARG OWNER=jupyter\nARG BASE_IMAGE=$REGISTRY/$OWNER/scipy-notebook\nFROM $BASE_IMAGE\n\nLABEL maintainer=\"Jupyter Project \"\n\n# Fix: https://github.com/hadolint/hadolint/wiki/DL4006\n# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014\nSHELL [\"/bin/bash\", \"-o\", \"pipefail\", \"-c\"]\n\n# Pin protobuf version for tensorflow 2.20 to avoid user warning.\n# The fix can be removed once tensorflow is built with newer version of protobuf:\n# https://github.com/tensorflow/tensorflow/issues/98980\nRUN mamba install --yes \\\n 'jupyter-server-proxy' \\\n \"protobuf>=5.28.3,<6\" && \\\n mamba clean --all -f -y && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\n# Install TensorFlow, CUDA and cuDNN with pip\nRUN pip install --no-cache-dir \\\n 'tensorflow[and-cuda]' && \\\n fix-permissions \"${CONDA_DIR}\" && \\\n fix-permissions \"/home/${NB_USER}\"\n\nCOPY --chown=\"${NB_UID}:${NB_GID}\" 20tensorboard-proxy-env.sh /usr/local/bin/before-notebook.d/\n\n# workaround for https://github.com/tensorflow/tensorflow/issues/63362\nRUN mkdir -p \"${CONDA_DIR}/etc/conda/activate.d/\" && \\\n fix-permissions \"${CONDA_DIR}\"\n\nCOPY --chown=\"${NB_UID}:${NB_GID}\" nvidia-lib-dirs.sh \"${CONDA_DIR}/etc/conda/activate.d/\"\n\n# https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html#dockerfiles\nENV NVIDIA_VISIBLE_DEVICES=\"all\" \\\n NVIDIA_DRIVER_CAPABILITIES=\"compute,utility\"\n\n# Puts the nvidia-smi binary (system management interface) on path\n# with associated library files to execute it\nENV PATH=\"${PATH}:/usr/local/nvidia/bin\" \\\n LD_LIBRARY_PATH=\"${LD_LIBRARY_PATH}:/usr/local/nvidia/lib64\"\n" }, { "path": "images/tensorflow-notebook/cuda/nvidia-lib-dirs.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# This adds NVIDIA Python package libraries to the LD_LIBRARY_PATH.\n# Workaround for https://github.com/tensorflow/tensorflow/issues/63362\nNVIDIA_DIR=$(dirname \"$(python -c 'import nvidia;print(nvidia.__file__)')\")\nLD_LIBRARY_PATH=$(echo \"${NVIDIA_DIR}\"/*/lib/ | sed -r 's/\\s+/:/g')${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}\nexport LD_LIBRARY_PATH\n" }, { "path": "mypy.ini", "content": "# Mypy is an optional static type checker for Python that aims to combine\n# the benefits of dynamic (or \"duck\") typing and static typing.\n#\n# Documentation: https://www.mypy-lang.org\n# Project: https://github.com/python/mypy\n# Config reference: https://mypy.readthedocs.io/en/stable/config_file.html\n#\n# We use mypy as part of pre-commit checks\n\n[mypy]\npython_version = 3.12\nfollow_imports = error\nstrict = True\nno_incremental = True\n# This allows us to use pytest decorators, which are not typed yet\ndisallow_untyped_decorators = False\n\n# These sections allow us to ignore mypy errors for packages\n# which are not (hopefully yet) statically typed\n\n[mypy-Cython.*]\nignore_missing_imports = True\n\n[mypy-docker.*]\nignore_missing_imports = True\n\n[mypy-matplotlib.*]\nignore_missing_imports = True\n\n[mypy-pandas.*]\nignore_missing_imports = True\n\n[mypy-plumbum.*]\nignore_missing_imports = True\n\n[mypy-pyspark.*]\nignore_missing_imports = True\n\n[mypy-setuptools.*]\nignore_missing_imports = True\n\n[mypy-tensorflow.*]\nignore_missing_imports = True\n\n[mypy-torch.*]\nignore_missing_imports = True\n" }, { "path": "requirements-dev.txt", "content": "docker\nplumbum\npre-commit\npytest\npytest-rerunfailures\n# `pytest-xdist` is a plugin that provides the `--numprocesses` flag,\n# allowing us to run `pytest` tests in parallel\npytest-xdist\npython-dateutil\nrequests\ntabulate\ntenacity\n" }, { "path": "tagging/README.md", "content": "# Docker stacks tagging and manifest creation\n\nPlease, refer to the [tagging section of documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/maintaining/tagging.html) to see how tags and manifests are created.\n" }, { "path": "tagging/__init__.py", "content": "" }, { "path": "tagging/apps/__init__.py", "content": "" }, { "path": "tagging/apps/apply_tags.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\n\nimport plumbum\n\nfrom tagging.apps.common_cli_arguments import common_arguments_parser\nfrom tagging.apps.config import Config\nfrom tagging.utils.get_prefix import get_file_prefix_for_platform\n\ndocker = plumbum.local[\"docker\"]\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef apply_tags(config: Config) -> None:\n LOGGER.info(f\"Tagging image: {config.image}\")\n\n file_prefix = get_file_prefix_for_platform(\n platform=config.platform, variant=config.variant\n )\n filename = f\"{file_prefix}-{config.image}.txt\"\n tags = (config.tags_dir / filename).read_text().splitlines()\n\n for tag in tags:\n LOGGER.info(f\"Applying tag: {tag}\")\n docker[\"tag\", config.full_image(), tag] & plumbum.FG\n\n LOGGER.info(f\"All tags applied to image: {config.image}\")\n\n\nif __name__ == \"__main__\":\n logging.basicConfig(level=logging.INFO)\n\n config = common_arguments_parser(\n registry=True,\n owner=True,\n image=True,\n variant=True,\n platform=True,\n tags_dir=True,\n )\n apply_tags(config)\n" }, { "path": "tagging/apps/common_cli_arguments.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport argparse\nfrom pathlib import Path\n\nfrom tagging.apps.config import Config\nfrom tagging.utils.get_platform import unify_aarch64\n\n\ndef common_arguments_parser(\n *,\n registry: bool = False,\n owner: bool = False,\n image: bool = False,\n variant: bool = False,\n platform: bool = False,\n tags_dir: bool = False,\n hist_lines_dir: bool = False,\n manifests_dir: bool = False,\n repository: bool = False,\n) -> Config:\n \"\"\"Add common CLI arguments to parser\"\"\"\n\n parser = argparse.ArgumentParser()\n if registry:\n parser.add_argument(\n \"--registry\",\n required=True,\n choices=[\"docker.io\", \"quay.io\"],\n help=\"Image registry\",\n )\n if owner:\n parser.add_argument(\n \"--owner\",\n required=True,\n help=\"Owner of the image\",\n )\n if image:\n parser.add_argument(\n \"--image\",\n required=True,\n help=\"Short image name\",\n )\n if variant:\n parser.add_argument(\n \"--variant\",\n required=True,\n help=\"Variant tag prefix\",\n )\n if platform:\n parser.add_argument(\n \"--platform\",\n required=True,\n type=str,\n choices=[\"x86_64\", \"aarch64\", \"arm64\"],\n help=\"Image platform\",\n )\n if tags_dir:\n parser.add_argument(\n \"--tags-dir\",\n required=True,\n type=Path,\n help=\"Directory for tags file\",\n )\n if hist_lines_dir:\n parser.add_argument(\n \"--hist-lines-dir\",\n required=True,\n type=Path,\n help=\"Directory for hist_lines file\",\n )\n if manifests_dir:\n parser.add_argument(\n \"--manifests-dir\",\n required=True,\n type=Path,\n help=\"Directory for manifests file\",\n )\n if repository:\n parser.add_argument(\n \"--repository\",\n required=True,\n help=\"Repository name on GitHub\",\n )\n args = parser.parse_args()\n if platform:\n args.platform = unify_aarch64(args.platform)\n\n return Config(**vars(args))\n" }, { "path": "tagging/apps/config.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom dataclasses import dataclass\nfrom pathlib import Path\n\n\n@dataclass(frozen=True)\nclass Config:\n registry: str = \"\"\n owner: str = \"\"\n image: str = \"\"\n variant: str = \"\"\n platform: str = \"\"\n\n tags_dir: Path = Path()\n hist_lines_dir: Path = Path()\n manifests_dir: Path = Path()\n\n repository: str = \"\"\n\n def full_image(self) -> str:\n return f\"{self.registry}/{self.owner}/{self.image}\"\n" }, { "path": "tagging/apps/merge_tags.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nimport os\n\nimport plumbum\nfrom tenacity import ( # type: ignore\n RetryError,\n retry,\n stop_after_attempt,\n wait_exponential,\n)\n\nfrom tagging.apps.common_cli_arguments import common_arguments_parser\nfrom tagging.apps.config import Config\nfrom tagging.utils.get_platform import ALL_PLATFORMS\nfrom tagging.utils.get_prefix import get_file_prefix_for_platform\n\ndocker = plumbum.local[\"docker\"]\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef read_local_tags_from_files(config: Config) -> set[str]:\n LOGGER.info(f\"Read tags from file(s) for image: {config.image}\")\n\n merged_local_tags = set()\n for platform in ALL_PLATFORMS:\n LOGGER.info(f\"Reading tags for platform: {platform}\")\n\n file_prefix = get_file_prefix_for_platform(\n platform=platform, variant=config.variant\n )\n filename = f\"{file_prefix}-{config.image}.txt\"\n path = config.tags_dir / filename\n if not path.exists():\n LOGGER.info(f\"Tag file: {path} doesn't exist\")\n continue\n\n LOGGER.info(f\"Tag file: {path} found\")\n for tag in path.read_text().splitlines():\n merged_local_tags.add(tag.replace(platform + \"-\", \"\"))\n\n LOGGER.info(f\"Tags read for image: {config.image}\")\n return merged_local_tags\n\n\n@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4))\ndef inspect_manifest(tag: str) -> None:\n LOGGER.info(f\"Inspecting manifest for tag: {tag}\")\n docker[\"buildx\", \"imagetools\", \"inspect\", tag] & plumbum.FG\n LOGGER.info(f\"Manifest {tag} exists\")\n\n\ndef find_platform_tags(merged_tag: str) -> list[str]:\n platform_tags = []\n\n for platform in ALL_PLATFORMS:\n platform_tag = merged_tag.replace(\":\", f\":{platform}-\")\n LOGGER.warning(f\"Trying to inspect: {platform_tag} in the registry\")\n try:\n inspect_manifest(platform_tag)\n platform_tags.append(platform_tag)\n LOGGER.info(f\"Tag {platform_tag} found successfully\")\n except RetryError:\n LOGGER.warning(f\"Manifest for tag {platform_tag} doesn't exist\")\n\n return platform_tags\n\n\ndef merge_tags(merged_tag: str, push_to_registry: bool) -> None:\n LOGGER.info(f\"Trying to merge tag: {merged_tag}\")\n\n platform_tags = find_platform_tags(merged_tag)\n if not platform_tags:\n assert not push_to_registry, (\n f\"No platform tags found for merged tag: {merged_tag}, \"\n \"and push to registry is enabled. \"\n \"Cannot create a manifest for a non-existing image.\"\n )\n LOGGER.info(\n f\"Not running merge for tag: {merged_tag} as no platform tags found\"\n )\n return\n\n args = [\n \"buildx\",\n \"imagetools\",\n \"create\",\n *platform_tags,\n \"--tag\",\n merged_tag,\n ]\n if not push_to_registry:\n args.append(\"--dry-run\")\n\n LOGGER.info(f\"Running command: {' '.join(args)}\")\n docker[args] & plumbum.FG\n if push_to_registry:\n LOGGER.info(f\"Pushed merged tag: {merged_tag}\")\n else:\n LOGGER.info(f\"Skipped push for tag: {merged_tag}\")\n\n\nif __name__ == \"__main__\":\n logging.basicConfig(level=logging.INFO)\n\n config = common_arguments_parser(image=True, variant=True, tags_dir=True)\n push_to_registry = os.environ.get(\"PUSH_TO_REGISTRY\", \"false\").lower() == \"true\"\n\n LOGGER.info(f\"Merging tags for image: {config.image}\")\n\n merged_local_tags = read_local_tags_from_files(config)\n for tag in merged_local_tags:\n merge_tags(tag, push_to_registry)\n\n LOGGER.info(f\"Successfully merged tags for image: {config.image}\")\n" }, { "path": "tagging/apps/write_manifest.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport datetime\nimport logging\n\nfrom docker.models.containers import Container\n\nfrom tagging.apps.common_cli_arguments import common_arguments_parser\nfrom tagging.apps.config import Config\nfrom tagging.hierarchy.get_manifests import get_manifests\nfrom tagging.hierarchy.get_taggers import get_taggers\nfrom tagging.manifests.build_info import BuildInfoConfig, build_info_manifest\nfrom tagging.utils.docker_runner import DockerRunner\nfrom tagging.utils.get_prefix import get_file_prefix, get_tag_prefix\nfrom tagging.utils.git_helper import GitHelper\n\nLOGGER = logging.getLogger(__name__)\n\n# We use a manifest creation timestamp, which happens right after a build\nBUILD_TIMESTAMP = datetime.datetime.now(datetime.UTC).isoformat()[:-13] + \"Z\"\nMARKDOWN_LINE_BREAK = \"
\"\n\n\ndef get_build_history_line(config: Config, container: Container, filename: str) -> str:\n LOGGER.info(f\"Calculating build history line for image: {config.image}\")\n\n taggers = get_taggers(config.image)\n tags_prefix = get_tag_prefix(config.variant)\n all_tags = [tags_prefix + \"-\" + tagger(container) for tagger in taggers]\n\n date_column = f\"`{BUILD_TIMESTAMP}`\"\n image_column = MARKDOWN_LINE_BREAK.join(\n f\"`{config.full_image()}:{tag_value}`\" for tag_value in all_tags\n )\n commit_hash = GitHelper.commit_hash()\n links_column = MARKDOWN_LINE_BREAK.join(\n [\n f\"[Git diff](https://github.com/{config.repository}/commit/{commit_hash})\",\n f\"[Dockerfile](https://github.com/{config.repository}/blob/{commit_hash}/images/{config.image}/Dockerfile)\",\n f\"[Build manifest](./{filename})\",\n ]\n )\n build_history_line = f\"| {date_column} | {image_column} | {links_column} |\"\n\n LOGGER.info(f\"Build history line calculated for image: {config.image}\")\n return build_history_line\n\n\ndef write_build_history_line(\n config: Config, container: Container, filename: str\n) -> None:\n LOGGER.info(f\"Writing tags for image: {config.image}\")\n\n path = config.hist_lines_dir / f\"{filename}.txt\"\n path.parent.mkdir(parents=True, exist_ok=True)\n build_history_line = get_build_history_line(config, container, filename)\n path.write_text(build_history_line)\n\n LOGGER.info(f\"Build history line written to: {path}\")\n\n\ndef get_manifest(config: Config, container: Container, commit_hash_tag: str) -> str:\n LOGGER.info(f\"Calculating manifest file for image: {config.image}\")\n\n manifests = get_manifests(config.image)\n manifest_names = [manifest.__name__ for manifest in manifests]\n LOGGER.info(f\"Using manifests: {manifest_names}\")\n\n build_info_config = BuildInfoConfig(\n registry=config.registry,\n owner=config.owner,\n image=config.image,\n repository=config.repository,\n build_timestamp=BUILD_TIMESTAMP,\n )\n\n markdown_pieces = [\n f\"# Build manifest for image: {config.image}:{commit_hash_tag}\",\n build_info_manifest(build_info_config).get_str(),\n *(manifest(container).get_str() for manifest in manifests),\n ]\n markdown_content = \"\\n\\n\".join(markdown_pieces) + \"\\n\"\n\n LOGGER.info(f\"Manifest file calculated for image: {config.image}\")\n return markdown_content\n\n\ndef write_manifest(\n config: Config, container: Container, *, filename: str, commit_hash_tag: str\n) -> None:\n LOGGER.info(f\"Writing manifest file for image: {config.image}\")\n\n path = config.manifests_dir / f\"{filename}.md\"\n path.parent.mkdir(parents=True, exist_ok=True)\n manifest = get_manifest(config, container, commit_hash_tag)\n path.write_text(manifest)\n\n LOGGER.info(f\"Manifest file wrtitten to: {path}\")\n\n\ndef write_all(config: Config) -> None:\n LOGGER.info(f\"Writing all files for image: {config.image}\")\n\n file_prefix = get_file_prefix(config.variant)\n commit_hash_tag = GitHelper.commit_hash_tag()\n filename = f\"{file_prefix}-{config.image}-{commit_hash_tag}\"\n\n with DockerRunner(config.full_image()) as container:\n write_build_history_line(config, container, filename)\n write_manifest(\n config, container, filename=filename, commit_hash_tag=commit_hash_tag\n )\n\n LOGGER.info(f\"All files written for image: {config.image}\")\n\n\nif __name__ == \"__main__\":\n logging.basicConfig(level=logging.INFO)\n LOGGER.info(f\"Current build timestamp: {BUILD_TIMESTAMP}\")\n\n config = common_arguments_parser(\n registry=True,\n owner=True,\n image=True,\n variant=True,\n hist_lines_dir=True,\n manifests_dir=True,\n repository=True,\n )\n write_all(config)\n" }, { "path": "tagging/apps/write_tags_file.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\n\nfrom tagging.apps.common_cli_arguments import common_arguments_parser\nfrom tagging.apps.config import Config\nfrom tagging.hierarchy.get_taggers import get_taggers\nfrom tagging.utils.docker_runner import DockerRunner\nfrom tagging.utils.get_prefix import get_file_prefix, get_tag_prefix\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef get_tags(config: Config) -> list[str]:\n LOGGER.info(f\"Calculating tags for image: {config.image}\")\n\n taggers = get_taggers(config.image)\n tags_prefix = get_tag_prefix(config.variant)\n tags = [f\"{config.full_image()}:{tags_prefix}-latest\"]\n with DockerRunner(config.full_image()) as container:\n for tagger in taggers:\n tagger_name = tagger.__name__\n tag_value = tagger(container)\n LOGGER.info(\n f\"Calculated tag, tagger_name: {tagger_name} tag_value: {tag_value}\"\n )\n tags.append(f\"{config.full_image()}:{tags_prefix}-{tag_value}\")\n\n LOGGER.info(f\"Tags calculated for image: {config.image}\")\n return tags\n\n\ndef write_tags_file(config: Config) -> None:\n LOGGER.info(f\"Writing tags for image: {config.image}\")\n\n file_prefix = get_file_prefix(config.variant)\n filename = f\"{file_prefix}-{config.image}.txt\"\n path = config.tags_dir / filename\n path.parent.mkdir(parents=True, exist_ok=True)\n tags = get_tags(config)\n path.write_text(\"\\n\".join(tags))\n\n LOGGER.info(f\"Tags wrtitten to: {path}\")\n\n\nif __name__ == \"__main__\":\n logging.basicConfig(level=logging.INFO)\n\n config = common_arguments_parser(\n registry=True, owner=True, image=True, variant=True, tags_dir=True\n )\n write_tags_file(config)\n" }, { "path": "tagging/hierarchy/__init__.py", "content": "" }, { "path": "tagging/hierarchy/get_manifests.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom tagging.hierarchy.images_hierarchy import ALL_IMAGES\nfrom tagging.manifests.manifest_interface import ManifestInterface\n\n\ndef get_manifests(image: str | None) -> list[ManifestInterface]:\n if image is None:\n return []\n image_description = ALL_IMAGES[image]\n parent_manifests = get_manifests(image_description.parent_image)\n return parent_manifests + image_description.manifests\n" }, { "path": "tagging/hierarchy/get_taggers.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom tagging.hierarchy.images_hierarchy import ALL_IMAGES\nfrom tagging.taggers.tagger_interface import TaggerInterface\n\n\ndef get_taggers(image: str | None) -> list[TaggerInterface]:\n if image is None:\n return []\n image_description = ALL_IMAGES[image]\n parent_taggers = get_taggers(image_description.parent_image)\n return parent_taggers + image_description.taggers\n" }, { "path": "tagging/hierarchy/images_hierarchy.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom dataclasses import dataclass, field\n\nfrom tagging.manifests.apt_packages import apt_packages_manifest\nfrom tagging.manifests.conda_environment import conda_environment_manifest\nfrom tagging.manifests.julia_packages import julia_packages_manifest\nfrom tagging.manifests.manifest_interface import ManifestInterface\nfrom tagging.manifests.r_packages import r_packages_manifest\nfrom tagging.manifests.spark_info import spark_info_manifest\nfrom tagging.taggers import versions\nfrom tagging.taggers.date import date_tagger\nfrom tagging.taggers.sha import commit_sha_tagger\nfrom tagging.taggers.tagger_interface import TaggerInterface\nfrom tagging.taggers.ubuntu_version import ubuntu_version_tagger\n\n\n@dataclass\nclass ImageDescription:\n parent_image: str | None\n taggers: list[TaggerInterface] = field(default_factory=list)\n manifests: list[ManifestInterface] = field(default_factory=list)\n\n\nALL_IMAGES = {\n \"docker-stacks-foundation\": ImageDescription(\n parent_image=None,\n taggers=[\n commit_sha_tagger,\n date_tagger,\n ubuntu_version_tagger,\n versions.python_major_minor_tagger,\n versions.python_tagger,\n versions.mamba_tagger,\n versions.conda_tagger,\n ],\n manifests=[conda_environment_manifest, apt_packages_manifest],\n ),\n \"base-notebook\": ImageDescription(\n parent_image=\"docker-stacks-foundation\",\n taggers=[\n versions.jupyter_notebook_tagger,\n versions.jupyter_lab_tagger,\n versions.jupyter_hub_tagger,\n ],\n ),\n \"minimal-notebook\": ImageDescription(parent_image=\"base-notebook\"),\n \"scipy-notebook\": ImageDescription(parent_image=\"minimal-notebook\"),\n \"r-notebook\": ImageDescription(\n parent_image=\"minimal-notebook\",\n taggers=[versions.r_tagger],\n manifests=[r_packages_manifest],\n ),\n \"julia-notebook\": ImageDescription(\n parent_image=\"minimal-notebook\",\n taggers=[versions.julia_tagger],\n manifests=[julia_packages_manifest],\n ),\n \"tensorflow-notebook\": ImageDescription(\n parent_image=\"scipy-notebook\", taggers=[versions.tensorflow_tagger]\n ),\n \"pytorch-notebook\": ImageDescription(\n parent_image=\"scipy-notebook\", taggers=[versions.python_tagger]\n ),\n \"datascience-notebook\": ImageDescription(\n parent_image=\"scipy-notebook\",\n taggers=[versions.r_tagger, versions.julia_tagger],\n manifests=[r_packages_manifest, julia_packages_manifest],\n ),\n \"pyspark-notebook\": ImageDescription(\n parent_image=\"scipy-notebook\",\n taggers=[versions.spark_tagger, versions.java_tagger],\n manifests=[spark_info_manifest],\n ),\n \"all-spark-notebook\": ImageDescription(\n parent_image=\"pyspark-notebook\",\n taggers=[versions.r_tagger],\n manifests=[r_packages_manifest],\n ),\n}\n" }, { "path": "tagging/manifests/__init__.py", "content": "" }, { "path": "tagging/manifests/apt_packages.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom docker.models.containers import Container\n\nfrom tagging.manifests.manifest_interface import MarkdownPiece\nfrom tagging.utils.quoted_output import quoted_output\n\n\ndef apt_packages_manifest(container: Container) -> MarkdownPiece:\n return MarkdownPiece(\n title=\"## Apt Packages\",\n sections=[quoted_output(container, \"apt list --installed\")],\n )\n" }, { "path": "tagging/manifests/build_info.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport textwrap\nfrom dataclasses import dataclass\n\nimport plumbum\n\nfrom tagging.manifests.manifest_interface import MarkdownPiece\nfrom tagging.utils.git_helper import GitHelper\n\ndocker = plumbum.local[\"docker\"]\n\n\n@dataclass(frozen=True)\nclass BuildInfoConfig:\n registry: str\n owner: str\n image: str\n\n repository: str\n\n build_timestamp: str\n\n def full_image(self) -> str:\n return f\"{self.registry}/{self.owner}/{self.image}\"\n\n\ndef build_info_manifest(config: BuildInfoConfig) -> MarkdownPiece:\n \"\"\"BuildInfo doesn't fall under common interface, and we run it separately\"\"\"\n commit_hash = GitHelper.commit_hash()\n commit_hash_tag = GitHelper.commit_hash_tag()\n commit_message = GitHelper.commit_message()\n\n # Unfortunately, `docker images` doesn't work when specifying `docker.io` as registry\n fixed_registry = config.registry + \"/\" if config.registry != \"docker.io\" else \"\"\n\n image_size = docker[\n \"images\",\n f\"{fixed_registry}{config.owner}/{config.image}:latest\",\n \"--format\",\n \"{{.Size}}\",\n ]().rstrip()\n\n build_info = textwrap.dedent(f\"\"\"\\\n - Build timestamp: {config.build_timestamp}\n - Docker image: `{config.full_image()}:{commit_hash_tag}`\n - Docker image size: {image_size}\n - Git commit SHA: [{commit_hash}](https://github.com/{config.repository}/commit/{commit_hash})\n - Git commit message:\n\n ```text\n {{message}}\n ```\"\"\").format(message=commit_message)\n\n return MarkdownPiece(title=\"## Build Info\", sections=[build_info])\n" }, { "path": "tagging/manifests/conda_environment.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom docker.models.containers import Container\n\nfrom tagging.manifests.manifest_interface import MarkdownPiece\nfrom tagging.utils.docker_runner import DockerRunner\nfrom tagging.utils.quoted_output import quoted_output\n\n\ndef conda_environment_manifest(container: Container) -> MarkdownPiece:\n return MarkdownPiece(\n title=\"## Python Packages\",\n sections=[\n DockerRunner.exec_cmd(container, \"python --version\"),\n quoted_output(container, \"conda info\"),\n quoted_output(container, \"mamba info\"),\n quoted_output(container, \"mamba list\"),\n ],\n )\n" }, { "path": "tagging/manifests/julia_packages.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom docker.models.containers import Container\n\nfrom tagging.manifests.manifest_interface import MarkdownPiece\nfrom tagging.utils.quoted_output import quoted_output\n\n\ndef julia_packages_manifest(container: Container) -> MarkdownPiece:\n return MarkdownPiece(\n title=\"## Julia Packages\",\n sections=[\n quoted_output(\n container, \"julia -E 'using InteractiveUtils; versioninfo()'\"\n ),\n quoted_output(container, \"julia -E 'import Pkg; Pkg.status()'\"),\n ],\n )\n" }, { "path": "tagging/manifests/manifest_interface.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom collections.abc import Callable\nfrom dataclasses import dataclass\n\nfrom docker.models.containers import Container\n\n\n@dataclass(frozen=True)\nclass MarkdownPiece:\n title: str\n sections: list[str]\n\n def __post_init__(self) -> None:\n # All pieces are H2\n assert self.title.startswith(\"## \")\n\n def get_str(self) -> str:\n return \"\\n\\n\".join([self.title, *self.sections])\n\n\nManifestInterface = Callable[[Container], MarkdownPiece]\n" }, { "path": "tagging/manifests/r_packages.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom docker.models.containers import Container\n\nfrom tagging.manifests.manifest_interface import MarkdownPiece\nfrom tagging.utils.quoted_output import quoted_output\n\n\ndef r_packages_manifest(container: Container) -> MarkdownPiece:\n return MarkdownPiece(\n title=\"## R Packages\",\n sections=[\n quoted_output(container, \"R --version\"),\n quoted_output(\n container, \"R --silent -e 'installed.packages(.Library)[, c(1,3)]'\"\n ),\n ],\n )\n" }, { "path": "tagging/manifests/spark_info.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom docker.models.containers import Container\n\nfrom tagging.manifests.manifest_interface import MarkdownPiece\nfrom tagging.utils.quoted_output import quoted_output\n\n\ndef spark_info_manifest(container: Container) -> MarkdownPiece:\n return MarkdownPiece(\n title=\"## Apache Spark\",\n sections=[\n quoted_output(container, \"/usr/local/spark/bin/spark-submit --version\")\n ],\n )\n" }, { "path": "tagging/taggers/__init__.py", "content": "" }, { "path": "tagging/taggers/date.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport datetime\n\nfrom docker.models.containers import Container\n\n\ndef date_tagger(container: Container) -> str:\n return datetime.datetime.now(datetime.UTC).strftime(\"%Y-%m-%d\")\n" }, { "path": "tagging/taggers/sha.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom docker.models.containers import Container\n\nfrom tagging.utils.git_helper import GitHelper\n\n\ndef commit_sha_tagger(container: Container) -> str:\n return GitHelper.commit_hash_tag()\n" }, { "path": "tagging/taggers/tagger_interface.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom collections.abc import Callable\n\nfrom docker.models.containers import Container\n\nTaggerInterface = Callable[[Container], str]\n" }, { "path": "tagging/taggers/ubuntu_version.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom docker.models.containers import Container\n\nfrom tagging.utils.docker_runner import DockerRunner\n\n\ndef ubuntu_version_tagger(container: Container) -> str:\n os_release = DockerRunner.exec_cmd(\n container,\n \"cat /etc/os-release\",\n ).split(\"\\n\")\n for line in os_release:\n if line.startswith(\"VERSION_ID\"):\n return \"ubuntu-\" + line.split(\"=\")[1].strip('\"')\n raise RuntimeError(f\"did not find ubuntu version in: {os_release}\")\n" }, { "path": "tagging/taggers/versions.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom docker.models.containers import Container\n\nfrom tagging.utils.docker_runner import DockerRunner\n\n\ndef _get_program_version(container: Container, program: str) -> str:\n return DockerRunner.exec_cmd(container, cmd=f\"{program} --version\")\n\n\ndef _get_pip_package_version(container: Container, package: str) -> str:\n PIP_VERSION_PREFIX = \"Version: \"\n\n package_info = DockerRunner.exec_cmd(\n container,\n cmd=f\"pip show {package}\",\n )\n version_line = package_info.split(\"\\n\")[1]\n assert version_line.startswith(PIP_VERSION_PREFIX)\n return version_line[len(PIP_VERSION_PREFIX) :]\n\n\ndef python_tagger(container: Container) -> str:\n return \"python-\" + _get_program_version(container, \"python\").split()[1]\n\n\ndef python_major_minor_tagger(container: Container) -> str:\n full_version = python_tagger(container)\n return full_version[: full_version.rfind(\".\")]\n\n\ndef mamba_tagger(container: Container) -> str:\n return \"mamba-\" + _get_program_version(container, \"mamba\")\n\n\ndef conda_tagger(container: Container) -> str:\n return \"conda-\" + _get_program_version(container, \"conda\").split()[1]\n\n\ndef jupyter_notebook_tagger(container: Container) -> str:\n return \"notebook-\" + _get_program_version(container, \"jupyter-notebook\")\n\n\ndef jupyter_lab_tagger(container: Container) -> str:\n return \"lab-\" + _get_program_version(container, \"jupyter-lab\")\n\n\ndef jupyter_hub_tagger(container: Container) -> str:\n return \"hub-\" + _get_program_version(container, \"jupyterhub\")\n\n\ndef r_tagger(container: Container) -> str:\n return \"r-\" + _get_program_version(container, \"R\").split()[2]\n\n\ndef julia_tagger(container: Container) -> str:\n return \"julia-\" + _get_program_version(container, \"julia\").split()[2]\n\n\ndef tensorflow_tagger(container: Container) -> str:\n try:\n return \"tensorflow-\" + _get_pip_package_version(container, \"tensorflow\")\n except AssertionError:\n return \"tensorflow-\" + _get_pip_package_version(container, \"tensorflow-cpu\")\n\n\ndef pytorch_tagger(container: Container) -> str:\n return \"pytorch-\" + _get_pip_package_version(container, \"torch\").split(\"+\")[0]\n\n\ndef spark_tagger(container: Container) -> str:\n SPARK_VERSION_LINE_PREFIX = r\" /___/ .__/\\_,_/_/ /_/\\_\\ version\"\n\n spark_version = _get_program_version(container, \"spark-submit\")\n version_line = next(\n filter(\n lambda line: line.startswith(SPARK_VERSION_LINE_PREFIX),\n spark_version.split(\"\\n\"),\n )\n )\n return \"spark-\" + version_line.split(\" \")[-1]\n\n\ndef java_tagger(container: Container) -> str:\n return \"java-\" + _get_program_version(container, \"java\").split()[1]\n" }, { "path": "tagging/utils/__init__.py", "content": "" }, { "path": "tagging/utils/docker_runner.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nfrom types import TracebackType\n\nimport docker\nfrom docker.models.containers import Container\n\nLOGGER = logging.getLogger(__name__)\n\n\nclass DockerRunner:\n def __init__(\n self,\n image_name: str,\n docker_client: docker.DockerClient = docker.from_env(),\n command: str = \"sleep infinity\",\n ):\n self.container: Container | None = None\n self.image_name: str = image_name\n self.command: str = command\n self.docker_client: docker.DockerClient = docker_client\n\n def __enter__(self) -> Container:\n LOGGER.info(f\"Creating a container for the image: {self.image_name} ...\")\n default_kwargs = {\"detach\": True, \"tty\": True}\n self.container = self.docker_client.containers.run(\n image=self.image_name, command=self.command, **default_kwargs\n )\n LOGGER.info(f\"Container {self.container.name} created\")\n return self.container\n\n def __exit__(\n self,\n exc_type: type[BaseException] | None,\n exc_val: BaseException | None,\n exc_tb: TracebackType | None,\n ) -> None:\n assert self.container is not None\n LOGGER.info(f\"Removing container {self.container.name} ...\")\n self.container.remove(force=True)\n LOGGER.info(f\"Container {self.container.name} removed\")\n\n @staticmethod\n def exec_cmd(container: Container, cmd: str) -> str:\n LOGGER.info(f\"Running cmd: `{cmd}` on container: {container.name}\")\n exec_result = container.exec_run(cmd)\n output = exec_result.output.decode().rstrip()\n assert isinstance(output, str)\n if exec_result.exit_code != 0:\n LOGGER.error(f\"Command output:\\n{output}\")\n raise AssertionError(f\"Command: `{cmd}` failed\")\n else:\n LOGGER.debug(f\"Command output:\\n{output}\")\n return output\n" }, { "path": "tagging/utils/get_platform.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport platform\n\nALL_PLATFORMS = {\"x86_64\", \"aarch64\"}\n\n\ndef unify_aarch64(platform: str) -> str:\n \"\"\"\n Renames arm64->aarch64 to support local builds on aarch64 Macs\n \"\"\"\n return {\"arm64\": \"aarch64\"}.get(platform, platform)\n\n\ndef get_platform() -> str:\n machine = platform.machine()\n return unify_aarch64(machine)\n" }, { "path": "tagging/utils/get_prefix.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom tagging.utils.get_platform import get_platform\n\nDEFAULT_VARIANT = \"default\"\n\n\ndef get_file_prefix_for_platform(*, platform: str, variant: str) -> str:\n return f\"{platform}-{variant}\"\n\n\ndef _get_tag_prefix_for_platform(*, platform: str, variant: str) -> str:\n if variant == DEFAULT_VARIANT:\n return platform\n return f\"{platform}-{variant}\"\n\n\ndef get_file_prefix(variant: str) -> str:\n platform = get_platform()\n return get_file_prefix_for_platform(platform=platform, variant=variant)\n\n\ndef get_tag_prefix(variant: str) -> str:\n platform = get_platform()\n return _get_tag_prefix_for_platform(platform=platform, variant=variant)\n" }, { "path": "tagging/utils/git_helper.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport plumbum\n\ngit = plumbum.local[\"git\"]\n\n\nclass GitHelper:\n @staticmethod\n def commit_hash() -> str:\n return git[\"rev-parse\", \"HEAD\"]().strip() # type: ignore\n\n @staticmethod\n def commit_hash_tag() -> str:\n return GitHelper.commit_hash()[:12]\n\n @staticmethod\n def commit_message() -> str:\n return git[\"log\", -1, \"--pretty=%B\"]().strip() # type: ignore\n\n\nif __name__ == \"__main__\":\n print(\"Git hash:\", GitHelper.commit_hash())\n print(\"Git message:\", GitHelper.commit_message())\n" }, { "path": "tagging/utils/quoted_output.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport textwrap\n\nfrom docker.models.containers import Container\n\nfrom tagging.utils.docker_runner import DockerRunner\n\n\ndef quoted_output(container: Container, cmd: str) -> str:\n cmd_output = DockerRunner.exec_cmd(container, cmd)\n # For example, `mamba info` adds redundant empty lines\n cmd_output = cmd_output.strip(\"\\n\")\n # For example, R packages list contains trailing backspaces\n cmd_output = \"\\n\".join(line.rstrip() for line in cmd_output.split(\"\\n\"))\n\n assert cmd_output, f\"Command `{cmd}` returned empty output\"\n\n return textwrap.dedent(f\"\"\"\\\n `{cmd}`:\n\n ```text\n {{output}}\n ```\"\"\").format(output=cmd_output)\n" }, { "path": "tests/README.md", "content": "# Docker stacks testing\n\nPlease, refer to the [testing section of documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/contributing/tests.html) to see how the tests are run.\n" }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/by_image/all-spark-notebook/data/local_sparkR.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"library(SparkR)\\n\",\n \"\\n\",\n \"# Spark session & context\\n\",\n \"sc <- sparkR.session(\\\"local\\\")\\n\",\n \"\\n\",\n \"# Sum of the first 100 whole numbers\\n\",\n \"sdf <- createDataFrame(list(1:100))\\n\",\n \"dapplyCollect(sdf,\\n\",\n \" function(x) \\n\",\n \" { x <- sum(x)}\\n\",\n \" )\\n\",\n \"# 5050\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"R\",\n \"language\": \"R\",\n \"name\": \"ir\"\n },\n \"language_info\": {\n \"codemirror_mode\": \"r\",\n \"file_extension\": \".r\",\n \"mimetype\": \"text/x-r-source\",\n \"name\": \"R\",\n \"pygments_lexer\": \"r\",\n \"version\": \"3.6.3\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n" }, { "path": "tests/by_image/all-spark-notebook/data/local_sparklyr.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"library(sparklyr)\\n\",\n \"\\n\",\n \"# get the default config\\n\",\n \"conf <- spark_config()\\n\",\n \"# Set the catalog implementation in-memory\\n\",\n \"conf$spark.sql.catalogImplementation <- \\\"in-memory\\\"\\n\",\n \"\\n\",\n \"# Spark session & context\\n\",\n \"sc <- spark_connect(master = \\\"local\\\", config = conf)\\n\",\n \"\\n\",\n \"# Sum of the first 100 whole numbers\\n\",\n \"sdf_len(sc, 100, repartition = 1) %>% \\n\",\n \" spark_apply(function(e) sum(e))\\n\",\n \"# 5050\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"R\",\n \"language\": \"R\",\n \"name\": \"ir\"\n },\n \"language_info\": {\n \"codemirror_mode\": \"r\",\n \"file_extension\": \".r\",\n \"mimetype\": \"text/x-r-source\",\n \"name\": \"R\",\n \"pygments_lexer\": \"r\",\n \"version\": \"3.6.3\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n" }, { "path": "tests/by_image/all-spark-notebook/test_spark_r_nbconvert.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nfrom pathlib import Path\n\nimport pytest # type: ignore\n\nfrom tests.shared_checks.nbconvert_check import check_nbconvert\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\nTHIS_DIR = Path(__file__).parent.resolve()\n\n\n@pytest.mark.flaky(reruns=3, reruns_delay=1)\n@pytest.mark.parametrize(\n \"test_file,expected_warnings\",\n [\n (\"local_sparkR\", [\"WARNING: Using incubator modules: jdk.incubator.vector\"]),\n (\"local_sparklyr\", []),\n ],\n)\n@pytest.mark.parametrize(\"output_format\", [\"pdf\", \"html\", \"markdown\"])\ndef test_spark_r_nbconvert(\n container: TrackedContainer,\n test_file: str,\n output_format: str,\n expected_warnings: list[str],\n) -> None:\n host_data_file = THIS_DIR / \"data\" / f\"{test_file}.ipynb\"\n logs = check_nbconvert(\n container,\n host_data_file,\n output_format,\n execute=True,\n no_warnings=(not expected_warnings),\n )\n\n warnings = TrackedContainer.get_warnings(logs)\n assert warnings == expected_warnings\n" }, { "path": "tests/by_image/base-notebook/data/check_listening.py", "content": "#!/usr/bin/env python\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport socket\nimport time\n\nimport requests\n\n\ndef make_get_request() -> None:\n # Give some time for server to start\n finish_time = time.time() + 10\n sleep_time = 1\n while time.time() < finish_time:\n time.sleep(sleep_time)\n try:\n resp = requests.get(\"http://localhost:8888/api\")\n resp.raise_for_status()\n except requests.RequestException:\n pass\n resp.raise_for_status()\n\n\ndef check_addrs(family: socket.AddressFamily) -> None:\n assert family in {socket.AF_INET, socket.AF_INET6}\n\n # https://docs.python.org/3/library/socket.html#socket.getaddrinfo\n addrs = {\n s[4][0]\n for s in socket.getaddrinfo(host=socket.gethostname(), port=None, family=family)\n }\n loopback_addr = \"127.0.0.1\" if family == socket.AF_INET else \"::1\"\n addrs.discard(loopback_addr)\n\n assert addrs, f\"No external addresses found for family: {family}\"\n\n for addr in addrs:\n url = (\n f\"http://{addr}:8888/api\"\n if family == socket.AF_INET\n else f\"http://[{addr}]:8888/api\"\n )\n r = requests.get(url)\n r.raise_for_status()\n assert \"version\" in r.json()\n print(f\"Successfully connected to: {url}\")\n\n\ndef test_connect() -> None:\n make_get_request()\n\n check_addrs(socket.AF_INET)\n check_addrs(socket.AF_INET6)\n\n\nif __name__ == \"__main__\":\n test_connect()\n" }, { "path": "tests/by_image/base-notebook/test_container_options.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nimport time\n\nimport pytest # type: ignore\nimport requests\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef test_cli_args(\n container: TrackedContainer, http_client: requests.Session, free_host_port: int\n) -> None:\n \"\"\"Image should respect command line args (e.g., disabling token security)\"\"\"\n container.run_detached(\n command=[\"start-notebook.py\", \"--IdentityProvider.token=''\"],\n ports={\"8888/tcp\": free_host_port},\n )\n resp = http_client.get(f\"http://localhost:{free_host_port}\")\n resp.raise_for_status()\n logs = container.get_logs()\n LOGGER.debug(logs)\n assert \"ERROR\" not in logs\n warnings = TrackedContainer.get_warnings(logs)\n assert not warnings\n assert \"login_submit\" not in resp.text\n\n\ndef test_nb_user_change(container: TrackedContainer) -> None:\n \"\"\"Container should change the username (`NB_USER`) of the default user.\"\"\"\n nb_user = \"nayvoj\"\n container.run_detached(\n user=\"root\",\n environment=[f\"NB_USER={nb_user}\", \"CHOWN_HOME=yes\"],\n command=[\"sleep\", \"infinity\"],\n )\n\n # Give the chown time to complete.\n # Use sleep, not wait, because the container sleeps forever.\n time.sleep(5)\n LOGGER.info(\n f\"Checking if a home folder of {nb_user} contains the hidden '.jupyter' folder with appropriate permissions ...\"\n )\n command = f'stat -c \"%F %U %G\" /home/{nb_user}/.jupyter'\n expected_output = f\"directory {nb_user} users\"\n output = container.exec_cmd(command, workdir=f\"/home/{nb_user}\")\n assert (\n output == expected_output\n ), f\"Hidden folder .jupyter was not copied properly to {nb_user} home folder. stat: {output}, expected {expected_output}\"\n\n\n@pytest.mark.filterwarnings(\"ignore:Unverified HTTPS request\")\ndef test_unsigned_ssl(\n container: TrackedContainer, http_client: requests.Session, free_host_port: int\n) -> None:\n \"\"\"Container should generate a self-signed SSL certificate\n and Jupyter Server should use it to enable HTTPS.\n \"\"\"\n container.run_detached(\n environment=[\"GEN_CERT=yes\"],\n ports={\"8888/tcp\": free_host_port},\n )\n # NOTE: The requests.Session backing the http_client fixture\n # does not retry properly while the server is booting up.\n # An SSL handshake error seems to abort the retry logic.\n # Forcing a long sleep for the moment until I have time to dig more.\n time.sleep(1)\n resp = http_client.get(f\"https://localhost:{free_host_port}\", verify=False)\n resp.raise_for_status()\n assert \"login_submit\" in resp.text\n logs = container.get_logs()\n assert \"ERROR\" not in logs\n warnings = TrackedContainer.get_warnings(logs)\n assert not warnings\n\n\n@pytest.mark.parametrize(\n \"env\",\n [\n {},\n {\"JUPYTER_PORT\": 1234, \"DOCKER_STACKS_JUPYTER_CMD\": \"lab\"},\n {\"JUPYTER_PORT\": 2345, \"DOCKER_STACKS_JUPYTER_CMD\": \"notebook\"},\n {\"JUPYTER_PORT\": 3456, \"DOCKER_STACKS_JUPYTER_CMD\": \"server\"},\n {\"JUPYTER_PORT\": 4567, \"DOCKER_STACKS_JUPYTER_CMD\": \"nbclassic\"},\n {\"JUPYTER_PORT\": 5678, \"RESTARTABLE\": \"yes\"},\n {\"JUPYTER_PORT\": 6789},\n {\"JUPYTER_PORT\": 7890, \"DOCKER_STACKS_JUPYTER_CMD\": \"notebook\"},\n ],\n)\ndef test_custom_internal_port(\n container: TrackedContainer,\n http_client: requests.Session,\n free_host_port: int,\n env: dict[str, str],\n) -> None:\n \"\"\"Container should be accessible from the host\n when using custom internal port\"\"\"\n internal_port = env.get(\"JUPYTER_PORT\", 8888)\n container.run_detached(\n command=[\"start-notebook.py\", \"--IdentityProvider.token=''\"],\n environment=env,\n ports={internal_port: free_host_port},\n )\n resp = http_client.get(f\"http://localhost:{free_host_port}\")\n resp.raise_for_status()\n logs = container.get_logs()\n LOGGER.debug(logs)\n assert \"ERROR\" not in logs\n warnings = TrackedContainer.get_warnings(logs)\n assert not warnings\n" }, { "path": "tests/by_image/base-notebook/test_healthcheck.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nimport time\n\nimport pytest # type: ignore\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef get_healthy_status(\n container: TrackedContainer,\n *,\n env: list[str] | None,\n cmd: list[str] | None,\n user: str | None,\n) -> str:\n container.run_detached(\n environment=env,\n command=cmd,\n user=user,\n )\n\n # giving some time to let the server start\n finish_time = time.time() + 10\n sleep_time = 1\n while time.time() < finish_time:\n time.sleep(sleep_time)\n\n status = container.get_health()\n if status == \"healthy\":\n return status\n\n return status\n\n\n@pytest.mark.parametrize(\n \"env,cmd,user\",\n [\n (None, None, None),\n ([\"DOCKER_STACKS_JUPYTER_CMD=lab\"], None, None),\n ([\"DOCKER_STACKS_JUPYTER_CMD=notebook\"], None, None),\n ([\"DOCKER_STACKS_JUPYTER_CMD=server\"], None, None),\n ([\"DOCKER_STACKS_JUPYTER_CMD=nbclassic\"], None, None),\n ([\"RESTARTABLE=yes\"], None, None),\n ([\"JUPYTER_PORT=8171\"], None, None),\n ([\"JUPYTER_PORT=8117\", \"DOCKER_STACKS_JUPYTER_CMD=notebook\"], None, None),\n (None, [\"start-notebook.sh\"], None),\n (None, [\"start-notebook.py\", \"--ServerApp.base_url=/test\"], None),\n (None, [\"start-notebook.py\", \"--ServerApp.base_url=/test/\"], None),\n ([\"GEN_CERT=1\"], [\"start-notebook.py\", \"--ServerApp.base_url=/test\"], None),\n (\n [\"GEN_CERT=1\", \"JUPYTER_PORT=7891\"],\n [\"start-notebook.py\", \"--ServerApp.base_url=/test\"],\n None,\n ),\n ([\"NB_USER=testuser\", \"CHOWN_HOME=1\"], None, \"root\"),\n (\n [\"NB_USER=testuser\", \"CHOWN_HOME=1\"],\n [\"start-notebook.py\", \"--ServerApp.base_url=/test\"],\n \"root\",\n ),\n (\n [\"NB_USER=testuser\", \"CHOWN_HOME=1\", \"JUPYTER_PORT=8123\"],\n [\"start-notebook.py\", \"--ServerApp.base_url=/test\"],\n \"root\",\n ),\n ([\"JUPYTER_RUNTIME_DIR=/tmp/jupyter-runtime\"], [\"start-notebook.sh\"], None),\n (\n [\n \"NB_USER=testuser\",\n \"CHOWN_HOME=1\",\n \"JUPYTER_RUNTIME_DIR=/tmp/jupyter-runtime\",\n ],\n [\"start-notebook.sh\"],\n \"root\",\n ),\n ],\n)\ndef test_healthy(\n container: TrackedContainer,\n env: list[str] | None,\n cmd: list[str] | None,\n user: str | None,\n) -> None:\n assert get_healthy_status(container, env=env, cmd=cmd, user=user) == \"healthy\"\n\n\n@pytest.mark.parametrize(\n \"env,cmd,user\",\n [\n (\n [\n \"HTTPS_PROXY=https://host.docker.internal\",\n \"HTTP_PROXY=http://host.docker.internal\",\n ],\n None,\n None,\n ),\n (\n [\n \"NB_USER=testuser\",\n \"CHOWN_HOME=1\",\n \"JUPYTER_PORT=8123\",\n \"HTTPS_PROXY=https://host.docker.internal\",\n \"HTTP_PROXY=http://host.docker.internal\",\n ],\n [\"start-notebook.py\", \"--ServerApp.base_url=/test\"],\n \"root\",\n ),\n ],\n)\ndef test_healthy_with_proxy(\n container: TrackedContainer,\n env: list[str] | None,\n cmd: list[str] | None,\n user: str | None,\n) -> None:\n assert get_healthy_status(container, env=env, cmd=cmd, user=user) == \"healthy\"\n\n\n@pytest.mark.parametrize(\n \"env,cmd\",\n [\n ([\"NB_USER=testuser\", \"CHOWN_HOME=1\"], None),\n (\n [\"NB_USER=testuser\", \"CHOWN_HOME=1\"],\n [\"start-notebook.py\", \"--ServerApp.base_url=/test\"],\n ),\n (\n [\"NB_USER=testuser\", \"CHOWN_HOME=1\", \"JUPYTER_PORT=8123\"],\n [\"start-notebook.py\", \"--ServerApp.base_url=/test\"],\n ),\n ],\n)\ndef test_not_healthy(\n container: TrackedContainer,\n env: list[str] | None,\n cmd: list[str] | None,\n) -> None:\n assert (\n get_healthy_status(container, env=env, cmd=cmd, user=None) != \"healthy\"\n ), \"Container should not be healthy for this testcase\"\n" }, { "path": "tests/by_image/base-notebook/test_ips.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nfrom collections.abc import Generator\nfrom pathlib import Path\nfrom random import randint\n\nimport docker\nimport pytest # type: ignore\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\nTHIS_DIR = Path(__file__).parent.resolve()\n\n\n@pytest.fixture(scope=\"session\")\ndef ipv6_network(docker_client: docker.DockerClient) -> Generator[str, None, None]:\n \"\"\"Create a dual-stack IPv6 docker network\"\"\"\n # Doesn't have to be routable since we're testing inside the container\n subnet64 = \"fc00:\" + \":\".join(hex(randint(0, 2**16))[2:] for _ in range(3))\n name = subnet64.replace(\":\", \"-\")\n docker_client.networks.create(\n name,\n ipam=docker.types.IPAMPool(\n subnet=subnet64 + \"::/64\",\n gateway=subnet64 + \"::1\",\n ),\n enable_ipv6=True,\n internal=True,\n )\n yield name\n docker_client.networks.get(name).remove()\n\n\ndef test_ipv46(container: TrackedContainer, ipv6_network: str) -> None:\n \"\"\"Check server is listening on the expected IP families\"\"\"\n file_name = \"check_listening.py\"\n host_file = THIS_DIR / \"data\" / file_name\n cont_file = f\"/home/jovyan/data/{file_name}\"\n LOGGER.info(\"Testing that server is listening on IPv4 and IPv6 ...\")\n container.run_detached(\n network=ipv6_network,\n volumes={host_file: {\"bind\": cont_file, \"mode\": \"ro\"}},\n )\n container.exec_cmd(f\"python {cont_file}\")\n" }, { "path": "tests/by_image/base-notebook/test_notebook.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport requests\n\nfrom tests.utils.tracked_container import TrackedContainer\n\n\ndef test_secured_server(\n container: TrackedContainer, http_client: requests.Session, free_host_port: int\n) -> None:\n \"\"\"Jupyter Server should eventually request user login.\"\"\"\n container.run_detached(ports={\"8888/tcp\": free_host_port})\n resp = http_client.get(f\"http://localhost:{free_host_port}\")\n resp.raise_for_status()\n assert \"login_submit\" in resp.text, \"User login not requested\"\n" }, { "path": "tests/by_image/base-notebook/test_pandoc.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef test_pandoc(container: TrackedContainer) -> None:\n \"\"\"Pandoc shall be able to convert MD to HTML.\"\"\"\n logs = container.run_and_wait(\n timeout=10,\n command=[\"bash\", \"-c\", 'echo \"**BOLD**\" | pandoc'],\n )\n assert \"

BOLD

\" in logs\n" }, { "path": "tests/by_image/base-notebook/test_start_container.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nimport time\n\nimport pytest # type: ignore\nimport requests\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\n@pytest.mark.parametrize(\n \"env,expected_command,expected_start,expected_warnings\",\n [\n (None, \"jupyter lab\", True, []),\n ([\"DOCKER_STACKS_JUPYTER_CMD=lab\"], \"jupyter lab\", True, []),\n ([\"RESTARTABLE=yes\"], \"run-one-constantly jupyter lab\", True, []),\n ([\"DOCKER_STACKS_JUPYTER_CMD=notebook\"], \"jupyter notebook\", True, []),\n ([\"DOCKER_STACKS_JUPYTER_CMD=server\"], \"jupyter server\", True, []),\n ([\"DOCKER_STACKS_JUPYTER_CMD=nbclassic\"], \"jupyter nbclassic\", True, []),\n (\n [\"JUPYTERHUB_API_TOKEN=my_token\"],\n \"jupyterhub-singleuser\",\n False,\n [\"WARNING: using start-singleuser.py\"],\n ),\n ],\n)\ndef test_start_notebook(\n container: TrackedContainer,\n http_client: requests.Session,\n free_host_port: int,\n env: list[str] | None,\n expected_command: str,\n expected_start: bool,\n expected_warnings: list[str],\n) -> None:\n \"\"\"Test the notebook start-notebook.py script\"\"\"\n LOGGER.info(\n f\"Test that the start-notebook.py launches the {expected_command} server from the env {env} ...\"\n )\n container.run_detached(environment=env, ports={\"8888/tcp\": free_host_port})\n # sleeping some time to let the server start\n time.sleep(2)\n logs = container.get_logs()\n LOGGER.debug(logs)\n # checking that the expected command is launched\n assert (\n f\"Executing: {expected_command}\" in logs\n ), f\"Not the expected command ({expected_command}) was launched\"\n # checking errors and warnings in logs\n assert \"ERROR\" not in logs, \"ERROR(s) found in logs\"\n for exp_warning in expected_warnings:\n assert exp_warning in logs, f\"Expected warning {exp_warning} not found in logs\"\n warnings = TrackedContainer.get_warnings(logs)\n assert len(expected_warnings) == len(warnings)\n # checking if the server is listening\n if expected_start:\n resp = http_client.get(f\"http://localhost:{free_host_port}\")\n assert resp.status_code == 200, \"Server is not listening\"\n\n\ndef test_tini_entrypoint(\n container: TrackedContainer, pid: int = 1, command: str = \"tini\"\n) -> None:\n \"\"\"Check that tini is launched as PID 1\n\n Credits to the following answer for the ps options used in the test:\n https://superuser.com/questions/632979/if-i-know-the-pid-number-of-a-process-how-can-i-get-its-name\n \"\"\"\n LOGGER.info(f\"Test that {command} is launched as PID {pid} ...\")\n container.run_detached()\n # Select the PID 1 and get the corresponding command\n output = container.exec_cmd(f\"ps -p {pid} -o comm=\")\n assert \"ERROR\" not in output\n assert \"WARNING\" not in output\n assert output == command, f\"{command} shall be launched as pid {pid}, got {output}\"\n" }, { "path": "tests/by_image/datascience-notebook/test_julia_datascience.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom tests.utils.tracked_container import TrackedContainer\n\n\ndef test_julia(container: TrackedContainer) -> None:\n container.run_and_wait(timeout=10, command=[\"julia\", \"--version\"])\n" }, { "path": "tests/by_image/datascience-notebook/test_mimetypes.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom tests.shared_checks.R_mimetype_check import check_r_mimetypes\nfrom tests.utils.tracked_container import TrackedContainer\n\n\ndef test_mimetypes(container: TrackedContainer) -> None:\n \"\"\"Check if Rscript command for mimetypes can be executed\"\"\"\n check_r_mimetypes(container)\n" }, { "path": "tests/by_image/datascience-notebook/test_pluto_datascience.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport requests\n\nfrom tests.shared_checks.pluto_check import check_pluto_proxy\nfrom tests.utils.tracked_container import TrackedContainer\n\n\ndef test_pluto_proxy(\n container: TrackedContainer, http_client: requests.Session, free_host_port: int\n) -> None:\n \"\"\"Pluto proxy starts Pluto correctly\"\"\"\n check_pluto_proxy(container, http_client, free_host_port)\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/change/a.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nexport MY_VAR=123\necho \"Inside a.sh MY_VAR variable has ${MY_VAR} value\"\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/change/b.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\necho \"Inside b.sh MY_VAR variable has ${MY_VAR} value\"\necho \"Changing value of MY_VAR\"\nexport MY_VAR=456\necho \"After change inside b.sh MY_VAR variable has ${MY_VAR} value\"\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/change/c.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\necho \"Inside c.sh MY_VAR variable has ${MY_VAR} value\"\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/executables/executable.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nprint(\"Executable python file was successfully run\")\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/executables/non_executable.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nassert False\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/executables/run-me.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nexport SOME_VAR=123\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/failures/a.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\necho \"Started: a.sh\"\n\nexport OTHER_VAR=456\n\nrun-unknown-command\n\necho \"Finished: a.sh\"\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/failures/b.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport os\nimport sys\n\nprint(\"Started: b.py\")\nprint(f\"OTHER_VAR={os.environ['OTHER_VAR']}\")\n\nsys.exit(1)\n\nprint(\"Finished: b.py\")\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/failures/c.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\necho \"Started: c.sh\"\n\nrun-unknown-command\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/failures/d.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nset -e\n\necho \"Started: d.sh\"\n\nrun-unknown-command\n\necho \"Finished: d.sh\"\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/sh-files/executable.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nexport MY_VAR=0\necho \"Inside executable.sh MY_VAR variable has ${MY_VAR} value\"\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/sh-files/non-executable.sh", "content": "# shellcheck disable=SC2148\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nexport MY_VAR=1\necho \"Inside non-executable.sh MY_VAR variable has ${MY_VAR} value\"\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/unset/a.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\nexport MY_VAR=123\necho \"Inside a.sh MY_VAR variable has ${MY_VAR} value\"\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/unset/b.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\necho \"Inside b.sh MY_VAR variable has ${MY_VAR} value\"\necho \"Unsetting MY_VAR\"\nunset MY_VAR\n" }, { "path": "tests/by_image/docker-stacks-foundation/data/run-hooks/unset/c.sh", "content": "#!/bin/bash\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\necho \"Inside c.sh MY_VAR variable has ${MY_VAR} value\"\n" }, { "path": "tests/by_image/docker-stacks-foundation/test_outdated.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\n\nimport pytest # type: ignore\n\nfrom tests.utils.conda_package_helper import CondaPackageHelper\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\n@pytest.mark.parametrize(\"requested_only\", [True, False])\n@pytest.mark.info\ndef test_outdated_packages(container: TrackedContainer, requested_only: bool) -> None:\n \"\"\"Getting the list of updatable packages\"\"\"\n LOGGER.info(f\"Checking outdated packages in {container.image_name} ...\")\n pkg_helper = CondaPackageHelper(container)\n updatable = pkg_helper.find_updatable_packages(requested_only)\n LOGGER.info(pkg_helper.get_outdated_summary(updatable, requested_only))\n LOGGER.info(\n f\"Outdated packages table:\\n{pkg_helper.get_outdated_table(updatable)}\\n\"\n )\n" }, { "path": "tests/by_image/docker-stacks-foundation/test_package_managers.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport pytest # type: ignore\n\nfrom tests.utils.tracked_container import TrackedContainer\n\n\n@pytest.mark.parametrize(\n \"package_manager_command\",\n [\"apt\", \"conda\", \"mamba\", \"pip\"],\n)\ndef test_package_manager(\n container: TrackedContainer, package_manager_command: str\n) -> None:\n \"\"\"Test that package managers are installed and run.\"\"\"\n container.run_and_wait(timeout=10, command=[package_manager_command, \"--version\"])\n" }, { "path": "tests/by_image/docker-stacks-foundation/test_packages.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n\"\"\"\ntest_packages\n~~~~~~~~~~~~~~~\nThis test module tests if the R and Python packages installed can be imported.\nIt's a basic test aiming to prove that the package is working properly.\n\nThe goal is to detect import errors that can be caused by incompatibilities between packages, for example:\n\n- #1012: issue importing `sympy`\n- #966: issue importing `pyarrow`\n\nThis module checks dynamically, through the `CondaPackageHelper`,\nonly the requested packages i.e. packages requested by `mamba install` in the `Dockerfile`s.\nThis means that it does not check dependencies.\nThis choice is a tradeoff to cover the main requirements while achieving a reasonable test duration.\nHowever, it could be easily changed (or completed) to cover dependencies as well.\nUse `package_helper.installed_packages` instead of `package_helper.requested_packages`.\n\"\"\"\n\nimport logging\nfrom collections.abc import Callable\n\nimport pytest # type: ignore\n\nfrom tests.utils.conda_package_helper import CondaPackageHelper\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n# Mapping between package and module name\nPACKAGE_MAPPING = {\n # Python\n \"beautifulsoup4\": \"bs4\",\n \"jupyter-pluto-proxy\": \"jupyter_pluto_proxy\",\n \"matplotlib-base\": \"matplotlib\",\n \"pytables\": \"tables\",\n \"scikit-image\": \"skimage\",\n \"scikit-learn\": \"sklearn\",\n # R\n \"randomforest\": \"randomForest\",\n \"rcurl\": \"RCurl\",\n \"rodbc\": \"RODBC\",\n \"rsqlite\": \"DBI\",\n}\n\n# List of packages that cannot be tested in a standard way\nEXCLUDED_PACKAGES = [\n \"conda-forge::blas=*\",\n \"grpcio-status\",\n \"grpcio\",\n \"jupyter-server-proxy\",\n \"jupyterhub-singleuser\",\n \"jupyterlab-git\",\n \"mamba\",\n \"notebook>\",\n \"protobuf\",\n \"protobuf>=5.28.3<6\",\n \"python\",\n \"r-irkernel\",\n \"unixodbc\",\n]\n\n\ndef is_r_package(package: str) -> bool:\n \"\"\"Check if a package is an R package\"\"\"\n return package.startswith(\"r-\")\n\n\ndef get_package_import_name(package: str) -> str:\n \"\"\"Perform a mapping between the package name and the name used for the import\"\"\"\n if is_r_package(package):\n package = package[2:]\n return PACKAGE_MAPPING.get(package, package)\n\n\ndef check_import_python_package(container: TrackedContainer, package: str) -> None:\n \"\"\"Try to import a Python package from the command line\"\"\"\n container.exec_cmd(f'python -c \"import {package}\"')\n\n\ndef check_import_r_package(container: TrackedContainer, package: str) -> None:\n \"\"\"Try to import an R package from the command line\"\"\"\n container.exec_cmd(f\"R --slave -e library({package})\")\n\n\ndef _check_import_packages(\n container: TrackedContainer,\n packages_to_check: list[str],\n check_function: Callable[[TrackedContainer, str], None],\n) -> None:\n \"\"\"Test if packages can be imported\"\"\"\n failed_imports = []\n LOGGER.info(\"Testing the import of packages ...\")\n for package in packages_to_check:\n LOGGER.info(f\"Trying to import {package}\")\n try:\n check_function(container, package)\n except AssertionError as err:\n failed_imports.append(package)\n LOGGER.error(f\"Failed to import package: {package}, output:\\n {err}\")\n if failed_imports:\n pytest.fail(f\"following packages are not import-able: {failed_imports}\")\n\n\ndef get_r_packages(package_helper: CondaPackageHelper) -> list[str]:\n \"\"\"Return a list of R packages\"\"\"\n return [\n get_package_import_name(pkg)\n for pkg in package_helper.requested_packages\n if is_r_package(pkg) and pkg not in EXCLUDED_PACKAGES\n ]\n\n\ndef test_r_packages(container: TrackedContainer) -> None:\n \"\"\"Test the import of specified R packages\"\"\"\n r_packages = get_r_packages(CondaPackageHelper(container))\n _check_import_packages(container, r_packages, check_import_r_package)\n\n\ndef get_python_packages(package_helper: CondaPackageHelper) -> list[str]:\n \"\"\"Return a list of Python packages\"\"\"\n return [\n get_package_import_name(pkg)\n for pkg in package_helper.requested_packages\n if not is_r_package(pkg) and pkg not in EXCLUDED_PACKAGES\n ]\n\n\ndef test_python_packages(container: TrackedContainer) -> None:\n \"\"\"Test the import of specified python packages\"\"\"\n python_packages = get_python_packages(CondaPackageHelper(container))\n _check_import_packages(container, python_packages, check_import_python_package)\n" }, { "path": "tests/by_image/docker-stacks-foundation/test_python_version.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\nEXPECTED_PYTHON_VERSION = \"3.13\"\n\n\ndef test_python_version(container: TrackedContainer) -> None:\n LOGGER.info(\n f\"Checking that python major.minor version is {EXPECTED_PYTHON_VERSION}\"\n )\n logs = container.run_and_wait(\n timeout=10,\n command=[\"python\", \"--version\"],\n )\n python = next(line for line in logs.splitlines() if line.startswith(\"Python \"))\n full_version = python.split()[1]\n major_minor_version = full_version[: full_version.rfind(\".\")]\n\n assert major_minor_version == EXPECTED_PYTHON_VERSION\n\n\ndef test_python_pinned_version(container: TrackedContainer) -> None:\n LOGGER.info(f\"Checking that pinned python version is {EXPECTED_PYTHON_VERSION}.*\")\n logs = container.run_and_wait(\n timeout=10,\n command=[\"cat\", \"/opt/conda/conda-meta/pinned\"],\n )\n assert f\"python {EXPECTED_PYTHON_VERSION}.*\" in logs\n" }, { "path": "tests/by_image/docker-stacks-foundation/test_run_hooks.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom __future__ import annotations\n\nimport logging\nfrom pathlib import Path\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\nTHIS_DIR = Path(__file__).parent.resolve()\n\n\ndef test_run_hooks_zero_args(container: TrackedContainer) -> None:\n stdout, stderr = container.run_and_wait(\n timeout=10,\n no_errors=False,\n no_failure=False,\n split_stderr=True,\n command=[\"bash\", \"-c\", \"source /usr/local/bin/run-hooks.sh\"],\n )\n assert not stdout\n assert \"Should pass exactly one directory\" in stderr\n\n\ndef test_run_hooks_two_args(container: TrackedContainer) -> None:\n stdout, stderr = container.run_and_wait(\n timeout=10,\n no_errors=False,\n no_failure=False,\n split_stderr=True,\n command=[\n \"bash\",\n \"-c\",\n \"source /usr/local/bin/run-hooks.sh first-arg second-arg\",\n ],\n )\n assert not stdout\n assert \"Should pass exactly one directory\" in stderr\n\n\ndef test_run_hooks_missing_dir(container: TrackedContainer) -> None:\n stdout, stderr = container.run_and_wait(\n timeout=10,\n no_errors=False,\n no_failure=False,\n split_stderr=True,\n command=[\n \"bash\",\n \"-c\",\n \"source /usr/local/bin/run-hooks.sh /tmp/missing-dir/\",\n ],\n )\n assert not stdout\n assert \"Directory /tmp/missing-dir/ doesn't exist or is not a directory\" in stderr\n\n\ndef test_run_hooks_dir_is_file(container: TrackedContainer) -> None:\n stdout, stderr = container.run_and_wait(\n timeout=10,\n no_errors=False,\n no_failure=False,\n split_stderr=True,\n command=[\n \"bash\",\n \"-c\",\n \"touch /tmp/some-file && source /usr/local/bin/run-hooks.sh /tmp/some-file\",\n ],\n )\n assert not stdout\n assert \"Directory /tmp/some-file doesn't exist or is not a directory\" in stderr\n\n\ndef test_run_hooks_empty_dir(container: TrackedContainer) -> None:\n stdout, stderr = container.run_and_wait(\n timeout=10,\n split_stderr=True,\n command=[\n \"bash\",\n \"-c\",\n \"mkdir /tmp/empty-dir && source /usr/local/bin/run-hooks.sh /tmp/empty-dir/\",\n ],\n )\n assert not stdout\n assert \"Running hooks in: /tmp/empty-dir/\" in stderr\n\n\ndef run_source_in_dir(\n container: TrackedContainer,\n *,\n subdir: str,\n command_suffix: str = \"\",\n no_errors: bool = True,\n no_failure: bool = True,\n) -> tuple[str, str]:\n host_data_dir = THIS_DIR / subdir\n cont_data_dir = \"/home/jovyan/data\"\n # https://forums.docker.com/t/all-files-appear-as-executable-in-file-paths-using-bind-mount/99921\n # Unfortunately, Docker treats all files in mounted dir as executable files\n # So we make a copy of the mounted dir inside a container\n command = (\n \"cp -r /home/jovyan/data/ /home/jovyan/data-copy/ &&\"\n \"source /usr/local/bin/run-hooks.sh /home/jovyan/data-copy/\" + command_suffix\n )\n return container.run_and_wait(\n timeout=10,\n volumes={host_data_dir: {\"bind\": cont_data_dir, \"mode\": \"ro\"}},\n no_errors=no_errors,\n no_failure=no_failure,\n split_stderr=True,\n command=[\"bash\", \"-c\", command],\n )\n\n\ndef test_run_hooks_change(container: TrackedContainer) -> None:\n stdout, logs = run_source_in_dir(container, subdir=\"data/run-hooks/change\")\n\n assert \"Inside a.sh MY_VAR variable has 123 value\" in stdout\n assert \"Inside b.sh MY_VAR variable has 123 value\" in stdout\n assert \"Changing value of MY_VAR\" in stdout\n assert \"After change inside b.sh MY_VAR variable has 456 value\" in stdout\n assert \"Inside c.sh MY_VAR variable has 456 value\" in stdout\n\n\ndef test_run_hooks_executables(container: TrackedContainer) -> None:\n stdout, logs = run_source_in_dir(\n container,\n subdir=\"data/run-hooks/executables\",\n command_suffix=\"&& echo SOME_VAR is ${SOME_VAR}\",\n )\n\n assert \"Executable python file was successfully run\" in stdout\n assert \"Ignoring non-executable: /home/jovyan/data-copy//non_executable.py\" in logs\n assert \"SOME_VAR is 123\" in stdout\n\n\ndef test_run_hooks_failures(container: TrackedContainer) -> None:\n stdout, logs = run_source_in_dir(\n container,\n subdir=\"data/run-hooks/failures\",\n no_errors=False,\n no_failure=False,\n )\n\n for file in [\"a.sh\", \"b.py\", \"c.sh\", \"d.sh\"]:\n assert f\"Started: {file}\" in stdout\n\n for file in [\"a.sh\"]:\n assert f\"Finished: {file}\" in stdout\n for file in [\"b.py\", \"c.sh\", \"d.sh\"]:\n assert f\"Finished: {file}\" not in stdout\n\n for file in [\"b.py\", \"c.sh\"]:\n assert (\n f\"/home/jovyan/data-copy//{file} has failed, continuing execution\" in logs\n )\n\n assert \"OTHER_VAR=456\" in stdout\n\n\ndef test_run_hooks_sh_files(container: TrackedContainer) -> None:\n stdout, _ = run_source_in_dir(container, subdir=\"data/run-hooks/sh-files\")\n\n assert \"Inside executable.sh MY_VAR variable has 0 value\" in stdout\n assert \"Inside non-executable.sh MY_VAR variable has 1 value\" in stdout\n\n\ndef test_run_hooks_unset(container: TrackedContainer) -> None:\n stdout, _ = run_source_in_dir(container, subdir=\"data/run-hooks/unset\")\n\n assert \"Inside a.sh MY_VAR variable has 123 value\" in stdout\n assert \"Inside b.sh MY_VAR variable has 123 value\" in stdout\n assert \"Unsetting MY_VAR\" in stdout\n assert \"Inside c.sh MY_VAR variable has value\" in stdout\n" }, { "path": "tests/by_image/docker-stacks-foundation/test_units.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\n\nfrom tests.hierarchy.get_test_dirs import get_test_dirs\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef test_units(container: TrackedContainer) -> None:\n \"\"\"Various units tests\n Add a py file in the `tests/by_image//units` dir, and it will be automatically tested\n \"\"\"\n image = container.image_name[container.image_name.rfind(\"/\") + 1 :]\n LOGGER.info(f\"Running unit tests for: {image}\")\n\n test_dirs = get_test_dirs(image)\n\n for test_dir in test_dirs:\n host_data_dir = test_dir / \"units\"\n LOGGER.info(f\"Searching for units tests in {host_data_dir}\")\n cont_data_dir = \"/home/jovyan/data\"\n\n LOGGER.info(f\"Units tests dir found: {host_data_dir.exists()}\")\n if not host_data_dir.exists():\n continue\n\n for host_file in host_data_dir.iterdir():\n cont_file = f\"{cont_data_dir}/{host_file.name}\"\n LOGGER.info(f\"Running unit test: {host_file}\")\n\n container.run_and_wait(\n timeout=30,\n volumes={host_file: {\"bind\": cont_file, \"mode\": \"ro\"}},\n command=[\"python\", cont_file],\n )\n" }, { "path": "tests/by_image/docker-stacks-foundation/test_user_options.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nimport pathlib\nimport time\n\nimport pytest # type: ignore\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef test_uid_change(container: TrackedContainer) -> None:\n \"\"\"Container should change the UID of the default user.\"\"\"\n logs = container.run_and_wait(\n timeout=120, # usermod is slow so give it some time\n user=\"root\",\n environment=[\"NB_UID=1010\"],\n command=[\"bash\", \"-c\", \"id && touch /opt/conda/test-file\"],\n )\n assert \"uid=1010(jovyan)\" in logs\n\n\ndef test_gid_change(container: TrackedContainer) -> None:\n \"\"\"Container should change the GID of the default user.\"\"\"\n logs = container.run_and_wait(\n timeout=20,\n user=\"root\",\n environment=[\"NB_GID=110\"],\n command=[\"id\"],\n )\n assert \"gid=110(jovyan)\" in logs\n assert \"groups=110(jovyan),100(users)\" in logs\n\n\ndef test_nb_user_change(container: TrackedContainer) -> None:\n \"\"\"Container should change the username (`NB_USER`) of the default user.\"\"\"\n nb_user = \"nayvoj\"\n container.run_detached(\n user=\"root\",\n environment=[f\"NB_USER={nb_user}\", \"CHOWN_HOME=yes\"],\n command=[\"sleep\", \"infinity\"],\n )\n\n # Give the chown time to complete.\n # Use sleep, not wait, because the container sleeps forever.\n time.sleep(1)\n LOGGER.info(f\"Checking if the user is changed to {nb_user} by the start script ...\")\n output = container.get_logs()\n assert \"ERROR\" not in output\n assert \"WARNING\" not in output\n assert (\n f\"username: jovyan -> {nb_user}\" in output\n ), f\"User is not changed to {nb_user}\"\n\n LOGGER.info(f\"Checking {nb_user} id ...\")\n command = \"id\"\n expected_output = f\"uid=1000({nb_user}) gid=100(users) groups=100(users)\"\n output = container.exec_cmd(command, user=nb_user, workdir=f\"/home/{nb_user}\")\n assert output == expected_output, f\"Bad user {output}, expected {expected_output}\"\n\n LOGGER.info(f\"Checking if {nb_user} owns his home folder ...\")\n command = f'stat -c \"%U %G\" /home/{nb_user}/'\n expected_output = f\"{nb_user} users\"\n output = container.exec_cmd(command, workdir=f\"/home/{nb_user}\")\n assert (\n output == expected_output\n ), f\"Bad owner for the {nb_user} home folder {output}, expected {expected_output}\"\n\n LOGGER.info(\n f\"Checking if a home folder of {nb_user} contains the 'work' folder with appropriate permissions ...\"\n )\n command = f'stat -c \"%F %U %G\" /home/{nb_user}/work'\n expected_output = f\"directory {nb_user} users\"\n output = container.exec_cmd(command, workdir=f\"/home/{nb_user}\")\n assert (\n output == expected_output\n ), f\"Folder work was not copied properly to {nb_user} home folder. stat: {output}, expected {expected_output}\"\n\n\ndef test_chown_extra(container: TrackedContainer) -> None:\n \"\"\"Container should change the UID/GID of a comma-separated\n CHOWN_EXTRA list of folders.\"\"\"\n logs = container.run_and_wait(\n timeout=120, # chown is slow so give it some time\n user=\"root\",\n environment=[\n \"NB_UID=1010\",\n \"NB_GID=101\",\n \"CHOWN_EXTRA=/home/jovyan,/opt/conda/bin\",\n \"CHOWN_EXTRA_OPTS=-R\",\n ],\n command=[\n \"stat\",\n \"-c\",\n \"%n:%u:%g\",\n \"/home/jovyan/.bashrc\",\n \"/opt/conda/bin/jupyter\",\n ],\n )\n assert \"/home/jovyan/.bashrc:1010:101\" in logs\n assert \"/opt/conda/bin/jupyter:1010:101\" in logs\n\n\ndef test_chown_home(container: TrackedContainer) -> None:\n \"\"\"Container should change the NB_USER home directory owner and\n group to the current value of NB_UID and NB_GID.\"\"\"\n logs = container.run_and_wait(\n timeout=120, # chown is slow so give it some time\n user=\"root\",\n environment=[\n \"CHOWN_HOME=yes\",\n \"CHOWN_HOME_OPTS=-R\",\n \"NB_USER=kitten\",\n \"NB_UID=1010\",\n \"NB_GID=101\",\n ],\n command=[\"stat\", \"-c\", \"%n:%u:%g\", \"/home/kitten/.bashrc\"],\n )\n assert \"/home/kitten/.bashrc:1010:101\" in logs\n\n\ndef test_sudo(container: TrackedContainer) -> None:\n \"\"\"Container should grant passwordless sudo to the default user.\"\"\"\n logs = container.run_and_wait(\n timeout=10,\n user=\"root\",\n environment=[\"GRANT_SUDO=yes\"],\n command=[\"sudo\", \"id\"],\n )\n assert \"uid=0(root)\" in logs\n\n\ndef test_sudo_path(container: TrackedContainer) -> None:\n \"\"\"Container should include /opt/conda/bin in the sudo secure_path.\"\"\"\n logs = container.run_and_wait(\n timeout=10,\n user=\"root\",\n environment=[\"GRANT_SUDO=yes\"],\n command=[\"sudo\", \"which\", \"jupyter\"],\n )\n assert logs.rstrip().endswith(\"/opt/conda/bin/jupyter\")\n\n\ndef test_sudo_path_without_grant(container: TrackedContainer) -> None:\n \"\"\"Container should include /opt/conda/bin in the sudo secure_path.\"\"\"\n logs = container.run_and_wait(\n timeout=10,\n user=\"root\",\n command=[\"which\", \"jupyter\"],\n )\n assert logs.rstrip().endswith(\"/opt/conda/bin/jupyter\")\n\n\ndef test_group_add(container: TrackedContainer) -> None:\n \"\"\"Container should run with the specified uid, gid, and secondary\n group. It won't be possible to modify /etc/passwd since gid is nonzero, so\n additionally verify that setting gid=0 is suggested in a warning.\n \"\"\"\n logs = container.run_and_wait(\n timeout=10,\n no_warnings=False,\n user=\"1010:1010\",\n group_add=[\"users\"], # Ensures write access to /home/jovyan\n command=[\"id\"],\n )\n warnings = TrackedContainer.get_warnings(logs)\n assert len(warnings) == 1\n assert \"Try setting gid=0\" in warnings[0]\n assert \"uid=1010 gid=1010 groups=1010,100(users)\" in logs\n\n\ndef test_set_uid(container: TrackedContainer) -> None:\n \"\"\"Container should run with the specified uid and NB_USER.\n The /home/jovyan directory will not be writable since it's owned by 1000:users.\n Additionally, verify that \"--group-add=users\" is suggested in a warning to restore\n write access.\n \"\"\"\n # This test needs to have tty disabled, the reason is explained here:\n # https://github.com/jupyter/docker-stacks/pull/2260#discussion_r2008821257\n logs = container.run_and_wait(\n timeout=10, no_warnings=False, user=\"1010\", command=[\"id\"], tty=False\n )\n assert \"uid=1010(jovyan) gid=0(root)\" in logs\n warnings = TrackedContainer.get_warnings(logs)\n assert len(warnings) == 1\n assert \"--group-add=users\" in warnings[0]\n\n\ndef test_set_uid_and_nb_user(container: TrackedContainer) -> None:\n \"\"\"Container should run with the specified uid and NB_USER.\"\"\"\n logs = container.run_and_wait(\n timeout=10,\n no_warnings=False,\n user=\"1010\",\n environment=[\"NB_USER=kitten\"],\n group_add=[\"users\"], # Ensures write access to /home/jovyan\n command=[\"id\"],\n )\n assert \"uid=1010(kitten) gid=0(root)\" in logs\n warnings = TrackedContainer.get_warnings(logs)\n assert len(warnings) == 1\n assert \"user is kitten but home is /home/jovyan\" in warnings[0]\n\n\ndef test_container_not_delete_bind_mount(\n container: TrackedContainer, tmp_path: pathlib.Path\n) -> None:\n \"\"\"Container should not delete host system files when using the (docker)\n -v bind mount flag and mapping to /home/jovyan.\n \"\"\"\n host_data_dir = tmp_path / \"data\"\n host_data_dir.mkdir()\n host_file = host_data_dir / \"foo.txt\"\n host_file.write_text(\"some-content\")\n\n container.run_and_wait(\n timeout=10,\n user=\"root\",\n working_dir=\"/home/\",\n environment=[\n \"NB_USER=user\",\n \"CHOWN_HOME=yes\",\n ],\n volumes={host_data_dir: {\"bind\": \"/home/jovyan/data\", \"mode\": \"rw\"}},\n command=[\"ls\"],\n )\n assert host_file.read_text() == \"some-content\"\n assert len(list(tmp_path.iterdir())) == 1\n\n\n@pytest.mark.parametrize(\"enable_root\", [False, True])\ndef test_jupyter_env_vars_to_unset(\n container: TrackedContainer, enable_root: bool\n) -> None:\n \"\"\"Environment variables names listed in JUPYTER_ENV_VARS_TO_UNSET\n should be unset in the final environment.\"\"\"\n root_args = {\"user\": \"root\"} if enable_root else {}\n logs = container.run_and_wait(\n timeout=10,\n environment=[\n \"JUPYTER_ENV_VARS_TO_UNSET=SECRET_ANIMAL,UNUSED_ENV,SECRET_FRUIT\",\n \"FRUIT=bananas\",\n \"SECRET_ANIMAL=cats\",\n \"SECRET_FRUIT=mango\",\n ],\n command=[\n \"bash\",\n \"-c\",\n \"echo I like ${FRUIT} and ${SECRET_FRUIT:-stuff}, and love ${SECRET_ANIMAL:-to keep secrets}!\",\n ],\n **root_args, # type: ignore\n )\n assert \"I like bananas and stuff, and love to keep secrets!\" in logs\n\n\ndef test_secure_path(container: TrackedContainer, tmp_path: pathlib.Path) -> None:\n \"\"\"Make sure that the sudo command has conda's python (not system's) on PATH.\n See .\n \"\"\"\n host_data_dir = tmp_path / \"data\"\n host_data_dir.mkdir()\n host_file = host_data_dir / \"wrong_python.sh\"\n host_file.write_text('#!/bin/bash\\necho \"Wrong python executable invoked!\"')\n host_file.chmod(0o755)\n\n logs = container.run_and_wait(\n timeout=10,\n user=\"root\",\n volumes={host_file: {\"bind\": \"/usr/bin/python\", \"mode\": \"ro\"}},\n command=[\"python\", \"--version\"],\n )\n assert \"Wrong python\" not in logs\n assert \"Python\" in logs\n\n\ndef test_startsh_multiple_exec(container: TrackedContainer) -> None:\n \"\"\"If start.sh is executed multiple times check that configuration only occurs once.\"\"\"\n logs = container.run_and_wait(\n timeout=10,\n no_warnings=False,\n user=\"root\",\n environment=[\"GRANT_SUDO=yes\"],\n command=[\"start.sh\", \"sudo\", \"id\"],\n )\n assert \"uid=0(root)\" in logs\n warnings = TrackedContainer.get_warnings(logs)\n assert len(warnings) == 1\n assert (\n \"WARNING: start.sh is the default ENTRYPOINT, do not include it in CMD\"\n in warnings[0]\n )\n\n\ndef test_rootless_triplet_change(container: TrackedContainer) -> None:\n \"\"\"Container should change the username (`NB_USER`), the UID and the GID of the default user.\"\"\"\n logs = container.run_and_wait(\n timeout=10,\n user=\"root\",\n environment=[\"NB_USER=root\", \"NB_UID=0\", \"NB_GID=0\"],\n command=[\"id\"],\n )\n assert \"uid=0(root)\" in logs\n assert \"gid=0(root)\" in logs\n assert \"groups=0(root)\" in logs\n\n\ndef test_rootless_triplet_home(container: TrackedContainer) -> None:\n \"\"\"Container should change the home directory for triplet NB_USER=root, NB_UID=0, NB_GID=0.\"\"\"\n logs = container.run_and_wait(\n timeout=10,\n user=\"root\",\n environment=[\"NB_USER=root\", \"NB_UID=0\", \"NB_GID=0\"],\n command=[\"bash\", \"-c\", \"echo HOME=${HOME} && getent passwd root\"],\n )\n assert \"HOME=/home/root\" in logs\n assert \"root:x:0:0:root:/home/root:/bin/bash\" in logs\n\n\ndef test_rootless_triplet_sudo(container: TrackedContainer) -> None:\n \"\"\"Container should not be started with sudo for triplet NB_USER=root, NB_UID=0, NB_GID=0.\"\"\"\n logs = container.run_and_wait(\n timeout=10,\n user=\"root\",\n environment=[\"NB_USER=root\", \"NB_UID=0\", \"NB_GID=0\"],\n command=[\"env\"],\n )\n assert \"SUDO\" not in logs\n\n\ndef test_log_stderr(container: TrackedContainer) -> None:\n \"\"\"Logs should go to stderr, not stdout\"\"\"\n stdout, stderr = container.run_and_wait(\n timeout=10,\n user=\"root\",\n environment=[\"NB_USER=root\", \"NB_UID=0\", \"NB_GID=0\"],\n command=[\"echo\", \"stdout\"],\n split_stderr=True,\n )\n # no logs should be on stdout\n assert stdout.strip() == \"stdout\"\n # check that logs were captured\n assert \"Entered start.sh\" in stderr\n assert \"Running as root\" in stderr\n" }, { "path": "tests/by_image/julia-notebook/test_julia.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom tests.utils.tracked_container import TrackedContainer\n\n\ndef test_julia(container: TrackedContainer) -> None:\n container.run_and_wait(timeout=10, command=[\"julia\", \"--version\"])\n" }, { "path": "tests/by_image/julia-notebook/test_pluto.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport requests\n\nfrom tests.shared_checks.pluto_check import check_pluto_proxy\nfrom tests.utils.tracked_container import TrackedContainer\n\n\ndef test_pluto_proxy(\n container: TrackedContainer, http_client: requests.Session, free_host_port: int\n) -> None:\n \"\"\"Pluto proxy starts Pluto correctly\"\"\"\n check_pluto_proxy(container, http_client, free_host_port)\n" }, { "path": "tests/by_image/minimal-notebook/data/notebook_math.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"id\": \"0\",\n \"metadata\": {},\n \"source\": [\n \"# A simple SymPy example\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"1\",\n \"metadata\": {},\n \"source\": [\n \"First we import SymPy and initialize printing:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"2\",\n \"metadata\": {\n \"jupyter\": {\n \"outputs_hidden\": false\n }\n },\n \"outputs\": [],\n \"source\": [\n \"from sympy import diff, init_printing, integrate, sin, symbols\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"3\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"init_printing()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"4\",\n \"metadata\": {},\n \"source\": [\n \"Create a few symbols:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"5\",\n \"metadata\": {\n \"jupyter\": {\n \"outputs_hidden\": false\n }\n },\n \"outputs\": [],\n \"source\": [\n \"x, y, z = symbols(\\\"x y z\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"id\": \"6\",\n \"metadata\": {},\n \"source\": [\n \"Here is a basic expression:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"7\",\n \"metadata\": {\n \"jupyter\": {\n \"outputs_hidden\": false\n }\n },\n \"outputs\": [],\n \"source\": [\n \"e = x**2 + 2.0 * y + sin(z)\\n\",\n \"e\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"8\",\n \"metadata\": {\n \"jupyter\": {\n \"outputs_hidden\": false\n }\n },\n \"outputs\": [],\n \"source\": [\n \"diff(e, x)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"9\",\n \"metadata\": {\n \"jupyter\": {\n \"outputs_hidden\": false\n }\n },\n \"outputs\": [],\n \"source\": [\n \"integrate(e, z)\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.9.10\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n" }, { "path": "tests/by_image/minimal-notebook/data/notebook_svg.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from IPython.display import SVG, display\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"display(SVG(filename=\\\"Jupyter_logo.svg\\\"))\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.9.10\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n" }, { "path": "tests/by_image/minimal-notebook/test_nbconvert.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nfrom pathlib import Path\n\nimport pytest # type: ignore\n\nfrom tests.shared_checks.nbconvert_check import check_nbconvert\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\nTHIS_DIR = Path(__file__).parent.resolve()\n\n\n@pytest.mark.parametrize(\"test_file\", [\"notebook_math\", \"notebook_svg\"])\n@pytest.mark.parametrize(\"output_format\", [\"pdf\", \"html\", \"markdown\"])\ndef test_nbconvert(\n container: TrackedContainer, test_file: str, output_format: str\n) -> None:\n host_data_file = THIS_DIR / \"data\" / f\"{test_file}.ipynb\"\n check_nbconvert(container, host_data_file, output_format, execute=False)\n" }, { "path": "tests/by_image/pyspark-notebook/data/issue_1168.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"This is a test for the issue [#1168](https://github.com/jupyter/docker-stacks/issues/1168)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from pyspark.sql import SparkSession\\n\",\n \"\\n\",\n \"# Spark session & context\\n\",\n \"spark = SparkSession.builder.master(\\\"local\\\").getOrCreate()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"df = spark.createDataFrame([(1, 21), (2, 30)], (\\\"id\\\", \\\"age\\\"))\\n\",\n \"\\n\",\n \"\\n\",\n \"def filter_func(iterator):\\n\",\n \" for pdf in iterator:\\n\",\n \" yield pdf[pdf.id == 1]\\n\",\n \"\\n\",\n \"\\n\",\n \"df.mapInPandas(filter_func, df.schema).show()\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.9.10\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n" }, { "path": "tests/by_image/pyspark-notebook/data/local_pyspark.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from pyspark.sql import SparkSession\\n\",\n \"\\n\",\n \"# Spark session & context\\n\",\n \"spark = SparkSession.builder.master(\\\"local\\\").getOrCreate()\\n\",\n \"sc = spark.sparkContext\\n\",\n \"\\n\",\n \"# Sum of the first 100 whole numbers\\n\",\n \"rdd = sc.parallelize(range(100 + 1))\\n\",\n \"rdd.sum()\\n\",\n \"# 5050\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3 (ipykernel)\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.9.10\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 4\n}\n" }, { "path": "tests/by_image/pyspark-notebook/test_spark.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef test_spark_shell(container: TrackedContainer) -> None:\n \"\"\"Checking if Spark (spark-shell) is running properly\"\"\"\n logs = container.run_and_wait(\n timeout=60,\n no_warnings=False,\n command=[\"bash\", \"-c\", 'spark-shell <<< \"1+1\"'],\n )\n warnings = TrackedContainer.get_warnings(logs)\n assert warnings == [\"WARNING: Using incubator modules: jdk.incubator.vector\"]\n assert \"res0: Int = 2\" in logs, \"spark-shell does not work\"\n" }, { "path": "tests/by_image/pyspark-notebook/test_spark_nbconvert.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nfrom pathlib import Path\n\nimport pytest # type: ignore\n\nfrom tests.shared_checks.nbconvert_check import check_nbconvert\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\nTHIS_DIR = Path(__file__).parent.resolve()\n\n\n@pytest.mark.parametrize(\"test_file\", [\"issue_1168\", \"local_pyspark\"])\n@pytest.mark.parametrize(\"output_format\", [\"pdf\", \"html\", \"markdown\"])\ndef test_spark_nbconvert(\n container: TrackedContainer, test_file: str, output_format: str\n) -> None:\n host_data_file = THIS_DIR / \"data\" / f\"{test_file}.ipynb\"\n logs = check_nbconvert(\n container, host_data_file, output_format, execute=True, no_warnings=False\n )\n\n warnings = TrackedContainer.get_warnings(logs)\n assert warnings == [\"WARNING: Using incubator modules: jdk.incubator.vector\"]\n" }, { "path": "tests/by_image/pyspark-notebook/units/unit_pandas_version.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport pandas\n\nassert pandas.__version__ == \"2.2.3\"\n" }, { "path": "tests/by_image/pyspark-notebook/units/unit_spark.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport pyspark # noqa: F401\n" }, { "path": "tests/by_image/pytorch-notebook/units/unit_pytorch.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport torch\n\nprint(torch.tensor([[1.0, 4.0, 7.0], [4.0, 9.0, 11.0]]))\n\n# Check if GPU is available\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n# Create large tensors\nsize = 10000\ntorch.randn(size, size, device=device)\n" }, { "path": "tests/by_image/r-notebook/test_R_mimetypes.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom tests.shared_checks.R_mimetype_check import check_r_mimetypes\nfrom tests.utils.tracked_container import TrackedContainer\n\n\ndef test_mimetypes(container: TrackedContainer) -> None:\n \"\"\"Check if Rscript command for mimetypes can be executed\"\"\"\n check_r_mimetypes(container)\n" }, { "path": "tests/by_image/scipy-notebook/data/cython/helloworld.pyx", "content": "print(\"Hello World\")\n" }, { "path": "tests/by_image/scipy-notebook/data/cython/setup.py", "content": "# These lines are not sorted by isort on purpose\n# see: https://stackoverflow.com/a/53356077/4881441\nfrom setuptools import setup # isort:skip\nfrom Cython.Build import cythonize # isort:skip\n\nsetup(ext_modules=cythonize(\"helloworld.pyx\"))\n" }, { "path": "tests/by_image/scipy-notebook/data/matplotlib/matplotlib_1.py", "content": "# type: ignore\n# Matplotlib: Create a simple plot example.\n# Refs: https://matplotlib.org/stable/gallery/lines_bars_and_markers/simple_plot.html\n\n# Optional test with [Matplotlib Jupyter Integration](https://github.com/matplotlib/ipympl)\n# %matplotlib widget\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n# Data for plotting\nt = np.arange(0.0, 2.0, 0.01)\ns = 1 + np.sin(2 * np.pi * t)\n\nfig, ax = plt.subplots()\nax.plot(t, s)\n\nax.set(\n xlabel=\"time (s)\",\n ylabel=\"voltage (mV)\",\n title=\"About as simple as it gets, folks\",\n)\nax.grid()\n\n# Note that the test can be run headless by checking if an image is produced\nfile_path = \"/tmp/test.png\"\nfig.savefig(file_path)\nprint(f\"File {file_path} saved\")\n" }, { "path": "tests/by_image/scipy-notebook/data/matplotlib/matplotlib_fonts_1.py", "content": "# Matplotlib: Test tex fonts\nimport matplotlib\nimport matplotlib.pyplot as plt\n\nmatplotlib.rcParams[\"pgf.texsystem\"] = \"pdflatex\"\nmatplotlib.rcParams.update(\n {\n \"font.family\": \"serif\",\n \"font.size\": 18,\n \"axes.labelsize\": 20,\n \"axes.titlesize\": 24,\n \"figure.titlesize\": 28,\n }\n)\nmatplotlib.rcParams[\"text.usetex\"] = True\n\nfig, ax = plt.subplots(1, 1)\nx = [1, 2]\ny = [1, 2]\nax.plot(x, y, label=\"a label\")\nax.legend(fontsize=15)\n\nfile_path = \"/tmp/test_fonts.png\"\nfig.savefig(file_path)\nprint(f\"File {file_path} saved\")\n" }, { "path": "tests/by_image/scipy-notebook/test_cython.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom pathlib import Path\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nTHIS_DIR = Path(__file__).parent.resolve()\n\n\ndef test_cython(container: TrackedContainer) -> None:\n host_data_dir = THIS_DIR / \"data/cython\"\n cont_data_dir = \"/home/jovyan/data\"\n\n logs = container.run_and_wait(\n timeout=10,\n volumes={host_data_dir: {\"bind\": cont_data_dir, \"mode\": \"ro\"}},\n command=[\n \"bash\",\n \"-c\",\n # We copy our data to a temporary folder to be able to modify the directory\n f\"cp -r {cont_data_dir}/ /tmp/test/ && cd /tmp/test && python3 setup.py build_ext\",\n ],\n )\n assert \"building 'helloworld' extension\" in logs\n" }, { "path": "tests/by_image/scipy-notebook/test_extensions.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\n\nimport pytest # type: ignore\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\n@pytest.mark.skip(reason=\"Not yet compliant with JupyterLab 4\")\n@pytest.mark.parametrize(\n \"extension\",\n [\n \"@bokeh/jupyter_bokeh\",\n \"@jupyter-widgets/jupyterlab-manager\",\n \"jupyter-matplotlib\",\n ],\n)\ndef test_check_extension(container: TrackedContainer, extension: str) -> None:\n \"\"\"Basic check of each extension\n\n The list of installed extensions can be obtained through this command:\n\n $ jupyter labextension list\n\n \"\"\"\n LOGGER.info(f\"Checking the extension: {extension} ...\")\n container.run_and_wait(\n timeout=10,\n command=[\"jupyter\", \"labextension\", \"check\", extension],\n )\n" }, { "path": "tests/by_image/scipy-notebook/test_matplotlib.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nfrom pathlib import Path\n\nimport pytest # type: ignore\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\nTHIS_DIR = Path(__file__).parent.resolve()\n\n\n@pytest.mark.parametrize(\n \"test_file,expected_file,description\",\n [\n (\n \"matplotlib_1.py\",\n \"test.png\",\n \"Test that matplotlib can plot a graph and write it as an image ...\",\n ),\n (\n \"matplotlib_fonts_1.py\",\n \"test_fonts.png\",\n \"Test cm-super latex labels in matplotlib ...\",\n ),\n ],\n)\ndef test_matplotlib(\n container: TrackedContainer, test_file: str, expected_file: str, description: str\n) -> None:\n \"\"\"Various tests performed on matplotlib\n\n - Test that matplotlib is able to plot a graph and write it as an image\n - Test matplotlib latex fonts, which depend on the cm-super package\n \"\"\"\n host_file = THIS_DIR / \"data/matplotlib\" / test_file\n cont_file = f\"/home/jovyan/data/{test_file}\"\n output_dir = \"/tmp\"\n LOGGER.info(description)\n container.run_detached(\n volumes={host_file: {\"bind\": cont_file, \"mode\": \"ro\"}},\n command=[\"sleep\", \"infinity\"],\n )\n container.exec_cmd(f\"python {cont_file}\")\n\n # Checking if the file is generated\n # https://stackoverflow.com/a/15895594/4413446\n command = f\"test -s {output_dir}/{expected_file}\"\n container.exec_cmd(command)\n" }, { "path": "tests/by_image/scipy-notebook/units/unit_pandas.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n# type: ignore\nimport numpy as np\nimport pandas as pd\n\nnp.random.seed(0)\nprint(pd.Series(np.random.randint(0, 7, size=10)).sum())\n" }, { "path": "tests/by_image/tensorflow-notebook/units/unit_tensorflow.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport os\nimport sys\n\nif \"NVIDIA_VISIBLE_DEVICES\" in os.environ:\n print(\"Not running this test in GPU mode\")\n sys.exit(0)\n\nimport tensorflow as tf\n\nprint(tf.constant(\"Hello, TensorFlow\"))\nprint(tf.reduce_sum(tf.random.normal([1000, 1000])))\n" }, { "path": "tests/conftest.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nimport socket\nfrom collections.abc import Generator\nfrom contextlib import closing\n\nimport docker\nimport pytest # type: ignore\nimport requests\nfrom requests.adapters import HTTPAdapter\nfrom urllib3.util.retry import Retry\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\n@pytest.fixture(scope=\"session\")\ndef http_client() -> requests.Session:\n \"\"\"Requests session with retries and backoff.\"\"\"\n s = requests.Session()\n retries = Retry(total=5, backoff_factor=1)\n s.mount(\"http://\", HTTPAdapter(max_retries=retries))\n s.mount(\"https://\", HTTPAdapter(max_retries=retries))\n return s\n\n\n@pytest.fixture(scope=\"session\")\ndef docker_client() -> docker.DockerClient:\n \"\"\"Docker client configured based on the host environment\"\"\"\n client = docker.from_env()\n LOGGER.debug(f\"Docker client created: {client.version()}\")\n return client\n\n\ndef pytest_addoption(parser: pytest.Parser) -> None:\n \"\"\"Add custom command-line options to pytest.\"\"\"\n parser.addoption(\n \"--registry\",\n required=True,\n choices=[\"docker.io\", \"quay.io\"],\n help=\"Image registry\",\n )\n parser.addoption(\n \"--owner\",\n required=True,\n help=\"Owner of the image\",\n )\n parser.addoption(\n \"--image\",\n required=True,\n help=\"Short image name\",\n )\n\n\n@pytest.fixture(scope=\"session\")\ndef image_name(request: pytest.FixtureRequest) -> str:\n \"\"\"Image name to test\"\"\"\n\n def option(name: str) -> str:\n value = request.config.getoption(name)\n assert isinstance(value, str)\n return value\n\n return f\"{option('--registry')}/{option('--owner')}/{option('--image')}\"\n\n\n@pytest.fixture(scope=\"function\")\ndef container(\n docker_client: docker.DockerClient, image_name: str\n) -> Generator[TrackedContainer]:\n \"\"\"Notebook container with initial configuration appropriate for testing\n (e.g., HTTP port exposed to the host for HTTP calls).\n\n Yields the container instance and kills it when the caller is done with it.\n \"\"\"\n container = TrackedContainer(\n docker_client,\n image_name,\n )\n yield container\n container.remove()\n\n\n@pytest.fixture(scope=\"function\")\ndef free_host_port() -> Generator[int]:\n \"\"\"Finds a free port on the host machine\"\"\"\n with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:\n s.bind((\"\", 0))\n s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n yield s.getsockname()[1]\n" }, { "path": "tests/hierarchy/__init__.py", "content": "" }, { "path": "tests/hierarchy/get_test_dirs.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom pathlib import Path\n\nfrom tests.hierarchy.images_hierarchy import IMAGE_PARENT\n\nTHIS_DIR = Path(__file__).parent.resolve()\nIMAGE_SPECIFIC_TESTS_DIR = THIS_DIR.parent / \"by_image\"\n\nassert IMAGE_SPECIFIC_TESTS_DIR.exists(), f\"{IMAGE_SPECIFIC_TESTS_DIR} does not exist.\"\n\n\ndef get_test_dirs(image: str | None) -> list[Path]:\n if image is None:\n return []\n\n test_dirs = get_test_dirs(IMAGE_PARENT[image])\n current_test_dir = IMAGE_SPECIFIC_TESTS_DIR / image\n assert current_test_dir.exists(), f\"{current_test_dir} does not exist.\"\n test_dirs.append(current_test_dir)\n return test_dirs\n" }, { "path": "tests/hierarchy/images_hierarchy.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# Please, take a look at the hierarchy of the images here:\n# https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships\nIMAGE_PARENT = {\n \"docker-stacks-foundation\": None,\n \"base-notebook\": \"docker-stacks-foundation\",\n \"minimal-notebook\": \"base-notebook\",\n \"scipy-notebook\": \"minimal-notebook\",\n \"r-notebook\": \"minimal-notebook\",\n \"julia-notebook\": \"minimal-notebook\",\n \"tensorflow-notebook\": \"scipy-notebook\",\n \"pytorch-notebook\": \"scipy-notebook\",\n \"datascience-notebook\": \"scipy-notebook\",\n \"pyspark-notebook\": \"scipy-notebook\",\n \"all-spark-notebook\": \"pyspark-notebook\",\n}\n" }, { "path": "tests/pytest.ini", "content": "[pytest]\naddopts = -ra --color=yes\nlog_cli = 1\nlog_cli_level = INFO\nlog_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)\nlog_cli_date_format=%Y-%m-%d %H:%M:%S\nmarkers =\n info: marks tests as info (deselect with '-m \"not info\"')\n" }, { "path": "tests/run_tests.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport argparse\nimport logging\n\nimport plumbum\n\nfrom tests.hierarchy.get_test_dirs import get_test_dirs\n\npython3 = plumbum.local[\"python3\"]\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef test_image(*, registry: str, owner: str, image: str) -> None:\n LOGGER.info(f\"Testing image: {image}\")\n test_dirs = get_test_dirs(image)\n LOGGER.info(f\"Test dirs to be run: {test_dirs}\")\n (\n python3[\n \"-m\",\n \"pytest\",\n \"--numprocesses\",\n \"auto\",\n \"-m\",\n \"not info\",\n test_dirs,\n \"--registry\",\n registry,\n \"--owner\",\n owner,\n \"--image\",\n image,\n ]\n & plumbum.FG\n )\n\n\nif __name__ == \"__main__\":\n logging.basicConfig(level=logging.INFO)\n\n arg_parser = argparse.ArgumentParser()\n arg_parser.add_argument(\n \"--registry\",\n required=True,\n choices=[\"docker.io\", \"quay.io\"],\n help=\"Image registry\",\n )\n arg_parser.add_argument(\n \"--owner\",\n required=True,\n help=\"Owner of the image\",\n )\n arg_parser.add_argument(\n \"--image\",\n required=True,\n help=\"Short image name\",\n )\n args = arg_parser.parse_args()\n\n test_image(**vars(args))\n" }, { "path": "tests/shared_checks/R_mimetype_check.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef check_r_mimetypes(container: TrackedContainer) -> None:\n \"\"\"Check if Rscript command can be executed\"\"\"\n LOGGER.info(\"Test that R command can be executed ...\")\n R_MIMETYPES_CHECK_CMD = 'if (length(getOption(\"jupyter.plot_mimetypes\")) != 5) {stop(\"missing jupyter.plot_mimetypes\")}'\n command = [\"Rscript\", \"-e\", R_MIMETYPES_CHECK_CMD]\n logs = container.run_and_wait(timeout=10, command=command)\n LOGGER.debug(f\"{logs=}\")\n # If there is any output after this it means there was an error\n assert logs.splitlines()[-1] == \"Executing the command: \" + \" \".join(\n command\n ), f\"Command {R_MIMETYPES_CHECK_CMD=} failed\"\n" }, { "path": "tests/shared_checks/__init__.py", "content": "" }, { "path": "tests/shared_checks/nbconvert_check.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nfrom pathlib import Path\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef check_nbconvert(\n container: TrackedContainer,\n host_file: Path,\n output_format: str,\n *,\n execute: bool,\n no_warnings: bool = True,\n) -> str:\n \"\"\"Check if nbconvert is able to convert a notebook file\"\"\"\n cont_data_file = \"/home/jovyan/\" + host_file.name\n\n output_dir = \"/tmp\"\n LOGGER.info(\n f\"Test that the example notebook {host_file.name} can be converted to {output_format} ...\"\n )\n command = [\n \"jupyter\",\n \"nbconvert\",\n cont_data_file,\n \"--output-dir\",\n output_dir,\n \"--to\",\n output_format,\n ]\n if execute:\n conversion_timeout_ms = 5000\n command += [\n \"--execute\",\n f\"--ExecutePreprocessor.timeout={conversion_timeout_ms}\",\n ]\n logs = container.run_and_wait(\n timeout=60,\n volumes={host_file: {\"bind\": cont_data_file, \"mode\": \"ro\"}},\n command=command,\n no_warnings=no_warnings,\n )\n output_ext = \"md\" if output_format == \"markdown\" else output_format\n expected_file = f\"{output_dir}/{host_file.stem}.{output_ext}\"\n assert expected_file in logs, f\"Expected file {expected_file} not generated\"\n\n return logs\n" }, { "path": "tests/shared_checks/pluto_check.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nimport secrets\nimport time\n\nimport requests\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\ndef check_pluto_proxy(\n container: TrackedContainer, http_client: requests.Session, free_host_port: int\n) -> None:\n token = secrets.token_hex()\n container.run_detached(\n command=[\n \"start-notebook.py\",\n f\"--IdentityProvider.token={token}\",\n ],\n ports={\"8888/tcp\": free_host_port},\n )\n # Give the server a bit of time to start\n time.sleep(2)\n resp = http_client.get(f\"http://localhost:{free_host_port}/pluto?token={token}\")\n resp.raise_for_status()\n assert \"Pluto.jl notebooks\" in resp.text, \"Pluto.jl text not found in /pluto page\"\n" }, { "path": "tests/utils/__init__.py", "content": "" }, { "path": "tests/utils/conda_package_helper.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\n\n# CondaPackageHelper is partially based on the work https://oerpli.github.io/post/2019/06/conda-outdated/.\n# See copyright below.\n#\n# MIT License\n# Copyright (c) 2019 Abraham Hinteregger\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n# The above copyright notice and this permission notice shall be included in all\n# copies or substantial portions of the Software.\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n\nimport json\nimport logging\nimport re\nfrom collections import defaultdict\nfrom functools import cached_property\nfrom itertools import chain\n\nfrom tabulate import tabulate\n\nfrom tests.utils.tracked_container import TrackedContainer\n\nLOGGER = logging.getLogger(__name__)\n\n\nclass CondaPackageHelper:\n \"\"\"Conda package helper permitting to get information about packages\"\"\"\n\n def __init__(self, container: TrackedContainer):\n self.container = container\n self.container.run_detached(command=[\"sleep\", \"infinity\"])\n\n @cached_property\n def installed_packages(self) -> dict[str, set[str]]:\n \"\"\"Return the installed packages\"\"\"\n LOGGER.info(\"Grabbing the list of installed packages ...\")\n env_export = self.container.exec_cmd(\"mamba env export --no-build --json\")\n return self._parse_package_versions(env_export)\n\n @cached_property\n def requested_packages(self) -> dict[str, set[str]]:\n \"\"\"Return the requested package (i.e. `mamba install `)\"\"\"\n LOGGER.info(\"Grabbing the list of manually requested packages ...\")\n env_export = self.container.exec_cmd(\n \"mamba env export --no-build --json --from-history\"\n )\n return self._parse_package_versions(env_export)\n\n @staticmethod\n def _parse_package_versions(env_export: str) -> dict[str, set[str]]:\n \"\"\"Extract packages and versions from the lines returned by the list of specifications\"\"\"\n try:\n dependencies = json.loads(env_export).get(\"dependencies\")\n except json.JSONDecodeError:\n # Try to fix the invalid JSON from mamba (bug in mamba 2.x)\n # which doesn't escape double quotes in version specs\n # e.g. \"protobuf[version=\">=5.28.3,<6\"]\"\n fixed_export = re.sub(r'(\\[[^\\]]*=)\"([^\"]+)\"(\\])', r\"\\1'\\2'\\3\", env_export)\n dependencies = json.loads(fixed_export).get(\"dependencies\")\n\n # Filtering packages installed through pip\n # since we only manage packages installed through mamba here\n # They are represented by a dict with a key 'pip'\n dependencies = filter(lambda x: isinstance(x, str), dependencies)\n packages_dict: dict[str, set[str]] = {}\n for dependency in dependencies:\n # If it's a package with [] notation, we strip the [] part for the package name\n # but we keep it if we want to try to extract version (not needed for these tests)\n package_with_name = dependency.split(\"[\")[0]\n split = re.split(\"=?=\", package_with_name)\n\n # default values\n package = split[0]\n version = set()\n # This normally means we have package=version notation\n if len(split) > 1:\n # checking if it's a proper version by testing if the first char is a digit\n if split[1][0].isdigit():\n # package + version case\n version = set(split[1:])\n # The split was incorrect and the package shall not be split\n else:\n package = f\"{split[0]}={split[1]}\"\n packages_dict[package] = version\n return packages_dict\n\n @cached_property\n def available_packages(self) -> dict[str, set[str]]:\n \"\"\"Return the available packages\"\"\"\n LOGGER.info(\"Grabbing the list of available packages (can take a while) ...\")\n return self._extract_available(\n self.container.exec_cmd(\"conda search --outdated --quiet\")\n )\n\n @staticmethod\n def _extract_available(lines: str) -> defaultdict[str, set[str]]:\n \"\"\"Extract packages and versions from the lines returned by the list of packages\"\"\"\n ddict = defaultdict(set)\n for line in lines.splitlines()[2:]:\n match = re.match(r\"^(\\S+)\\s+(\\S+)\", line, re.MULTILINE)\n assert match is not None\n pkg, version = match.groups()\n ddict[pkg].add(version)\n return ddict\n\n def find_updatable_packages(self, requested_only: bool) -> list[dict[str, str]]:\n \"\"\"Check the updatable packages including or not dependencies\"\"\"\n updatable = []\n for pkg, inst_vs in self.installed_packages.items():\n avail_vs = self.available_packages[pkg]\n if not avail_vs or (requested_only and pkg not in self.requested_packages):\n continue\n newest = max(avail_vs, key=CondaPackageHelper.semantic_cmp)\n current = min(inst_vs, key=CondaPackageHelper.semantic_cmp)\n if CondaPackageHelper.semantic_cmp(\n current\n ) < CondaPackageHelper.semantic_cmp(newest):\n updatable.append({\"Package\": pkg, \"Current\": current, \"Newest\": newest})\n return updatable\n\n @staticmethod\n def semantic_cmp(version_string: str) -> tuple[int, ...]:\n \"\"\"Manage semantic versioning for comparison\"\"\"\n\n def my_split(string: str) -> list[list[str]]:\n def version_substrs(x: str) -> list[str]:\n return re.findall(r\"([A-Za-z]+|\\d+)\", x)\n\n return list(chain(map(version_substrs, string.split(\".\"))))\n\n def str_ord(string: str) -> int:\n num = 0\n for char in string:\n num *= 255\n num += ord(char)\n return num\n\n def try_int(version_str: str) -> int:\n try:\n return int(version_str)\n except ValueError:\n return str_ord(version_str)\n\n mss = list(chain(*my_split(version_string)))\n return tuple(map(try_int, mss))\n\n def get_outdated_summary(\n self, updatable: list[dict[str, str]], requested_only: bool\n ) -> str:\n \"\"\"Return a summary of outdated packages\"\"\"\n packages = (\n self.requested_packages if requested_only else self.installed_packages\n )\n nb_packages = len(packages)\n nb_updatable = len(updatable)\n updatable_ratio = nb_updatable / nb_packages\n return f\"{nb_updatable}/{nb_packages} ({updatable_ratio:.0%}) packages could be updated\"\n\n def get_outdated_table(self, updatable: list[dict[str, str]]) -> str:\n \"\"\"Return a table of outdated packages\"\"\"\n return tabulate(updatable, headers=\"keys\")\n" }, { "path": "tests/utils/tracked_container.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport logging\nfrom typing import Any, Literal, LiteralString, overload\n\nimport docker\nfrom docker.models.containers import Container\n\nLOGGER = logging.getLogger(__name__)\n\n\nclass TrackedContainer:\n \"\"\"Wrapper that collects docker container configuration and delays\n container creation/execution.\n\n Parameters\n ----------\n docker_client: docker.DockerClient\n Docker client instance\n image_name: str\n Name of the docker image to launch\n \"\"\"\n\n def __init__(\n self,\n docker_client: docker.DockerClient,\n image_name: str,\n ):\n self.container: Container | None = None\n self.docker_client: docker.DockerClient = docker_client\n self.image_name: str = image_name\n\n def run_detached(self, **kwargs: Any) -> None:\n \"\"\"Runs a docker container using the pre-configured image name\n and a mix of the pre-configured container options and those passed\n to this method.\n\n Keeps track of the docker.Container instance spawned to kill it\n later.\n\n Parameters\n ----------\n **kwargs: dict, optional\n Keyword arguments to pass to docker.DockerClient.containers.run\n extending and/or overriding key/value pairs passed to the constructor\n \"\"\"\n LOGGER.info(\n f\"Creating a container for the image: {self.image_name} with args: {kwargs} ...\"\n )\n default_kwargs = {\"detach\": True, \"tty\": True}\n final_kwargs = default_kwargs | kwargs\n self.container = self.docker_client.containers.run(\n self.image_name, **final_kwargs\n )\n LOGGER.info(f\"Container {self.container.name} created\")\n\n def get_logs(self, *, stdout: bool = True, stderr: bool = True) -> str:\n assert self.container is not None\n logs = self.container.logs(stdout=stdout, stderr=stderr).decode()\n assert isinstance(logs, str)\n return logs\n\n def get_health(self) -> str:\n assert self.container is not None\n self.container.reload()\n return self.container.health # type: ignore\n\n def exec_cmd(self, cmd: str, **kwargs: Any) -> str:\n assert self.container is not None\n container = self.container\n\n LOGGER.info(f\"Running cmd: `{cmd}` on container: {container.name}\")\n default_kwargs = {\"tty\": True}\n final_kwargs = default_kwargs | kwargs\n exec_result = container.exec_run(cmd, **final_kwargs)\n output = exec_result.output.decode().rstrip()\n assert isinstance(output, str)\n if exec_result.exit_code != 0:\n LOGGER.error(f\"Command output:\\n{output}\")\n raise AssertionError(f\"Command: `{cmd}` failed\")\n else:\n LOGGER.debug(f\"Command output:\\n{output}\")\n return output\n\n @overload\n def run_and_wait(\n self,\n timeout: int,\n *,\n no_warnings: bool = True,\n no_errors: bool = True,\n no_failure: bool = True,\n split_stderr: Literal[True],\n **kwargs: Any,\n ) -> tuple[str, str]: ...\n\n @overload\n def run_and_wait(\n self,\n timeout: int,\n *,\n no_warnings: bool = True,\n no_errors: bool = True,\n no_failure: bool = True,\n split_stderr: Literal[False] = False,\n **kwargs: Any,\n ) -> str: ...\n\n def run_and_wait(\n self,\n timeout: int,\n *,\n no_warnings: bool = True,\n no_errors: bool = True,\n no_failure: bool = True,\n split_stderr: bool = False,\n **kwargs: Any,\n ) -> str | tuple[str, str]:\n if split_stderr:\n kwargs.setdefault(\"tty\", False)\n assert kwargs[\"tty\"] is False, \"split_stderr only works with tty=False\"\n self.run_detached(**kwargs)\n assert self.container is not None\n rv = self.container.wait(timeout=timeout)\n stdout: str\n stderr: str\n if split_stderr:\n stdout = self.get_logs(stdout=True, stderr=False)\n stderr = logs = self.get_logs(stdout=False, stderr=True)\n else:\n logs = self.get_logs()\n rc_success = rv[\"StatusCode\"] == 0\n should_report = not (\n no_failure == rc_success\n and no_warnings == (not self.get_warnings(logs))\n and no_errors == (not self.get_errors(logs))\n )\n\n if not rc_success or should_report:\n LOGGER.error(f\"Command output:\\n{logs}\")\n else:\n LOGGER.debug(f\"Command output:\\n{logs}\")\n self.remove()\n\n # To see the reason, we run assert statements separately\n assert (\n no_failure == rc_success\n ), f\"Container exited with code {rv['StatusCode']}\"\n warnings = self.get_warnings(logs)\n assert no_warnings == (not warnings), f\"Warnings found: {warnings}\"\n errors = self.get_errors(logs)\n assert no_errors == (not errors), f\"Errors found: {errors}\"\n\n if split_stderr:\n return (stdout, stderr)\n else:\n return logs\n\n @staticmethod\n def get_errors(logs: str) -> list[str]:\n return TrackedContainer._lines_starting_with(logs, \"ERROR\")\n\n @staticmethod\n def get_warnings(logs: str) -> list[str]:\n warnings = TrackedContainer._lines_starting_with(logs, \"WARNING\")\n warnings = [\n line\n for line in warnings\n if \"WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\"\n not in line\n ]\n return warnings\n\n @staticmethod\n def _lines_starting_with(logs: str, pattern: LiteralString) -> list[str]:\n return [line for line in logs.splitlines() if line.startswith(pattern)]\n\n def remove(self) -> None:\n \"\"\"Kills and removes the tracked docker container.\"\"\"\n if self.container is None:\n LOGGER.debug(\"No container to remove\")\n else:\n LOGGER.info(f\"Removing container {self.container.name} ...\")\n self.container.remove(force=True)\n LOGGER.info(f\"Container {self.container.name} removed\")\n self.container = None\n" }, { "path": "wiki/Home.md", "content": "# Jupyter Docker Stacks build manifests\n\n\n\nWelcome!\nPlease see [the documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/) for help with\nusing, contributing to, and maintaining the Jupyter Docker stacks images.\n\n## Build History\n\nThis is an auto-generated index of information from the build system.\nIn this index, you can find image tags, links to commits, and build manifests that describe the image.\nAll the builds are grouped by year and then month.\n\nNote: we only store the last 4500 manifest files because of GitHub limits.\nThat's why old manifest files might not be available.\nIf you want to clone this repo and access the Git history, use the following command: `git clone git@github.com:{REPOSITORY}.wiki.git`\n\nIn the tables below, each line represents:\n\n- `YYYY-MM`: link to a page with a list of images built\n- `Builds`: # of times build workflow finished\n- `Images`: # of single platform images pushed\n- `Commits`: # of commits made and a GitHub link\n\n\n\n" }, { "path": "wiki/__init__.py", "content": "" }, { "path": "wiki/config.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom dataclasses import dataclass\nfrom pathlib import Path\n\n\n@dataclass(frozen=True)\nclass Config:\n wiki_dir: Path\n hist_lines_dir: Path\n manifests_dir: Path\n\n repository: str\n allow_no_files: bool\n" }, { "path": "wiki/manifest_time.py", "content": "# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nfrom pathlib import Path\n\n\ndef get_manifest_timestamp(manifest_file: Path) -> str:\n file_content = manifest_file.read_text()\n TIMESTAMP_PREFIX = \"Build timestamp: \"\n TIMESTAMP_LENGTH = 20\n timestamp = file_content[\n file_content.find(TIMESTAMP_PREFIX) + len(TIMESTAMP_PREFIX) :\n ][:TIMESTAMP_LENGTH]\n # Should be good enough till year 2100\n assert timestamp.startswith(\"20\"), timestamp\n assert timestamp.endswith(\"Z\"), timestamp\n return timestamp\n\n\ndef get_manifest_year_month(manifest_file: Path) -> str:\n return get_manifest_timestamp(manifest_file)[:7]\n" }, { "path": "wiki/update_wiki.py", "content": "#!/usr/bin/env python3\n# Copyright (c) Jupyter Development Team.\n# Distributed under the terms of the Modified BSD License.\nimport argparse\nimport datetime\nimport logging\nimport shutil\nimport textwrap\nfrom dataclasses import dataclass\nfrom pathlib import Path\n\nimport plumbum\nimport tabulate\nfrom dateutil import relativedelta\n\nfrom wiki.config import Config\nfrom wiki.manifest_time import get_manifest_timestamp, get_manifest_year_month\n\ngit = plumbum.local[\"git\"]\n\nLOGGER = logging.getLogger(__name__)\nTHIS_DIR = Path(__file__).parent.resolve()\n\n\n@dataclass\nclass YearMonthFile:\n month: int\n content: str\n\n\n@dataclass\nclass Statistics:\n builds: int\n images: int\n commits: int\n\n\ndef calculate_monthly_stat(\n year_month_file: YearMonthFile, year_month_date: datetime.date\n) -> Statistics:\n builds = sum(\n \"/base-notebook\" in line and \"aarch64\" not in line\n for line in year_month_file.content.split(\"\\n\")\n )\n\n images = year_month_file.content.count(\"Build manifest\")\n\n with plumbum.local.env(TZ=\"UTC\"):\n git_log = git[\n \"log\",\n \"--oneline\",\n \"--since\",\n f\"{year_month_date}.midnight\",\n \"--until\",\n f\"{year_month_date + relativedelta.relativedelta(months=1)}.midnight\",\n \"--first-parent\",\n ]()\n commits = len(git_log.splitlines())\n return Statistics(builds=builds, images=images, commits=commits)\n\n\n@dataclass\nclass YearFiles:\n year: int\n files: list[YearMonthFile]\n\n\ndef generate_home_wiki_tables(repository: str, all_years: list[YearFiles]) -> str:\n tables = \"\"\n\n GITHUB_COMMITS_URL = (\n f\"[{{}}](https://github.com/{repository}/commits/main/?since={{}}&until={{}})\"\n )\n\n YEAR_TABLE_HEADERS = [\"Month\", \"Builds\", \"Images\", \"Commits\"]\n\n for year_files in all_years:\n year = year_files.year\n\n tables += f\"\\n\\n## {year}\\n\\n\"\n year_table_rows = []\n\n year_stat = Statistics(builds=0, images=0, commits=0)\n for year_month_file in year_files.files:\n month = year_month_file.month\n year_month_date = datetime.date(year=year, month=month, day=1)\n month_stat = calculate_monthly_stat(year_month_file, year_month_date)\n\n year_stat.builds += month_stat.builds\n year_stat.images += month_stat.images\n year_stat.commits += month_stat.commits\n\n commits_url = GITHUB_COMMITS_URL.format(\n month_stat.commits,\n year_month_date,\n year_month_date + relativedelta.relativedelta(day=31),\n )\n year_month = f\"{year}-{month:0>2}\"\n year_table_rows.append(\n [\n f\"[`{year_month}`](./{year_month})\",\n month_stat.builds,\n month_stat.images,\n commits_url,\n ]\n )\n\n year_commits_url = GITHUB_COMMITS_URL.format(\n year_stat.commits, f\"{year}-01-01\", f\"{year}-12-31\"\n )\n year_table_rows.append(\n [\"**Total**\", year_stat.builds, year_stat.images, year_commits_url]\n )\n\n tables += tabulate.tabulate(\n year_table_rows, YEAR_TABLE_HEADERS, tablefmt=\"github\"\n )\n LOGGER.info(\"Generated home wiki tables\")\n return tables\n\n\ndef write_home_wiki_page(wiki_dir: Path, repository: str) -> None:\n all_years = []\n for year_dir in sorted((wiki_dir / \"monthly-files\").glob(\"*\"), reverse=True):\n files = sorted(year_dir.glob(\"*.md\"), reverse=True)\n all_years.append(\n YearFiles(\n int(year_dir.name),\n [\n YearMonthFile(month=int(f.stem[5:]), content=f.read_text())\n for f in files\n ],\n )\n )\n wiki_home_tables = generate_home_wiki_tables(repository, all_years)\n\n wiki_home_content = (THIS_DIR / \"Home.md\").read_text()\n YEAR_MONTHLY_TABLES = \"\"\n\n assert YEAR_MONTHLY_TABLES in wiki_home_content\n wiki_home_content = wiki_home_content[\n : wiki_home_content.find(YEAR_MONTHLY_TABLES) + len(YEAR_MONTHLY_TABLES)\n ]\n wiki_home_content = wiki_home_content.format(REPOSITORY=repository)\n wiki_home_content += wiki_home_tables + \"\\n\"\n\n (wiki_dir / \"Home.md\").write_text(wiki_home_content)\n LOGGER.info(\"Updated Home page\")\n\n\ndef update_monthly_wiki_page(wiki_dir: Path, build_history_line: str) -> None:\n assert build_history_line.startswith(\"| `\")\n year_month = build_history_line[3:10]\n\n MONTHLY_PAGE_HEADER = textwrap.dedent(f\"\"\"\\\n # Images built during {year_month}\n\n | Date | Image | Links |\n | - | - | - |\n \"\"\")\n year = year_month[:4]\n monthly_page = wiki_dir / \"monthly-files\" / year / (year_month + \".md\")\n if not monthly_page.exists():\n monthly_page.parent.mkdir(parents=True, exist_ok=True)\n monthly_page.write_text(MONTHLY_PAGE_HEADER)\n LOGGER.info(f\"Created monthly page: {monthly_page.relative_to(wiki_dir)}\")\n\n monthly_page_content = monthly_page.read_text()\n assert MONTHLY_PAGE_HEADER in monthly_page_content\n monthly_page_content = monthly_page_content.replace(\n MONTHLY_PAGE_HEADER, MONTHLY_PAGE_HEADER + build_history_line + \"\\n\"\n )\n monthly_page.write_text(monthly_page_content)\n LOGGER.info(f\"Updated monthly page: {monthly_page.relative_to(wiki_dir)}\")\n\n\ndef remove_old_manifests(wiki_dir: Path) -> None:\n MAX_NUMBER_OF_MANIFESTS = 4500\n\n manifest_files: list[tuple[str, Path]] = []\n for file in (wiki_dir / \"manifests\").rglob(\"*.md\"):\n manifest_files.append((get_manifest_timestamp(file), file))\n\n manifest_files.sort(reverse=True)\n for _, file in manifest_files[MAX_NUMBER_OF_MANIFESTS:]:\n file.unlink()\n LOGGER.info(f\"Removed manifest: {file.relative_to(wiki_dir)}\")\n\n\ndef copy_manifest_files(config: Config) -> None:\n manifest_files = list(config.manifests_dir.rglob(\"*.md\"))\n if not config.allow_no_files:\n assert manifest_files, \"expected to have some manifest files\"\n for manifest_file in manifest_files:\n year_month = get_manifest_year_month(manifest_file)\n year = year_month[:4]\n copy_to = config.wiki_dir / \"manifests\" / year / year_month / manifest_file.name\n copy_to.parent.mkdir(parents=True, exist_ok=True)\n shutil.copy(manifest_file, copy_to)\n LOGGER.info(f\"Added manifest file: {copy_to.relative_to(config.wiki_dir)}\")\n\n\ndef update_wiki(config: Config) -> None:\n LOGGER.info(\"Updating wiki\")\n\n copy_manifest_files(config)\n\n build_history_line_files = sorted(config.hist_lines_dir.rglob(\"*.txt\"))\n if not config.allow_no_files:\n assert (\n build_history_line_files\n ), \"expected to have some build history line files\"\n for build_history_line_file in build_history_line_files:\n build_history_line = build_history_line_file.read_text()\n update_monthly_wiki_page(config.wiki_dir, build_history_line)\n\n write_home_wiki_page(config.wiki_dir, config.repository)\n remove_old_manifests(config.wiki_dir)\n\n LOGGER.info(\"Wiki updated\")\n\n\nif __name__ == \"__main__\":\n logging.basicConfig(level=logging.INFO)\n\n arg_parser = argparse.ArgumentParser()\n arg_parser.add_argument(\n \"--wiki-dir\",\n required=True,\n type=Path,\n help=\"Directory of the wiki repo\",\n )\n arg_parser.add_argument(\n \"--hist-lines-dir\",\n required=True,\n type=Path,\n help=\"Directory with history lines\",\n )\n arg_parser.add_argument(\n \"--manifests-dir\",\n required=True,\n type=Path,\n help=\"Directory with manifest files\",\n )\n arg_parser.add_argument(\n \"--repository\",\n required=True,\n help=\"Repository name on GitHub\",\n )\n arg_parser.add_argument(\n \"--allow-no-files\",\n action=\"store_true\",\n help=\"Allow no manifest or history line files\",\n )\n args = arg_parser.parse_args()\n\n config = Config(**vars(args))\n update_wiki(config)\n" } ]