Repository: hjwp/Book-TDD-Web-Dev-Python Branch: main Commit: 4f61e6abb4a9 Files: 209 Total size: 3.0 MB Directory structure: gitextract_l59mn1di/ ├── !README_FOR_PRODUCTION.txt ├── .dockerignore ├── .git-blame-ignore-revs ├── .github/ │ └── workflows/ │ └── tests.yml ├── .gitignore ├── .gitmodules ├── .python-version ├── CITATION.md ├── Dockerfile ├── ER_sampleTOC.html ├── LICENSE.md ├── Makefile ├── README.md ├── Vagrantfile ├── acknowledgments.asciidoc ├── ai_preface.asciidoc ├── analytics.html ├── appendix_CD.asciidoc ├── appendix_DjangoRestFramework.asciidoc ├── appendix_Django_Class-Based_Views.asciidoc ├── appendix_IV_testing_migrations.asciidoc ├── appendix_IX_cheat_sheet.asciidoc ├── appendix_X_what_to_do_next.asciidoc ├── appendix_bdd.asciidoc ├── appendix_fts_for_external_dependencies.asciidoc ├── appendix_github_links.asciidoc ├── appendix_logging.asciidoc ├── appendix_purist_unit_tests.asciidoc ├── appendix_rest_api.asciidoc ├── appendix_tradeoffs.asciidoc ├── asciidoc.conf ├── asciidoctor.css ├── atlas.json ├── author_bio.html ├── bibliography.asciidoc ├── book.asciidoc ├── buy_the_book_banner.html ├── chapter_01.asciidoc ├── chapter_02_unittest.asciidoc ├── chapter_03_unit_test_first_view.asciidoc ├── chapter_04_philosophy_and_refactoring.asciidoc ├── chapter_05_post_and_database.asciidoc ├── chapter_06_explicit_waits_1.asciidoc ├── chapter_07_working_incrementally.asciidoc ├── chapter_08_prettification.asciidoc ├── chapter_09_docker.asciidoc ├── chapter_10_production_readiness.asciidoc ├── chapter_11_server_prep.asciidoc ├── chapter_12_ansible.asciidoc ├── chapter_13_organising_test_files.asciidoc ├── chapter_14_database_layer_validation.asciidoc ├── chapter_15_simple_form.asciidoc ├── chapter_16_advanced_forms.asciidoc ├── chapter_17_javascript.asciidoc ├── chapter_18_second_deploy.asciidoc ├── chapter_19_spiking_custom_auth.asciidoc ├── chapter_20_mocking_1.asciidoc ├── chapter_21_mocking_2.asciidoc ├── chapter_22_fixtures_and_wait_decorator.asciidoc ├── chapter_23_debugging_prod.asciidoc ├── chapter_24_outside_in.asciidoc ├── chapter_25_CI.asciidoc ├── chapter_26_page_pattern.asciidoc ├── chapter_27_hot_lava.asciidoc ├── check-links.py ├── coderay-asciidoctor.css ├── colo.html ├── copy_html_to_site_and_print_toc.py ├── copyright.html ├── count-todos.py ├── cover.html ├── disqus_comments.html ├── docs/ │ ├── ORM_style_guide.htm │ ├── ORM_style_guide_files/ │ │ └── main.css │ ├── asciidoc-cheatsheet.html │ ├── asciidoc-cheatsheet_files/ │ │ ├── Content.css │ │ ├── asciidoc.asc │ │ ├── asciidoc.css │ │ ├── asciidoc.js │ │ ├── jquery-1.js │ │ └── pygments.css │ ├── asciidoc-userguide.html │ ├── asciidoc-userguide_files/ │ │ ├── Content.css │ │ ├── asciidoc.css │ │ ├── asciidoc.js │ │ └── layout2.css │ └── example_book.txt ├── epilogue.asciidoc ├── index.txt ├── ix.html ├── load_toc.js ├── misc/ │ ├── chapters.rst │ ├── chapters_v2.rst │ ├── chimera_comments_scraper.py │ ├── curl │ ├── get_stats.py │ ├── get_stats.sh │ ├── isolation-talks/ │ │ ├── djangoisland.md │ │ ├── djangoisland.py │ │ ├── extra_styling_for_djangoisland.css │ │ ├── outline.txt │ │ └── webcast-commits.hist │ ├── plot.py │ ├── reddit_post.md │ ├── redditnotesresponse.txt │ ├── tdd-flowchart.dot │ └── tdd_diagram.odp ├── outline_and_future_chapters.asciidoc ├── part1.asciidoc ├── part2.asciidoc ├── part3.asciidoc ├── part4.asciidoc ├── praise.forbook.asciidoc ├── praise.html ├── pre-requisite-installations.asciidoc ├── preface.asciidoc ├── pygments-default.css ├── pyproject.toml ├── rename-chapter.sh ├── research/ │ ├── js-testing.rst │ └── literary_agencies.ods ├── run_test_tests.sh ├── server-quickstart.md ├── source/ │ ├── blackify-chap.sh │ ├── feed-thru-cherry-picks.sh │ ├── fix-commit-numbers.py │ └── push-back.sh ├── tests/ │ ├── actual_manage_py_test.output │ ├── book_parser.py │ ├── book_tester.py │ ├── chapters.py │ ├── check_links.py │ ├── conftest.py │ ├── examples.py │ ├── my-phantomjs-qunit-runner.js │ ├── run-js-spec.py │ ├── slimerjs-0.9.0/ │ │ ├── LICENSE │ │ ├── README.md │ │ ├── application.ini │ │ ├── omni.ja │ │ ├── slimerjs │ │ ├── slimerjs.bat │ │ └── slimerjs.py │ ├── source_updater.py │ ├── sourcetree.py │ ├── test_appendix_DjangoRestFramework.py │ ├── test_appendix_Django_Class-Based_Views.py │ ├── test_appendix_bdd.py │ ├── test_appendix_purist_unit_tests.py │ ├── test_appendix_rest_api.py │ ├── test_book_parser.py │ ├── test_book_tester.py │ ├── test_chapter_01.py │ ├── test_chapter_02_unittest.py │ ├── test_chapter_03_unit_test_first_view.py │ ├── test_chapter_04_philosophy_and_refactoring.py │ ├── test_chapter_05_post_and_database.py │ ├── test_chapter_06_explicit_waits_1.py │ ├── test_chapter_07_working_incrementally.py │ ├── test_chapter_08_prettification.py │ ├── test_chapter_09_docker.py │ ├── test_chapter_10_production_readiness.py │ ├── test_chapter_11_server_prep.py │ ├── test_chapter_12_ansible.py │ ├── test_chapter_13_organising_test_files.py │ ├── test_chapter_14_database_layer_validation.py │ ├── test_chapter_15_simple_form.py │ ├── test_chapter_16_advanced_forms.py │ ├── test_chapter_17_javascript.py │ ├── test_chapter_19_spiking_custom_auth.py │ ├── test_chapter_20_mocking_1.py │ ├── test_chapter_21_mocking_2.py │ ├── test_chapter_22_fixtures_and_wait_decorator.py │ ├── test_chapter_23_debugging_prod.py │ ├── test_chapter_24_outside_in.py │ ├── test_chapter_25_CI.py │ ├── test_chapter_26_page_pattern.py │ ├── test_source_updater.py │ ├── test_sourcetree.py │ ├── test_write_to_file.py │ ├── update_source_repo.py │ └── write_to_file.py ├── theme/ │ ├── epub/ │ │ ├── epub.css │ │ ├── epub.xsl │ │ └── layout.html │ ├── html/ │ │ └── html.xsl │ ├── mobi/ │ │ ├── layout.html │ │ ├── mobi.css │ │ └── mobi.xsl │ └── pdf/ │ ├── pdf.css │ └── pdf.xsl ├── titlepage.html ├── toc.html ├── todos.txt ├── tools/ │ ├── figure_renaming_report.tsv │ ├── intake_report.txt │ └── oneoffs/ │ ├── oneoff.css │ └── oneoff.xsl ├── video_plug.asciidoc ├── wordcount └── workshops/ ├── intermediate_workshop_notes.md ├── js-testing-with-jasmine.asciidoc ├── pycon.uk.2015.dirigible-talk.md ├── pycon.uk.2015.tutorial-beginners.md ├── pycon.us.2015.study-group.md ├── study-group.md ├── working-incrementally-handout.md ├── working-incrementally-notes.md └── workshop.asciidoc ================================================ FILE CONTENTS ================================================ ================================================ FILE: !README_FOR_PRODUCTION.txt ================================================ Note for production editor about page-breaking: the author has requested a global solution to keep code blocks from breaking across pages in the PDF, so there is CSS in place to that effect. ================================================ FILE: .dockerignore ================================================ .venv ================================================ FILE: .git-blame-ignore-revs ================================================ # matt hacker bulk edit for prod 2cb0ed8c264cee682303288ba5a5cea80956fb8d 2735e383f1281c5f200e64cfb7cda0457cfe8d1e 6fce793a9a4d701313684de11ca2cd3f5e89a041 ================================================ FILE: .github/workflows/tests.yml ================================================ --- name: Book tests on: schedule: - cron: "45 15 * * *" push: branches: main pull_request: jobs: chapter-tests: runs-on: ubuntu-latest strategy: fail-fast: false matrix: test_chapter: [ test_chapter_01, test_chapter_02_unittest, test_chapter_03_unit_test_first_view, test_chapter_04_philosophy_and_refactoring, test_chapter_05_post_and_database, test_chapter_06_explicit_waits_1, test_chapter_07_working_incrementally, test_chapter_08_prettification, test_chapter_09_docker, test_chapter_10_production_readiness, test_chapter_11_server_prep, test_chapter_12_ansible, test_chapter_13_organising_test_files, test_chapter_14_database_layer_validation, test_chapter_15_simple_form, test_chapter_16_advanced_forms, test_chapter_17_javascript, test_chapter_19_spiking_custom_auth, test_chapter_20_mocking_1, test_chapter_21_mocking_2, test_chapter_22_fixtures_and_wait_decorator, test_chapter_23_debugging_prod, test_chapter_24_outside_in, test_chapter_25_CI, test_chapter_26_page_pattern, ] env: PY_COLORS: "1" # enable coloured output in pytest EMAIL_PASSWORD: ${{ secrets.GMAIL_APP_PASSWORD }} steps: - uses: actions/checkout@v4 - name: checkout submodules shell: bash run: | sed -i 's_git@github.com:_https://github.com/_' .gitmodules git submodule init git submodule status | cut -d" " -f2 | xargs -n1 -P0 git submodule update - name: setup Git shell: bash run: | git config --global user.email "elspeth@example.com" git config --global user.name "Elspeth See-Eye" git config --global init.defaultBranch main - name: Set up Python 3.14 uses: actions/setup-python@v6 with: python-version: '3.14' - name: Install apt stuff and other dependencies shell: bash run: | sudo apt remove -y --purge firefox sudo add-apt-repository ppa:mozillateam/ppa sudo apt update -y sudo apt install -y \ asciidoctor \ language-pack-en \ ruby-coderay \ ruby-pygments.rb \ firefox-esr \ tree # fix failed to install firefox bin/symlink which firefox || sudo ln -s /usr/bin/firefox-esr /usr/bin/firefox # remove old geckodriver which geckodriver && sudo rm $(which geckodriver) || exit 0 pip install uv - name: Install Python requirements.txt globally shell: bash run: | uv pip install --system . - name: Install Python requirements.txt into virtualenv shell: bash run: | make .venv/bin - name: Display firefox version shell: bash run: | apt show firefox-esr dpkg -L firefox-esr firefox --version which geckodriver && geckodriver --version || exit 0 - name: Run chapter test shell: bash run: | make ${{ matrix.test_chapter }} - name: Save tempdir path to an env var if: always() shell: bash run: | TMPDIR_PATH=$(cat .tmpdir.${{ matrix.test_chapter }}) echo "TMPDIR_PATH=$TMPDIR_PATH" >> $GITHUB_ENV - name: Archive the temp dir uses: actions/upload-artifact@v4 if: always() with: name: test-source-${{ matrix.test_chapter }} path: ${{ env.TMPDIR_PATH }} - name: Archive the built html files uses: actions/upload-artifact@v4 if: always() with: name: built-html-${{ matrix.test_chapter }} path: | *.html *.css other-tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install apt stuff and other dependencies shell: bash run: | sudo apt update -y sudo apt install -y \ asciidoctor \ language-pack-en \ ruby-coderay \ ruby-pygments.rb \ tree \ libxml2-utils pip install uv - name: Install Python requirements.txt into virtualenv shell: bash run: | make .venv/bin - name: setup Git shell: bash run: | git config --global user.email "elspeth@example.com" git config --global user.name "Elspeth See-Eye" git config --global init.defaultBranch main - name: prep tests submodule shell: bash run: | sed -i 's_git@github.com:_https://github.com/_' .gitmodules git submodule init git submodule update tests/testrepo - name: Run unit tests shell: bash run: | make unit-test - name: Run xml linter shell: bash run: | make xmllint_book ================================================ FILE: .gitignore ================================================ *.pyc .cache .vagrant *-cloudimg-console.log .venv .pytest_cache /chapter_*.html /appendix_*.html /part*.html /outline_and_future*.html /pre-requisite-installations.html /preface.html /epilogue.html /bibliography.html /acknowledgments.html /video_plug.html /part*.forbook.asciidoc /praise.forbook.html /ai_preface.html /misc/abandoned_roman_numerals_example /docs/atlas_docs/ /proposals /tags /pdf_drafts /pycon /source/*/static /source/*/database /wordcounts.* /feedback /downloads/*.js /downloads/mock* /misc/promo/ /misc/Vagrantfile /misc/superlists-repo-django16-backup.zip /tdd-tutorial-materials /misc/Invoice-Percival-1 /video /misc/*conference_report.md /tests/.cache/ /workshops/js-testing-with-jasmine.html /tech review/ .vagrant.d *.egg-info .tmpdir.* .env ================================================ FILE: .gitmodules ================================================ [submodule "source/chapter_01/superlists"] path = source/chapter_01/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_02/superlists"] path = source/chapter_02_unittest/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_03/superlists"] path = source/chapter_03_unit_test_first_view/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_04/superlists"] path = source/chapter_04_philosophy_and_refactoring/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_05/superlists"] path = source/chapter_05_post_and_database/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_06/superlists"] path = source/chapter_06_explicit_waits_1/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_07/superlists"] path = source/chapter_07_working_incrementally/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_08/superlists"] path = source/chapter_08_prettification/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_09/superlists"] path = source/chapter_09_docker/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_10_production_readiness/superlists"] path = source/chapter_10_production_readiness/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_11_server_prep/superlists"] path = source/chapter_11_server_prep/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_12_ansible/superlists"] path = source/chapter_12_ansible/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_13_organising_test_files/superlists"] path = source/chapter_13_organising_test_files/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_13/superlists"] path = source/chapter_14_database_layer_validation/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_14/superlists"] path = source/chapter_15_simple_form/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_15/superlists"] path = source/chapter_16_advanced_forms/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_16/superlists"] path = source/chapter_17_javascript/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_17/superlists"] path = source/chapter_18_second_deploy/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_18/superlists"] path = source/chapter_19_spiking_custom_auth/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_19/superlists"] path = source/chapter_20_mocking_1/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_21_mocking_2/superlists"] path = source/chapter_21_mocking_2/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_22_fixtures_and_wait_decorator/superlists"] path = source/chapter_22_fixtures_and_wait_decorator/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_21/superlists"] path = source/chapter_23_debugging_prod/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_24_outside_in/superlists"] path = source/chapter_24_outside_in/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_25_CI/superlists"] path = source/chapter_25_CI/superlists url = git@github.com:hjwp/book-example.git [submodule "source/chapter_26_page_pattern/superlists"] path = source/chapter_26_page_pattern/superlists url = git@github.com:hjwp/book-example.git [submodule "tests/testrepo"] path = tests/testrepo url = git@github.com:hjwp/booktesttestrepo.git [submodule "source/appendix_II/superlists"] path = source/appendix_Django_Class-Based_Views/superlists url = git@github.com:hjwp/book-example.git [submodule "source/appendix_III/superlists"] path = source/appendix_III/superlists url = git@github.com:hjwp/book-example.git [submodule "source/appendix_bdd/superlists"] path = source/appendix_bdd/superlists url = git@github.com:hjwp/book-example.git [submodule "source/appendix_VI_rest_api_backend/superlists"] path = source/appendix_rest_api/superlists url = git@github.com:hjwp/book-example.git [submodule "source/appendix_VIII_DjangoRestFramework/superlists"] path = source/appendix_DjangoRestFramework/superlists url = git@github.com:hjwp/book-example.git [submodule "source/appendix_purist_unit_tests/superlists"] path = source/appendix_purist_unit_tests/superlists url = git@github.com:hjwp/book-example.git ================================================ FILE: .python-version ================================================ 3.14 ================================================ FILE: CITATION.md ================================================ Bibtex: ```TeX @BOOK{percival:tdd:python, AUTHOR = "{Harry J.W.} Percival", TITLE = "Test-Driven Development with Python", SUBTITLE = "Obey the Testing Goat!", DATE = "2014", PUBLISHER = "O'Reilly Media, Inc.", ISBN = "9781449365141" } ``` ================================================ FILE: Dockerfile ================================================ FROM python:slim # -- WIP -- # this dockerfile is a work in progress, # the vague intention is to use it for CI. # RUN add-apt-repository ppa:mozillateam/ppa && \ RUN apt-get update -y RUN apt-get install -y \ git \ asciidoctor \ # language-pack-en \ ruby-pygments.rb \ firefox-esr \ tree \ locales \ vim RUN apt-get install -y \ make \ curl RUN locale-gen en_GB.UTF-8 # RUN pip install uv ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh RUN /install.sh && rm /install.sh RUN ln -s $HOME/.local/bin/uv /usr/bin/uv RUN git config --global user.email "elspeth@example.com" && \ git config --global user.name "Elspeth See-Eye" && \ git config --global init.defaultBranch main WORKDIR /app RUN uv venv .venv COPY pyproject.toml pyproject.toml RUN uv pip install . RUN uv pip install selenium ENV PATH=".venv/bin:$PATH" CMD bash ================================================ FILE: ER_sampleTOC.html ================================================

Brief Table of Contents (Not Yet Final)

Preface (AVAILABLE)

Prerequisites and Assumptions (AVAILABLE)

Companion Video (AVAILABLE)

Acknowledgments (UNAVAILABLE)

Part 1: The Basics of TDD and Django (AVAILABLE)

Chapter 1: Getting Django Set Up Using a Functional Test (AVAILABLE)

Chapter 2: Extending Our Functional Test Using the unittest Module (AVAILABLE)

Chapter 3: Testing a Simple Home Page with Unit Tests (AVAILABLE)

Chapter 4: What Are We Doing with All These Tests? (And, Refactoring) (AVAILABLE)

Chapter 5: Saving User Input: Testing the Database (AVAILABLE)

Chapter 6: Improving Functional Tests: Ensuring Isolation and Removing Voodoo Sleeps (AVAILABLE)

Chapter 7: Working Incrementally (AVAILABLE)

Part 2: Web Development Sine Qua Nons (AVAILABLE)

Chapter 8: Prettification: Layout and Styling, and What to Test About It (AVAILABLE)

Chapter 9: Containerization akaDocker (AVAILABLE)

Chapter 10: Making our App Production-Ready (AVAILABLE)

Chapter 11: Getting A Server Ready for Deployment (AVAILABLE)

Chapter 12: Infrastructure As Code: Automated Deployments With Ansible (AVAILABLE)

Chapter 13: Splitting Our Tests into Multiple Files, and a Generic Wait Helper (AVAILABLE)

Chapter 14: Validation at the Database Layer (AVAILABLE)

Chapter 15: A Simple Form (AVAILABLE)

Chapter 16: More Advanced Forms (AVAILABLE)

Chapter 17: A Gentle Excursion into JavaScript (AVAILABLE)

Chapter 18: Deploying Our New Code (AVAILABLE)

Part 3: More Advanced Topics in Testing (UNAVAILABLE)

Chapter 19: User Authentication, Spiking, and De-Spiking (UNAVAILABLE)

Chapter 20: Mocks and Mocking 1: Using Mocks to Test External Dependencies (UNAVAILABLE)

Chapter 21: Mocks and Mocking 2: Using Mocks for Test Isolation (UNAVAILABLE)

Chapter 22: Test Fixtures and a Decorator for Explicit Waits (UNAVAILABLE)

Chapter 23: Server-Side Debugging (UNAVAILABLE)

Chapter 24: Finishing “My Lists”: Outside-In TDD (UNAVAILABLE)

Chapter 25: Continuous Integration (CI) (UNAVAILABLE)

Chapter 26: The Token Social Bit, the Page Pattern, and an Exercise for the Reader (UNAVAILABLE)

Chapter 27: Fast Tests, Slow Tests, and Hot Lava (UNAVAILABLE)

Back Matter: Obey the Testing Goat! (UNAVAILABLE)

App A: PythonAnywhere (UNAVAILABLE)

App B: Django Class-Based Views (UNAVAILABLE)

App C: Provisioning with Ansible (UNAVAILABLE)

App D: Testing Database Migrations (UNAVAILABLE)

App E: Behaviour-Driven Development (BDD) (UNAVAILABLE)

App F: Building a REST API: JSON, Ajax, and Mocking with JavaScript (UNAVAILABLE)

App G: Django-Rest-Framework (UNAVAILABLE)

App H: Cheat Sheet (UNAVAILABLE)

App I: What to Do Next (UNAVAILABLE)

App J: Source Code Examples (UNAVAILABLE)

Bibliography (UNAVAILABLE)

================================================ FILE: LICENSE.md ================================================ This book, and all the associated source files, are being made available under the Creative Commons Attribution-NonCommercial-ShareAlike License (v3.0 United States) Full info at https://creativecommons.org/licenses/by-nc-sa/3.0/us/ ================================================ FILE: Makefile ================================================ SHELL := /bin/bash SOURCES := $(wildcard *.asciidoc) HTML_PAGES := $(patsubst %.asciidoc, %.html, ${SOURCES}) TESTS := $(patsubst %.asciidoc, test_%, ${SOURCES}) VENV ?= .venv RUN_ASCIIDOCTOR = asciidoctor -a source-highlighter=pygments -a pygments-style=default -a stylesheet=asciidoctor.css -a linkcss -a icons=font -a compat-mode -a '!example-caption' -a last-update-label='License: Creative Commons CC-BY-NC-ND. Last updated:' export PYTHONHASHSEED = 0 export PYTHONDONTWRITEBYTECODE = 1 export MOZ_HEADLESS = 1 # for warning introduce in selenium 4.10 export PYTHONWARNINGS=ignore::ResourceWarning export TMPDIR_CLEANUP = false part%.forbook.asciidoc: part%.asciidoc cat $(subst .forbook.,.,$@) \ | sed 's/^== /= /' \ | sed '/partintro/d' \ | sed '/^--$$/d' \ > $@ book.html: part1.forbook.asciidoc book.html: part2.forbook.asciidoc book.html: part3.forbook.asciidoc book.html: part4.forbook.asciidoc book.html: $(SOURCES) %.html: %.asciidoc # build an individual chapter's html page $(RUN_ASCIIDOCTOR) $< .PHONY: build build: $(HTML_PAGES) $(TMPDIR) $(VENV)/bin: which uv && uv venv $(VENV)|| python3 -m venv $(VENV) which uv && uv pip install -e . || $(VENV)/bin/pip install -e . .PHONY: install install: $(VENV)/bin which brew && brew install asciidoctor tree || apt install -y asciidoctor tree .PHONY: update-submodules update-submodules: git submodule update --init --recursive $(VENV)/bin/python tests/update_source_repo.py # this is to allow for a git remote called "local" for eg ./source/feed-thru-cherry-pick.sh ../book-example.git: mkdir -p ../book-example.git git init --bare ../book-example.git .PHONY: test test: build update-submodules $(VENV)/bin $(VENV)/bin/pytest tests/ .PHONY: testall testall: build $(VENV)/bin/pytest --numprocesses=auto tests/test_chapter_* .PHONY: testall4 testall4: build $(VENV)/bin/pytest --numprocesses=4 tests/test_chapter_* .PHONY: test_% test_%: %.html $(TMPDIR) $(VENV)/bin/pytest -s --no-summary ./tests/$@.py .PHONY: xmllint_% xmllint_%: %.asciidoc asciidoctor -b docbook $< -o - | sed \ -e 's/—/\—/g' \ -e 's/“/\“/g' \ -e 's/”/\”/g' \ -e 's/‘/\‘/g' \ -e 's/’/\’/g' \ -e 's/…/\…/g' \ -e 's/ /\ /g' \ -e 's/×/\×/g' \ | xmllint --noent --noout - %.xml: %.asciidoc asciidoctor -b docbook $< .PHONY: check-links check-links: book.html python check-links.py book.html .PHONY: clean-docker clean-docker: -docker kill $$(docker ps -q) docker rmi -f busybox docker rmi -f superlists # env PATH=misc:$PATH .PHONY: get-sudo get-sudo: sudo echo 'need sudo access for this test' .PHONY: no-runservers no-runservers: -pkill -f runserver # exhaustively list all test targets for nice tab-completion .PHONY: test_chapter_01 test_chapter_01: chapter_01.html $(TMPDIR) $(VENV)/bin no-runservers $(VENV)/bin/pytest -s ./tests/test_chapter_01.py .PHONY: test_chapter_02_unittest test_chapter_02_unittest: chapter_02_unittest.html $(TMPDIR) $(VENV)/bin no-runservers $(VENV)/bin/pytest -s ./tests/test_chapter_02_unittest.py .PHONY: test_chapter_03_unit_test_first_view test_chapter_03_unit_test_first_view: chapter_03_unit_test_first_view.html $(TMPDIR) $(VENV)/bin no-runservers $(VENV)/bin/pytest -s ./tests/test_chapter_03_unit_test_first_view.py .PHONY: test_chapter_04_philosophy_and_refactoring test_chapter_04_philosophy_and_refactoring: chapter_04_philosophy_and_refactoring.html $(TMPDIR) $(VENV)/bin no-runservers $(VENV)/bin/pytest -s ./tests/test_chapter_04_philosophy_and_refactoring.py .PHONY: test_chapter_05_post_and_database test_chapter_05_post_and_database: chapter_05_post_and_database.html $(TMPDIR) $(VENV)/bin no-runservers $(VENV)/bin/pytest -s ./tests/test_chapter_05_post_and_database.py .PHONY: test_chapter_06_explicit_waits_1 test_chapter_06_explicit_waits_1: chapter_06_explicit_waits_1.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_06_explicit_waits_1.py .PHONY: test_chapter_07_working_incrementally test_chapter_07_working_incrementally: chapter_07_working_incrementally.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_07_working_incrementally.py .PHONY: test_chapter_08_prettification test_chapter_08_prettification: chapter_08_prettification.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_08_prettification.py .PHONY: test_chapter_09_docker test_chapter_09_docker: chapter_09_docker.html $(TMPDIR) $(VENV)/bin clean-docker $(VENV)/bin/pytest -s ./tests/test_chapter_09_docker.py .PHONY: test_chapter_10_production_readiness test_chapter_10_production_readiness: get-sudo chapter_10_production_readiness.html $(TMPDIR) $(VENV)/bin clean-docker $(VENV)/bin/pytest -s ./tests/test_chapter_10_production_readiness.py .PHONY: test_chapter_11_server_prep test_chapter_11_server_prep: chapter_11_server_prep.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_11_server_prep.py .PHONY: test_chapter_13_organising_test_files .PHONY: test_chapter_12_ansible test_chapter_12_ansible: chapter_12_ansible.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_12_ansible.py .PHONY: test_chapter_13_organising_test_files test_chapter_13_organising_test_files: chapter_13_organising_test_files.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_13_organising_test_files.py .PHONY: test_chapter_14_database_layer_validation test_chapter_14_database_layer_validation: chapter_14_database_layer_validation.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_14_database_layer_validation.py .PHONY: test_chapter_15_simple_form test_chapter_15_simple_form: chapter_15_simple_form.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_15_simple_form.py .PHONY: test_chapter_16_advanced_forms test_chapter_16_advanced_forms: chapter_16_advanced_forms.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_16_advanced_forms.py .PHONY: test_chapter_17_javascript test_chapter_17_javascript: chapter_17_javascript.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_17_javascript.py .PHONY: test_chapter_18_second_deploy test_chapter_18_second_deploy: chapter_18_second_deploy.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_18_second_deploy.py .PHONY: test_chapter_19_spiking_custom_auth test_chapter_19_spiking_custom_auth: chapter_19_spiking_custom_auth.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_19_spiking_custom_auth.py .PHONY: test_chapter_20_mocking_1 test_chapter_20_mocking_1: chapter_20_mocking_1.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_20_mocking_1.py .PHONY: test_chapter_21_mocking_2 test_chapter_21_mocking_2: chapter_21_mocking_2.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_21_mocking_2.py .PHONY: test_chapter_22_fixtures_and_wait_decorator test_chapter_22_fixtures_and_wait_decorator: chapter_22_fixtures_and_wait_decorator.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_22_fixtures_and_wait_decorator.py .PHONY: test_chapter_23_debugging_prod test_chapter_23_debugging_prod: get-sudo chapter_23_debugging_prod.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_23_debugging_prod.py .PHONY: test_chapter_24_outside_in test_chapter_24_outside_in: chapter_24_outside_in.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_24_outside_in.py .PHONY: test_chapter_25_CI test_chapter_25_CI: chapter_25_CI.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_25_CI.py .PHONY: test_chapter_26_page_pattern test_chapter_26_page_pattern: chapter_26_page_pattern.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_chapter_26_page_pattern.py .PHONY: test_appendix_purist_unit_tests test_appendix_purist_unit_tests: appendix_purist_unit_tests.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest -s ./tests/test_appendix_purist_unit_tests.py .PHONY: silent_test_% silent_test_%: %.html $(TMPDIR) $(VENV)/bin $(VENV)/bin/pytest ./tests/$(subst silent_,,$@).py .PHONY: unit-test unit-test: chapter_01.html $(VENV)/bin SKIP_CHAPTER_SUBMODULES=1 ./tests/update_source_repo.py source $(VENV)/bin/activate && ./run_test_tests.sh # this is a hack to make 'Archive the temp dir' step work in CI echo "tests" > .tmpdir.unit-test .PHONY: clean clean: rm -rf $(TMPDIR) rm -v $(HTML_PAGES) ================================================ FILE: README.md ================================================ # Test-Driven Web Development With Python, the book. # License The sources for this book are published under the Creative Commons Attribution Non-Commercial No-Derivs license (CC-BY-NC-ND). *I wouldn't recommend using this version to read the book. Head over to [obeythetestinggoat.com](https://www.obeythetestinggoat.com/pages/book.html) when you can access a nicely formatted version of the full thing, still free and under CC license. And you'll also be able to buy an ebook or print version if you feel like it.* These sources are being made available for the purposes of curiosity (although if you're curious about the way the listings are tested, i would definitely recommend https://github.com/cosmicpython/book instead) and collaboration (typo-fixes by pull request are very much encouraged!). # Building the book as HTML - install [asciidoctor](http://asciidoctor.org/), and the *pygments/pygmentize* gem. - `make build` will build each chapter as its own html file - `make book.html` will create a single file - `make chapter_05_post_and_database.html`, eg, will build chapter 5 # Running the tests * Pre-requisites for the test suite: ```console make install ``` * Full test suite (probably, don't use this, it would take ages.) ```console $ make test ``` * To test an individual chapter, eg: ```console $ make test_chapter_06_explicit_waits_1 ``` If you see a problem that seems to be related to submodules, try: ```console make update-submodules ``` * Unit tests (tests for the tests for the tests in the testing book) ```console $ ./run_test_tests.sh ``` # Windows / WSL notes * `vagrant plugin install virtualbox_WSL2` is required # Making changes to the book's code examples Brief explanation: each chapter's code examples are reflected in a branch of the example code repository, https://github.com/hjwp/book-example in branches named after the chapter, so eg chapter_02_unittest.asciidoc has a branch called chapter_02_unittest. These branches are actually checked out, one by one, as submodules in source//superlists. Each branch starts at the end of the previous chapter's branch. Code listings _mostly_ map 1 to 1 with commits in the repo, and they are sometimes marked with little tags, eg ch03l007, meaning theoretically, the 7th listing in chapter 3, but that's not always accurate. When the tests run, they start by creating a new folder in /tmp checked out with the code from the end of the last chapter. Then they go through the code listings in the book one by one, and simulate typing them into to an editor. If the code listing has one of those little tags, the tests check the commit in the repo to see if the listing matches the commit exactly. (if there's no tag, there's some fiddly code based on string manipulation that tries to figure out how to insert the code listing into the existing file contents at the right place) When the tests come across a command, eg "ls", they actually run "ls" in the temp directory, to check whether the output that's printed in the book matches what would actually happen. One of the most common commands is to run the tests, obviously, so much so that if there is some console output in the book with no explicit command, the tests assume it's a test run, so they run "./manage.py test" or equivalent. In any case, back to our code listings - the point is that, if we want to change one of our code listings, we also need to change the commit in the branch / submodule... ...and all of the commits that come after it. ...for that chapter and every subsequent chapter. This is called "feeding through the changes" ## Changing a code listing 1. change the listing in the book, eg in in _chapter_03_unit_test_first_view.asciidoc_ 2. open up ./source/chapter_03_unit_test_first_view/superlists in a terminal 3. do a `git rebase --interactive $previous-chapter-name` 4. identify the commit that matches the listing that you've changed, and mark it for `edit` 5. edit the file when prompted, make it match the book 6. continue the rebase, and deal with an merge conflicts as you go, woo. 7. `git push local` once you're happy. ## feeding thru the changes Because we don't want to push WIP to github every time we change a chapter, we use a local bare repository to push and pull chapters ```console make ../book-example.git ``` will create it for you. TODO: helper to do `git remote add local` to each chapter/submodule Now you can attempt to feed thru the latest changes to this branch/chapter with ```console cd source ./feed_thru.sh chapter_03_unit_test_first_view chapter_04_philosophy_and_refactoring # chapter/branch names will tab complete, helpfully. ``` if all goes well, you can then run ```console ./push-back.sh chapter_04_philosophy_and_refactoring ``` and move on to the next chapter. woo! This may all seem a bit OTT, but the point is that if we change a variable early on in the book, git (along with the tests) will help us to make sure that it changes all the way through all the subsequent chapters. ================================================ FILE: Vagrantfile ================================================ # -*- mode: ruby -*- # vi: set ft=ruby : # All Vagrant configuration is done below. The "2" in Vagrant.configure # configures the configuration version (we support older styles for # backwards compatibility). Please don't change it unless you know what # you're doing. Vagrant.configure("2") do |config| # The most common configuration options are documented and commented below. # For a complete reference, please see the online documentation at # https://docs.vagrantup.com. # Every Vagrant development environment requires a box. You can search for # boxes at https://vagrantcloud.com/search. # config.vm.box = "ubuntu/jammy64" # virtualbox only # config.vm.box = "generic/ubuntu2204" # amd64 config.vm.box = "bento/ubuntu-22.04" # Disable automatic box update checking. If you disable this, then # boxes will only be checked for updates when the user runs # `vagrant box outdated`. This is not recommended. # config.vm.box_check_update = false # Create a forwarded port mapping which allows access to a specific port # within the machine from a port on the host machine. In the example below, # accessing "localhost:8080" will access port 80 on the guest machine. # NOTE: This will enable public access to the opened port # config.vm.network "forwarded_port", guest: 80, host: 8080 # Create a forwarded port mapping which allows access to a specific port # within the machine from a port on the host machine and only allow access # via 127.0.0.1 to disable public access # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" # Create a private network, which allows host-only access to the machine # using a specific IP. config.vm.network "private_network", ip: "192.168.56.10" # Create a public network, which generally matched to bridged network. # Bridged networks make the machine appear as another physical device on # your network. # config.vm.network "public_network" # prevent socket thingie to stop wsl /dev/null issue # config.vm.provider "virtualbox" do |vb| # vb.customize [ "modifyvm", :id, "--uartmode1", "disconnected" ] # end # Share an additional folder to the guest VM. The first argument is # the path on the host to the actual folder. The second argument is # the path on the guest to mount the folder. And the optional third # argument is a set of non-required options. # config.vm.synced_folder "../data", "/vagrant_data" # Disable the default share of the current code directory. Doing this # provides improved isolation between the vagrant box and your host # by making sure your Vagrantfile isn't accessable to the vagrant box. # If you use this you may want to enable additional shared subfolders as # shown above. config.vm.synced_folder ".", "/vagrant", disabled: true # Provider-specific configuration so you can fine-tune various # backing providers for Vagrant. These expose provider-specific options. # Example for VirtualBox: # # config.vm.provider "virtualbox" do |vb| # # Display the VirtualBox GUI when booting the machine # vb.gui = true # # # Customize the amount of memory on the VM: # vb.memory = "1024" # end # # View the documentation for the provider you are using for more # information on available options. # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies # such as FTP and Heroku are also available. See the documentation at # https://docs.vagrantup.com/v2/push/atlas.html for more information. # config.push.define "atlas" do |push| # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" # end # Enable provisioning with a shell script. Additional provisioners such as # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the # documentation for more information about their specific syntax and use. ssh_pub_key = File.readlines("#{Dir.home}/.ssh/id_rsa.pub").first.strip# config.vm.provision "shell", inline: <<-SHELL apt update apt upgrade -y apt install -y dtach tree useradd -m -s /bin/bash elspeth usermod -a -G sudo elspeth echo 'elspeth:elspieelspie' | chpasswd mkdir -p /home/elspeth/.ssh cp ~/.ssh/authorized_keys /home/elspeth/.ssh chown elspeth /home/elspeth/.ssh/authorized_keys echo 'elspeth ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/elspeth echo 'export DJANGO_COLORS=nocolor' >> /home/elspeth/.profile echo '#{ssh_pub_key}' >> /home/elspeth/.ssh/authorized_keys SHELL end ================================================ FILE: acknowledgments.asciidoc ================================================ [preface] == Acknowledgments Lots of people to thank, without whom this book would never have happened, and/or would have been even worse than it is. Thanks first to "Greg" at `$OTHER_PUBLISHER`, who was the first person to encourage me to believe it really could be done. Even though your employers turned out to have overly regressive views on copyright, I'm forever grateful that you believed in me. Thanks to Michael Foord, another ex-employee of Resolver Systems, for providing the original inspiration by writing a book himself, and thanks for his ongoing support for the project. Thanks also to my boss Giles Thomas, for foolishly allowing another one of his employees to write a book (although I believe he's now changed the standard employment contract to say "no books"). Thanks also for your ongoing wisdom and for setting me off on the testing path. Thanks to my other colleagues, Glenn Jones and Hansel Dunlop, for being invaluable sounding boards, and for your patience with my one-track-record conversation over the last year. Thanks to my wife, Clementine, and to both my families—without whose support and patience I would never have made it. I apologise for all the time spent with my nose in the computer on what should have been memorable family occasions. I had no idea when I set out what the book would do to my life ("Write it in my spare time, you say? That sounds reasonable..."). I couldn't have done it without you. Thanks to my tech reviewers, Jonathan Hartley, Nicholas Tollervey, and Emily Bache, for your encouragements and invaluable feedback. Especially Emily, who actually conscientiously read every single chapter. Partial credit to Nick and Jon, but that should still be read as eternal gratitude. Having y'all around made the whole thing less of a lonely endeavour. Without all of you, the book would have been little more than the nonsensical ramblings of an idiot. Thanks to everyone else who's given up their time to give some feedback on the book, out of nothing more than the goodness of their heart: Gary Bernhardt, Mark Lavin, Matt O'Donnell, Michael Foord, Hynek Schlawack, Russell Keith-Magee, Andrew Godwin, Kenneth Reitz, and Nathan Stocks. Thanks for being much smarter than I am, and for preventing me from saying several stupid things. Naturally, there are still plenty of stupid things left in the book, for which y'all can absolutely not be held responsible. Thanks to my editor, Meghan Blanchette, for being a very friendly and likeable slave driver, and for keeping the book on track, both in terms of timescales and by restraining my sillier ideas. Thanks to all the others at O'Reilly for your help, including Sarah Schneider, Kara Ebrahim, and Dan Fauxsmith for letting me keep British English. Thanks to Charles Roumeliotis for your help with style and grammar. We may never see eye-to-eye on the merits of Chicago School quotation/punctuation rules, but I sure am glad you were around. And thanks to the design department for giving us a goat for the cover! And thanks most especially to all my early release readers, for all your help picking out typos, for your feedback and suggestions, for all the ways in which you helped to smooth out the learning curve in the book, and most of all for your kind words of encouragement and support that kept me going. Thank you Jason Wirth, Dave Pawson, Jeff Orr, Kevin De Baere, crainbf, dsisson, Galeran, Michael Allan, James O'Donnell, Marek Turnovec, SoonerBourne, julz, Cody Farmer, William Vincent, Trey Hunner, David Souther, Tom Perkin, Sorcha Bowler, Jon Poler, Charles Quast, Siddhartha Naithani, Steve Young, Roger Camargo, Wesley Hansen, Johansen Christian Vermeer, Ian Laurain, Sean Robertson, Hari Jayaram, Bayard Randel, Konrad Korżel, Matthew Waller, Julian Harley, Barry McClendon, Simon Jakobi, Angelo Cordon, Jyrki Kajala, Manish Jain, Mahadevan Sreenivasan, Konrad Korżel, Deric Crago, Cosmo Smith, Markus Kemmerling, Andrea Costantini, Daniel Patrick, Ryan Allen, Jason Selby, Greg Vaughan, Jonathan Sundqvist, Richard Bailey, Diane Soini, Dale Stewart, Mark Keaton, Johan Wärlander, Simon Scarfe, Eric Grannan, Marc-Anthony Taylor, Maria McKinley, John McKenna, Rafał Szymański, Roel van der Goot, Ignacio Reguero, TJ Tolton, Jonathan Means, Theodor Nolte, Jungsoo Moon, Craig Cook, Gabriel Ewilazarus, Vincenzo Pandolfo, David "farbish2", Nico Coetzee, Daniel Gonzalez, Jared Contrascere, Zhao 赵亮, and many, many more. If I've missed your name, you have an absolute right to be aggrieved; I am incredibly grateful to you too, so write to me and I will try and make it up to you in any way I can. And finally thanks to you, the latest reader, for deciding to check out the book! I hope you enjoy it. [role="pagebreak-before less_space"] === Additional Thanks for the Second Edition Thanks to my wonderful editor for the second edition, Nan Barber, and to Susan Conant, Kristen Brown, and the whole team at O'Reilly. Thanks once again to Emily and Jonathan for tech reviewing, as well as to Edward Wong for his very thorough notes. Any remaining errors and inadequacies are all my own. Thanks also to the readers of the free edition who contributed comments, suggestions, and even some pull requests. I have definitely missed some of you on this list, so apologies if your name isn't here, but thanks to Emre Gonulates, Jésus Gómez, Jordon Birk, James Evans, Iain Houston, Jason DeWitt, Ronnie Raney, Spencer Ogden, Suresh Nimbalkar, Darius, Caco, LeBodro, Jeff, Duncan Betts, wasabigeek, joegnis, Lars, Mustafa, Jared, Craig, Sorcha, TJ, Ignacio, Roel, Justyna, Nathan, Andrea, Alexandr, bilyanhadzhi, mosegontar, sfarzy, henziger, hunterji, das-g, juanriaza, GeoWill, Windsooon, gonulate, Margie Roswell, Ben Elliott, Ramsay Mayka, peterj, 1hx, Wi, Duncan Betts, Matthew Senko, Neric "Kasu" Kaz, Dominic Scotto, Andrey Makarov, and many, many more. === Additional Thanks for the Third Edition Thanks to my editor, Rita Fernando, thanks to my tech reviewers Béres Csanád, David Seddon, Sebastian Buczyński, and Jan Giacomelli, and thanks to all the early release readers for your feedback, big and small, including Jonathan H., James Evans, Patrick Cantwell, Devin Schumacher, Nick Nielsen, Teemu Viikeri, Andrew Zipperer, artGonza, Joy Denebeim, mshob23, Romilly Cocking, Zachary Kerbo, Stephanie Goulet, David Carter, Jim Win Man, Alex Kennett, Ivan Schneider, Lars Berberich, Rodrigo Jacznik, Tom Nguyen, rokbot, Nikita Durne, and to anyone I've missed off this list, my sincere apologies, ping me and I'll add you, and thank you thank you once again. .Extra Thanks for Csanàd ******************************************************************************* Every single one of the tech reviewers for this edition was invaluable, and they all contributed in different and complementary ways. But I want to give extra thanks to Csanàd, who went beyond the normal remit of a tech reviewer, so far as to do substantial actual rewrites of several chapters in <>. You can't blame him for anything in there though, because I've been over them since, so any errors or problems you might spot are definitely things I've added since. Anyways, thanks so much Csanàd, you helped me feel like I wasn't entirely alone. ******************************************************************************* ================================================ FILE: ai_preface.asciidoc ================================================ [[ai_preface]] [preface] == Preface to the Third Edition: [.keep-together]#TDD in the Age of AI# Is there any point in learning TDD now that AI can write code for you? A single prompt could probably generate the entire example application in this book, including all its tests, and the infrastructure config for automated deployment too. The truth is that it's too early to tell. AI is still in its infancy, and who knows what it'll be able to do in a few years or even months' time. === AI Is Both Insanely Impressive and Incredibly Unreliable What we do know is that right now, AI is both insanely impressive and incredibly unreliable. Beyond being able to understand and respond to prompts in normal human language--it's easy to forget how absolutely extraordinary that is; literally science-fiction a few years ago--AI tools can generate working code, they can generate tests, they can help us to break down requirements, brainstorm solutions, quickly prototype new technologies. It's genuinely astonishing. As we're all finding out though, this all comes with a massive "but". AI outputs are frequently plagued by hallucinations, and in the world of code, that means things that just won't work, even if they look plausible. Worse than that, they can produce code that appears to work, but is full of subtle bugs, security issues, or performance nightmares. From a code quality point of view, we know that AI tools will often produce code that's filled with copy-paste and duplication, weird hacks, and undecipherable spaghetti code that spells a maintenance nightmare. === Mitigations for AI's Shortcomings Sure Look a Lot Like TDD If you read the advice, even from AI companies themselves, about the best way to work with AI, you'll find that it performs best when working in small, well-defined contexts, with frequent checks for correctness. When taking on larger tasks, the advice is to break them down into smaller, well-defined pieces, with clearly defined success criteria. When we're thinking about the problem of hallucinations, it sure seems like having a comprehensive test suite and running it frequently, is going to be a must-have. When we're thinking about code quality, the idea of having a human in the loop, with frequent pauses for review and refactoring, again seems like a key mitigation. In short, all of the techniques of test-driven development that are outlined in this book: * Defining a test that describes each small change of functionality, before we write the code for it * Breaking our problem down into small pieces and working incrementally, with frequent test runs to catch bugs, regressions, and hallucinations * The "refactor" step in TDD's red/green/refactor cycle, which gives us a regular reminder for the human in the loop to review and improve the code. TDD is all about finding a structured, safer way of developing software, reducing the risk of bugs and regressions and improving code quality, and these are very much the exact same things that we need to achieve when working with AI. === Leaky Abstractions and the Importance of Experience https://oreil.ly/PgWjL["Leaky abstractions"] are a diagnosis of a common problem in software development, whereby higher-level abstractions fail in subtle ways, and the complexities of the underlying system leak through. In the presence of leaky abstractions, you need to understand the lower-level system to be able to work effectively. It's for this reason that, when the switch to third-generation languages (3GLs) happened, programmers who understood the underlying machine code were often the most effective at using the new languages like C and Fortran. In a similar way, AI offers us a new, higher-level abstraction around writing code, but we can already see the "leaks" in the form of hallucinations and poor code quality. And by analogy to the 3GLs, the programmers who are going to be most effective with AI are going to be the ones who "know what good looks like", both in terms of code quality, test structure, and so on, but also in terms of what a safe and reliable workflow for software development looks like. ==== My Own Experiences with AI In my own experiences of working with AI, I've been very impressed at its ability to write tests, for example... as long as there was already a good first example test to copy from. Its ability to write that _first_ test, the one where, as we'll see, a lot of the design (and thinking) happens in TDD, was much more mixed. // SEBASTIAN: Idea for a frame with actionable advice: "Make sure there are at least few examples of tests before you'll make AI assistant write more. Show it what 'good' looks like." // SEBASTIAN: Another idea for a frame with actionable advice: "Simple things like writing a meaningful test name or describing in comment what you want to test can help immensely AI working in autocomplete mode." Similarly when working in a less "autocomplete" and more "agentic" mode, I saw AI tools do very well on simple problems with clear instructions, but when trying to deal with more complex logic and requirements with ambiguity, I've seen it get dreadfully stuck in loops and dead ends. When that happened, I found that trying to guide the AI agent back towards taking small steps, working on a single piece at a time, and clarifying requirements in tests, was the best way to get things back on track. I've also been able to experiment with using the "refactor" step to try and improve the often-terrible code that the AI produced. Here again I had mixed results, where the AI would need a lot of nudging before settling on a solution that felt sufficiently readable and maintainable, to me. So I'd echo what many others are saying, which is that AI works best when you, the user, are a discerning partner rather than passive recipient. NOTE: Ultimately, as software developers, we need to be able to stand by the code we produce, and be accountable for it, no matter what tools were used to write it. === The AI-Enabled Workflow of the Future The AI-enabled workflow of the future will look very different to what we have now, but all the indications are that the most effective approach is going to be incremental, have checks and balances to avoid hallucinations, and systematically involve humans in the loop to ensure quality. And the closest workflow we have to that today, is TDD. I'm excited to share it with you! ================================================ FILE: analytics.html ================================================ ================================================ FILE: appendix_CD.asciidoc ================================================ [[appendix_CD]] [appendix] == Continuous Deployment (CD) .Warning ******************************************************************************* This appendix is just a placeholder / rough sketch. It should have the outline of what you need to set up automated deploys tho! Why not give it a try? ******************************************************************************* ((("continuous delivery (CD)"))) This is the next step after CI. Once we have a server that automatically does things every time we push, we can take the next step in automating our deploys, and deploy our code to staging (and even production!) with every push. NOTE: "CD" sometimes stands for Continuous Deployment, when used to contrast with "CI", and sometimes it stands for "Continuous Delivery", which is basically a combination of CI and CD. Never forget, the purpose of acronyms is to differentiate insiders from outsiders, so the confusion _is_ the point. * This is an appendix because we get even more tied in to the particularities of an individual platform * It's also incredibly fiddly. the feedback cycle is annoying slow, and you have to commit and push with every small change. just look at my commit history! [role="skipme"] ---- f5d58736 some tidyup f28411a0 disable host key checking again a2933ad4 dammit forgot curl fb4132ec use private keyfile in ssh commands ce7219e3 install ssh for fts 957ca269 fix stage name dae47804 run fts against staging after deploy 17999c65 fix the way we get env vars in ansible script 87aecc62 make secrets files private for ssh a06d24e9 switch off host key checking 059fc15e lets try for superverbose debug output 021843db Revert "quick look at end of keypair" 56d79af4 quick look at end of keypair bc5664c6 fix path to secure files 857c803a install curl c37a538c get ssh key from secure files 5ffbf80f install ssh on python image d4f39755 duh stupid typo c34cf933 try to deploy using gitlab registry. add stages 62486de1 docker login using password from env 4bdc6f53 fix tags in docker push to gitlab registry c5a0056c try pushing to gitlab 81c8601f temporarily dont moujnt db 6bd41a1f forgot dind 2de01bf0 move python before-script stuff in to test step d11c21fe try to build docker 76f15efb temporarily dont run fts 16db3dc1 debug finding path to playbook 1f3f77f5 remove backslashes ad46cd12 just do it inline 1c887270 add deploy step 6f77b2df venv paths 801c8373 try and make actual ci work ba8be943 Gitlab yaml config ---- Tricky! Building and running a docker image can only be done on a `docker.git` image, but we want `python:slim` to run our tests, and to actually have Ansible installed *idea 1:* - build and push a docker image to gitlab registry after each ci run - deploy to staging using the new image tag - run tests against staging *idea 2:* - run tests inside docker (needs an image with firefox tho) - run fts inside docker against _another_ docker container - deploy from inside docker I've seen variants on both of these. Gave idea 1 a go first, and it worked out: first (or, very quickly), i commented out the fts part of the tests. one of the worst things about fiddling with ci is how slow it is to get feedback: [role="sourcecode"] ..gitlab-ci.yml ==== [source,yaml] ---- test: image: python:slim before_script: # TODO temporarily commented out # - apt update -y && apt install -y firefox-esr - python --version ; pip --version # For debugging - pip install virtualenv - virtualenv .venv - source .venv/bin/activate script: [...] ---- ==== recap: 1. run tests in python image (with firefox and our virtualenv / requirements.txt) 2. build docker image in a docker-in-docker image 3. deploy to staging (from the python image once again, needs ansible) 4. run fts against staging (from the python image, with firefox) now, deploy playbook currently assumes we're building the docker image as part of the deploy, but we can't do that because it happened on a different image we could use cache / "build artifacts" to move the image around, but we may as well do something that's more like real life. you remember i said the `docker push / docker load` dance was a simulation of `push+pull` from a "container registry"? well let's do that. 1. run tests (python image) 2. build our image AND push to registry (docker image) 3. deploy to staging referencing our image in the registry (python image) 4. run fts against staging (python image, with firefox) === Building our docker image and pushing it to Gitlab registry TODO: gitlab container registry screnshot [role="sourcecode"] ..gitlab-ci.yml ==== [source,yaml] ---- build: image: docker:git services: - docker:dind script: - docker build -t registry.gitlab.com/hjwp/book-example:$CI_COMMIT_SHA . - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY -u $CI_REGISTRY_USER --password-stdin - docker push registry.gitlab.com/hjwp/book-example:$CI_COMMIT_SHA ---- ==== link to gitlab registry docs, explain docker login, image tags. === Deploying from CI, working with secrets [role="sourcecode"] ..gitlab-ci.yml ==== [source,yaml] ---- deploy: stage: staging-deploy image: python:slim variables: ANSIBLE_HOST_KEY_CHECKING: "False" # <1> before_script: - apt update -y && apt install -y curl openssh-client - python --version ; pip --version # For debugging - pip install virtualenv - virtualenv .venv - source .venv/bin/activate script: - pip install -r requirements.txt - pip install ansible # download secure files to get private key # <2> - curl -s https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer | bash - chmod 600 .secure_files/* - ansible-playbook --private-key=.secure_files/keypair-for-gitlab # <2> --user=elspeth -i staging.ottg.co.uk, -vvv # <3> ${PWD}/infra/deploy-playbook.yaml ---- ==== <1> "known hosts" checking doesnt work well in ci <2> we needed a way to give the ci server permission to access our server. I used a new ssh key <3> super-verbose was necessary TODO: explain generating ssh key, adding to `/home/elpseth/.ssh/authorized_keys` on server. short listing, couple of hours of pain! eg had to run thru about 200 lines of verbose logs to find this, and then a bit of web-searching, to figure out that known-hosts was the problem: [role="skipme"] ---- debug1: Server host key: ssh-ed25519 SHA256:4kXU5nf93OCxgBMuhr+OC8OUct6xb8yGsRjrqmLTJ7g debug1: load_hostkeys: fopen /root/.ssh/known_hosts: No such file or directory debug1: load_hostkeys: fopen /root/.ssh/known_hosts2: No such file or directory debug1: load_hostkeys: fopen /etc/ssh/ssh_known_hosts: No such file or directory debug1: load_hostkeys: fopen /etc/ssh/ssh_known_hosts2: No such file or directory debug1: hostkeys_find_by_key_hostfile: hostkeys file /root/.ssh/known_hosts does not exist debug1: hostkeys_find_by_key_hostfile: hostkeys file /root/.ssh/known_hosts2 does not exist debug1: hostkeys_find_by_key_hostfile: hostkeys file /etc/ssh/ssh_known_hosts does not exist debug1: hostkeys_find_by_key_hostfile: hostkeys file /etc/ssh/ssh_known_hosts2 does not exist debug1: read_passphrase: can't open /dev/tty: No such device or address Host key verification failed.", "unreachable": true} ---- === Updating deploy playbook to use the container registry: We delete all the stages to do with building locally and uploading and re-importing: [role="sourcecode skipme"] .infra/deploy-playbook.yaml ==== [source,diff] ---- @@ -19,37 +19,6 @@ - name: Reset ssh connection to allow the user/group change to take effect ansible.builtin.meta: reset_connection - - name: Build container image locally - - name: Export container image locally - - name: Upload image to server - - name: Import container image on server ---- ==== And instead, we can just use the full path to the image in our `docker run` (with a login to the registry first): [role="sourcecode skipme"] .infra/deploy-playbook.yaml ==== [source,yaml] ---- - name: Login to gitlab container registry community.docker.docker_login: registry_url: "{{ lookup('env', 'CI_REGISTRY') }}" # <1> username: "{{ lookup('env', 'CI_REGISTRY_USER') }}" # <1> password: "{{ lookup('env', 'CI_REGISTRY_PASSWORD') }}" # <1> - name: Run container community.docker.docker_container: name: superlists image: registry.gitlab.com/hjwp/book-example:{{ lookup('env', 'CI_COMMIT_SHA') }} # <2> state: started recreate: true [...] ---- ==== <1> just like in the ci script, we use the env vars to get the login details <2> and we spell out the registry, with the commit sha, in the image name === Running Fts against staging Add explicit "stages" to make things run in order: [role="sourcecode"] ..gitlab-ci.yml ==== [source,yaml] ---- stages: - build-and-test - staging-deploy - staging-test test: image: python:slim stage: build-and-test [...] build: image: docker:git services: - docker:dind stage: build-and-test script: [...] test-staging: image: python:slim stage: staging-test [...] ---- ==== And here's how we run the tests against staging: [role="sourcecode"] ..gitlab-ci.yml ==== [source,yaml] ---- test-staging: image: python:slim stage: staging-test before_script: - apt update -y && apt install -y curl firefox-esr # <1> openssh-client - python --version ; pip --version # For debugging - pip install virtualenv - virtualenv .venv - source .venv/bin/activate script: - pip install -r requirements.txt - pip install selenium - curl -s https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer | bash - chmod 600 .secure_files/* # <2> - env TEST_SERVER=staging.ottg.co.uk SSH_PRIVATE_KEY_PATH=.secure_files/keypair-for-gitlab # <2> python src/manage.py test functional_tests ---- ==== <1> we need firefox for the fts <2> we needed the ssh key again, because as you might remember (i forgot!) the fts use ssh to talk to the db on the server, to manage the database. So we need some changes in the base FT too: [role="sourcecode"] .lists.tests.py (ch04l004) ==== [source,python] ---- def _exec_in_container_on_server(host, commands): print(f"Running {commands!r} on {host} inside docker container") keyfile = os.environ.get("SSH_PRIVATE_KEY_PATH") keyfile_arg = ["-i", keyfile, "-o", "StrictHostKeyChecking=no"] if keyfile else [] # <1><2> return _run_commands( ["ssh"] + keyfile_arg + [f"{USER}@{host}", "docker", "exec", "superlists"] + commands ) ---- ==== <1> `-i` tells ssh to use a specific private key <2> `-o StrictHostKeyChecking=no` is how we disable known_hosts for the ssh client at the command-line and that works TODO it works deploy screenshot .CD Recap ******************************************************************************* Feedback cycles:: Slow. try to make faster. Secrets:: secret key, email password. each platform is different but there's always a way. careful not to print things out! ******************************************************************************* ================================================ FILE: appendix_DjangoRestFramework.asciidoc ================================================ [[appendix_DjangoRestFramework]] [appendix] Django-Rest-Framework --------------------- ((("Django-Rest-Framework (DRF)", id="DRF33")))Having "rolled our own" REST API in the last appendix, it's time to take a look at http://www.django-rest-framework.org/[Django-Rest-Framework], which is a go-to choice for many Python/Django developers building APIs. Just as Django aims to give you all the basic tools that you'll need to build a database-driven website (an ORM, templates, and so on), so DRF aims to give you all the tools you need to build an API, and thus avoid you having to write boilerplate code over and over again. Writing this appendix, one of the main things I struggled with was getting the exact same API that I'd just implemented manually to be replicated by DRF. Getting the same URL layout and the same JSON data structures I'd defined proved to be quite a challenge, and I felt like I was fighting the framework. That's always a warning sign. The people who built Django-Rest-Framework are a lot smarter than I am, and they've seen a lot more REST APIs than I have, and if they're opinionated about the way that things "should" look, then maybe my time would be better spent seeing if I can adapt and work with their view of the world, rather than forcing my own preconceptions onto it. "Don't fight the framework" is one of the great pieces of advice I've heard. Either go with the flow, or perhaps reassess whether you want to be using a framework at all. We'll work from the API we had at the end of the https://www.obeythetestinggoat.com/book/appendix_rest_api.html[Online Appendix: Building a REST API] and see if we can rewrite it to use DRF. Installation ~~~~~~~~~~~~ ((("Django-Rest-Framework (DRF)", "installation")))A quick `pip install` gets us DRF. I'm just using the latest version, which was 3.5.4 at the time of writing: [subs="specialcharacters,quotes"] ---- $ *pip install djangorestframework* ---- And we add `rest_framework` to `INSTALLED_APPS` in 'settings.py': [role="sourcecode"] .superlists/settings.py ==== [source,python] ---- INSTALLED_APPS = [ #'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'lists', 'accounts', 'functional_tests', 'rest_framework', ] ---- ==== Serializers (Well, ModelSerializers, Really) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("Django-Rest-Framework (DRF)", "tutorials")))((("Django-Rest-Framework (DRF)", "ModelSerializers")))The http://bit.ly/2t6T6eX[Django-Rest-Framework tutorial] is a pretty good resource to learn DRF. The first thing you'll come across is serializers, and specifically in our case, "ModelSerializers". They are DRF's way of converting from Django database models to JSON (or possibly other formats) that you can send over the wire: // IDEA: add an explicit unit test or two for serialization [role="sourcecode"] .lists/api.py (ch37l003) ==== [source,python] ---- from lists.models import List, Item [...] from rest_framework import routers, serializers, viewsets class ItemSerializer(serializers.ModelSerializer): class Meta: model = Item fields = ('id', 'text') class ListSerializer(serializers.ModelSerializer): items = ItemSerializer(many=True, source='item_set') class Meta: model = List fields = ('id', 'items',) ---- ==== [role="pagebreak-before"] Viewsets (Well, ModelViewsets, Really) and Routers ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("Django-Rest-Framework (DRF)", "ModelViewsets")))A ModelViewSet is DRF's way of defining all the different ways you can interact with the objects for a particular model via your API. Once you tell it which models you're interested in (via the `queryset` attribute) and how to serialize them (`serializer_class`), it will then do the rest--automatically building views for you that will let you list, retrieve, update, and even delete objects. Here's all we need to do for a ViewSet that'll be able to retrieve items for a particular list: [role="sourcecode"] .lists/api.py (ch37l004) ==== [source,python] ---- class ListViewSet(viewsets.ModelViewSet): queryset = List.objects.all() serializer_class = ListSerializer router = routers.SimpleRouter() router.register(r'lists', ListViewSet) ---- ==== A 'router' is DRF's way of building URL configuration automatically, and mapping them to the functionality provided by the ViewSet. At this point we can start pointing our 'urls.py' at our new router, bypassing the old API code and seeing how our tests do with the new stuff: [role="sourcecode"] .superlists/urls.py (ch37l005) ==== [source,python] ---- [...] # from lists.api import urls as api_urls from lists.api import router urlpatterns = [ url(r'^$', list_views.home_page, name='home'), url(r'^lists/', include(list_urls)), url(r'^accounts/', include(accounts_urls)), # url(r'^api/', include(api_urls)), url(r'^api/', include(router.urls)), ] ---- ==== That makes loads of our tests fail: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] django.urls.exceptions.NoReverseMatch: Reverse for 'api_list' not found. 'api_list' is not a valid view function or pattern name. [...] AssertionError: 405 != 400 [...] AssertionError: {'id': 2, 'items': [{'id': 2, 'text': 'item 1'}, {'id': 3, 'text': 'item 2'}]} != [{'id': 2, 'text': 'item 1'}, {'id': 3, 'text': 'item 2'}] --------------------------------------------------------------------- Ran 54 tests in 0.243s FAILED (failures=4, errors=10) ---- Let's take a look at those 10 errors first, all saying they cannot reverse `api_list`. It's because the DRF router uses a different naming convention for URLs than the one we used when we coded it manually. You'll see from the tracebacks that they're happening when we render a template. It's 'list.html'. We can fix that in just one place; `api_list` becomes `list-detail`: [role="sourcecode"] .lists/templates/list.html (ch37l006) ==== [source,html] ---- ---- ==== That will get us down to just four failures: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] FAIL: test_POSTing_a_new_item (lists.tests.test_api.ListAPITest) [...] FAIL: test_duplicate_items_error (lists.tests.test_api.ListAPITest) [...] FAIL: test_for_invalid_input_returns_error_code (lists.tests.test_api.ListAPITest) [...] FAIL: test_get_returns_items_for_correct_list (lists.tests.test_api.ListAPITest) [...] FAILED (failures=4) ---- //TODO use @skip Let's DONT-ify all the validation tests for now, and save that complexity for later: [role="sourcecode"] .lists/tests/test_api.py (ch37l007) ==== [source,python] ---- [...] def DONTtest_for_invalid_input_nothing_saved_to_db(self): [...] def DONTtest_for_invalid_input_returns_error_code(self): [...] def DONTtest_duplicate_items_error(self): [...] ---- ==== And now we have just two failures: [subs="specialcharacters,macros"] ---- FAIL: test_POSTing_a_new_item (lists.tests.test_api.ListAPITest) [...] self.assertEqual(response.status_code, 201) AssertionError: 405 != 201 [...] FAIL: test_get_returns_items_for_correct_list (lists.tests.test_api.ListAPITest) [...] AssertionError: {'id': 2, 'items': [{'id': 2, 'text': 'item 1'}, {'id': 3, 'text': 'item 2'}]} != [{'id': 2, 'text': 'item 1'}, {'id': 3, 'text': 'item 2'}] [...] FAILED (failures=2) ---- Let's take a look at that last one first. DRF's default configuration does provide a slightly different data structure to the one we built by hand--doing a GET for a list gives you its ID, and then the list items are inside a key called "items". That means a slight modification to our unit test, before it gets back to passing: [role="sourcecode"] .lists/tests/test_api.py (ch37l008) ==== [source,diff] ---- @@ -23,10 +23,10 @@ class ListAPITest(TestCase): response = self.client.get(self.base_url.format(our_list.id)) self.assertEqual( json.loads(response.content.decode('utf8')), - [ + {'id': our_list.id, 'items': [ {'id': item1.id, 'text': item1.text}, {'id': item2.id, 'text': item2.text}, - ] + ]} ) ---- ==== That's the GET for retrieving list items sorted (and, as we'll see later, we've got a bunch of other stuff for free too). How about adding new ones, using POST? A Different URL for POST Item ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("Django-Rest-Framework (DRF)", "POST requests")))This is the point at which I gave up on fighting the framework and just saw where DRF wanted to take me. Although it's possible, it's quite torturous to do a POST to the "lists" ViewSet in order to add an item to a list. Instead, the simplest thing is to post to an item view, not a list view: [role="sourcecode"] .lists/api.py (ch37l009) ==== [source,python] ---- class ItemViewSet(viewsets.ModelViewSet): serializer_class = ItemSerializer queryset = Item.objects.all() [...] router.register(r'items', ItemViewSet) ---- ==== So that means we change the test slightly, moving all the POST tests out of the [keep-together]#`ListAPITest`# and into a new test class, `ItemsAPITest`: [role="sourcecode"] .lists/tests/test_api.py (ch37l010) ==== [source,python] ---- @@ -1,3 +1,4 @@ import json +from django.core.urlresolvers import reverse from django.test import TestCase from lists.models import List, Item @@ -31,9 +32,13 @@ class ListAPITest(TestCase): + +class ItemsAPITest(TestCase): + base_url = reverse('item-list') + def test_POSTing_a_new_item(self): list_ = List.objects.create() response = self.client.post( - self.base_url.format(list_.id), - {'text': 'new item'}, + self.base_url, + {'list': list_.id, 'text': 'new item'}, ) self.assertEqual(response.status_code, 201) ---- ==== That will give us: ---- django.db.utils.IntegrityError: NOT NULL constraint failed: lists_item.list_id ---- Until we add the list ID to our serialization of items; otherwise, we don't know what list it's for: [role="sourcecode"] .lists/api.py (ch37l011) ==== [source,python] ---- class ItemSerializer(serializers.ModelSerializer): class Meta: model = Item fields = ('id', 'list', 'text') ---- ==== And that causes another small associated test change: [role="sourcecode"] .lists/tests/test_api.py (ch37l012) ==== [source,python] ---- @@ -25,8 +25,8 @@ class ListAPITest(TestCase): self.assertEqual( json.loads(response.content.decode('utf8')), {'id': our_list.id, 'items': [ - {'id': item1.id, 'text': item1.text}, - {'id': item2.id, 'text': item2.text}, + {'id': item1.id, 'list': our_list.id, 'text': item1.text}, + {'id': item2.id, 'list': our_list.id, 'text': item2.text}, ]} ) ---- ==== Adapting the Client Side ~~~~~~~~~~~~~~~~~~~~~~~~ ((("Django-Rest-Framework (DRF)", "client-side adaptations")))Our API no longer returns a flat array of the items in a list. It returns an object, with a `.items` attribute that represents the items. That means a small tweak to our +update​Items+ function: [role="sourcecode"] .lists/static/list.js (ch37l013) ==== [source,diff] ---- @@ -3,8 +3,8 @@ window.Superlists = {}; window.Superlists.updateItems = function (url) { $.get(url).done(function (response) { var rows = ''; - for (var i=0; i'; } $('#id_list_table').html(rows); ---- ==== And because we're using different URLs for GETing lists and POSTing items, we tweak the `initialize` function slightly too. Rather than multiple arguments, we'll switch to using a `params` object containing the required config: [role="sourcecode small-code"] .lists/static/list.js ==== [source,diff] ---- @@ -11,23 +11,24 @@ window.Superlists.updateItems = function (url) { }); }; -window.Superlists.initialize = function (url) { +window.Superlists.initialize = function (params) { $('input[name="text"]').on('keypress', function () { $('.has-error').hide(); }); - if (url) { - window.Superlists.updateItems(url); + if (params) { + window.Superlists.updateItems(params.listApiUrl); var form = $('#id_item_form'); form.on('submit', function(event) { event.preventDefault(); - $.post(url, { + $.post(params.itemsApiUrl, { + 'list': params.listId, 'text': form.find('input[name="text"]').val(), 'csrfmiddlewaretoken': form.find('input[name="csrfmiddlewaretoken"]').val(), }).done(function () { $('.has-error').hide(); - window.Superlists.updateItems(url); + window.Superlists.updateItems(params.listApiUrl); }).fail(function (xhr) { $('.has-error').show(); if (xhr.responseJSON && xhr.responseJSON.error) { ---- ==== We reflect that in 'list.html': [role="sourcecode"] .lists/templates/list.html (ch37l014) ==== [source,html] ---- $(document).ready(function () { window.Superlists.initialize({ listApiUrl: "{% url 'list-detail' list.id %}", itemsApiUrl: "{% url 'item-list' %}", listId: {{ list.id }}, }); }); ---- ==== And that's actually enough to get the basic FT working again: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests.test_simple_list_creation*] [...] Ran 2 tests in 15.635s OK ---- There's a few more changes to do with error handling, which you can explore in the https://github.com/hjwp/book-example/blob/appendix_DjangoRestFramework/lists/api.py[repo for this appendix] if you're curious. What Django-Rest-Framework Gives You ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("Django-Rest-Framework (DRF)", "benefits of")))You may be wondering what the point of using this framework was. Configuration Instead of Code ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Well, the first advantage is that I've transformed my old procedural view function into a more declarative syntax: [role="sourcecode currentcontents dofirst-ch37l016"] .lists/api.py ==== [source,python] ---- def list(request, list_id): list_ = List.objects.get(id=list_id) if request.method == 'POST': form = ExistingListItemForm(for_list=list_, data=request.POST) if form.is_valid(): form.save() return HttpResponse(status=201) else: return HttpResponse( json.dumps({'error': form.errors['text'][0]}), content_type='application/json', status=400 ) item_dicts = [ {'id': item.id, 'text': item.text} for item in list_.item_set.all() ] return HttpResponse( json.dumps(item_dicts), content_type='application/json' ) ---- ==== If you compare this to the final DRF version, you'll notice that we are actually now entirely configured: [role="sourcecode currentcontents dofirst-ch37l019"] .lists/api.py ==== [source,python] ---- class ItemSerializer(serializers.ModelSerializer): text = serializers.CharField( allow_blank=False, error_messages={'blank': EMPTY_ITEM_ERROR} ) class Meta: model = Item fields = ('id', 'list', 'text') validators = [ UniqueTogetherValidator( queryset=Item.objects.all(), fields=('list', 'text'), message=DUPLICATE_ITEM_ERROR ) ] class ListSerializer(serializers.ModelSerializer): items = ItemSerializer(many=True, source='item_set') class Meta: model = List fields = ('id', 'items',) class ListViewSet(viewsets.ModelViewSet): queryset = List.objects.all() serializer_class = ListSerializer class ItemViewSet(viewsets.ModelViewSet): serializer_class = ItemSerializer queryset = Item.objects.all() router = routers.SimpleRouter() router.register(r'lists', ListViewSet) router.register(r'items', ItemViewSet) ---- ==== Free Functionality ^^^^^^^^^^^^^^^^^^ The second advantage is that, by using DRF's ModelSerializer, ViewSet, and routers, I've actually ended up with a much more extensive API than the one I'd rolled by hand. * All the HTTP methods, GET, POST, PUT, PATCH, DELETE, and OPTIONS, now work, out of the box, for all list and items URLs. * And a browsable/self-documenting version of the API is available at pass:[http://localhost:8000/api/lists/] and pass:[http://localhost:8000/api/items]. (<>; try it!) [[figag01]] .A free browsable API for your users image::images/twp2_ag01.png["Screenshot of DRF browsable api page at http://localhost:8000/api/items/"] There's more information in http://www.django-rest-framework.org/topics/documenting-your-api/#self-describing-apis[the DRF docs], but those are both seriously neat features to be able to offer the end users of your API. In short, DRF is a great way of generating APIs, almost automatically, based on your existing models structure. If you're using Django, definitely check it out before you start hand-rolling your own API code. .Django-Rest-Framework Tips ******************************************************************************* ((("Django-Rest-Framework (DRF)", "tips for")))Don't fight the framework:: Going with the flow is often the best way to stay productive. That, or maybe don't use the framework. Or use it at a lower level. Routers and ViewSets for the principle of least surprise:: One of the advantages of DRF is that its generic tools like routers and ViewSets will give you a very predictable API, with sensible defaults for its endpoints, URL structure, and responses for different HTTP methods. Check out the self-documenting, browsable version:: Check out your API endpoints in a browser. DRF responds differently when it detects your API is being accessed by a "normal" web browser, and displays a very nice, self-documenting version of itself, which you can share with your users.((("", startref="DRF33"))) ******************************************************************************* ================================================ FILE: appendix_Django_Class-Based_Views.asciidoc ================================================ [[appendix_Django_Class-Based_Views]] [appendix] Django Class-Based Views ------------------------ ((("Django framework", "class-based generic views", id="DJFclass28")))This appendix follows on from <>, in which we implemented Django forms for validation and refactored our views. By the end of that chapter, our views were still using functions. The new shiny in the Django world, however, is class-based views. In this appendix, we'll refactor our application to use them instead of view functions. More specifically, we'll have a go at using class-based 'generic' views. Class-Based Generic Views ~~~~~~~~~~~~~~~~~~~~~~~~~ ((("class-based generic views (CBGVs)", "vs. class-based views", secondary-sortas="class-based views")))There's a difference between class-based views and class-based 'generic' views. Class-based views (CBVs) are just another way of defining view functions. They make few assumptions about what your views will do, and they offer one main advantage over view functions, which is that they can be subclassed. This comes, arguably, at the expense of being less readable than traditional function-based views. The main use case for 'plain' class-based views is when you have several views that reuse the same logic. We want to obey the DRY principle. With function-based views, you would use helper functions or decorators. The theory is that using a class structure may give you a more elegant solution. Class-based 'generic' views (CBGVs) are class-based views that attempt to provide ready-made solutions to common use cases: fetching an object from the database and passing it to a template, fetching a list of objects, saving user input from a POST request using a +ModelForm+, and so on. These sound very much like our use cases, but as we'll soon see, the devil is in the details. I should say at this point that I've not used either kind of class-based views much. I can definitely see the sense in them, and there are potentially many use cases in Django apps where CBGVs would fit in perfectly. However, as soon as your use case is slightly outside the basics--as soon as you have more than one model you want to use, for example--I find that using class-based views can (again, debatably) lead to code that's much harder to read than a classic view function. Still, because we're forced to use several of the customisation options for class-based views, implementing them in this case can teach us a lot about how they work, and how we can unit test them. My hope is that the same unit tests we use for function-based views should work just as well for class-based views. Let's see how we get on. The Home Page as a FormView ~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("class-based generic views (CBGVs)", "home page as a FormView")))Our home page just displays a form on a template: [role="sourcecode currentcontents"] .lists/views.py ==== [source,python] ---- def home_page(request): return render(request, 'home.html', {'form': ItemForm()}) ---- ==== https://docs.djangoproject.com/en/5.2/ref/class-based-views/[Looking through the options], Django has a generic view called `FormView`—let's see how that goes: [role="sourcecode"] .lists/views.py (ch31l001) ==== [source,python] ---- from django.views.generic import FormView [...] class HomePageView(FormView): template_name = 'home.html' form_class = ItemForm ---- ==== We tell it what template we want to use, and which form. Then, we just need to update 'urls.py', replacing the line that used to say `lists.views.home_page`: [role="sourcecode"] .superlists/urls.py (ch31l002) ==== [source,python] ---- [...] urlpatterns = [ url(r'^$', list_views.HomePageView.as_view(), name='home'), url(r'^lists/', include(list_urls)), ] ---- ==== And the tests all check out! That was easy... [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] Ran 34 tests in 0.119s OK ---- [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests*] [...] Ran 5 tests in 15.160s OK ---- So far, so good. We've replaced a one-line view function with a two-line class, but it's still very readable. This would be a good time for a commit... Using form_valid to Customise a CreateView ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("class-based generic views (CBGVs)", "customizing a CreateView", id="CBGVcreate28")))((("form_valid")))Next we have a crack at the view we use to create a brand new list, currently the `new_list` function. Here's what it looks like now: [role="sourcecode currentcontents"] .lists/views.py ==== [source,python] ---- def new_list(request): form = ItemForm(data=request.POST) if form.is_valid(): list_ = List.objects.create() form.save(for_list=list_) return redirect(list_) else: return render(request, 'home.html', {"form": form}) ---- ==== Looking through the possible CBGVs, we probably want a `CreateView`, and we know we're using the `ItemForm` class, so let's see how we get on with them, and whether the tests will help us: [role="sourcecode"] .lists/views.py (ch31l003) ==== [source,python] ---- from django.views.generic import FormView, CreateView [...] class NewListView(CreateView): form_class = ItemForm def new_list(request): [...] ---- ==== I'm going to leave the old view function in 'views.py', so that we can copy code across from it. We can delete it once everything is working. It's harmless as soon as we switch over the URL mappings, this time in: [role="sourcecode"] .lists/urls.py (ch31l004) ==== [source,python] ---- [...] urlpatterns = [ url(r'^new$', views.NewListView.as_view(), name='new_list'), url(r'^(\d+)/$', views.view_list, name='view_list'), ] ---- ==== Now running the tests gives six errors: [subs="specialcharacters,macros"] [role="small-code"] ---- $ pass:quotes[*python manage.py test lists*] [...] ERROR: test_can_save_a_POST_request (lists.tests.test_views.NewListTest) TypeError: save() missing 1 required positional argument: 'for_list' ERROR: test_for_invalid_input_passes_form_to_template (lists.tests.test_views.NewListTest) django.core.exceptions.ImproperlyConfigured: TemplateResponseMixin requires either a definition of 'template_name' or an implementation of 'get_template_names()' ERROR: test_for_invalid_input_renders_home_template (lists.tests.test_views.NewListTest) django.core.exceptions.ImproperlyConfigured: TemplateResponseMixin requires either a definition of 'template_name' or an implementation of 'get_template_names()' ERROR: test_invalid_list_items_arent_saved (lists.tests.test_views.NewListTest) django.core.exceptions.ImproperlyConfigured: TemplateResponseMixin requires either a definition of 'template_name' or an implementation of 'get_template_names()' ERROR: test_redirects_after_POST (lists.tests.test_views.NewListTest) TypeError: save() missing 1 required positional argument: 'for_list' ERROR: test_validation_errors_are_shown_on_home_page (lists.tests.test_views.NewListTest) django.core.exceptions.ImproperlyConfigured: TemplateResponseMixin requires either a definition of 'template_name' or an implementation of 'get_template_names()' FAILED (errors=6) ---- Let's start with the third--maybe we can just add the template? [role="sourcecode"] .lists/views.py (ch31l005) ==== [source,python] ---- class NewListView(CreateView): form_class = ItemForm template_name = 'home.html' ---- ==== That gets us down to just two failures: we can see they're both happening in the generic view's `form_valid` function, and that's one of the ones that you can override to provide custom behaviour in a CBGV. As its name implies, it's run when the view has detected a valid form. We can just copy some of the code from our old view function, that used to live after `if form.is_valid():`: [role="sourcecode"] .lists/views.py (ch31l006) ==== [source,python] ---- class NewListView(CreateView): template_name = 'home.html' form_class = ItemForm def form_valid(self, form): list_ = List.objects.create() form.save(for_list=list_) return redirect(list_) ---- ==== That gets us a full pass! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] Ran 34 tests in 0.119s OK $ pass:quotes[*python manage.py test functional_tests*] Ran 5 tests in 15.157s OK ---- And we 'could' even save two more lines, trying to obey "DRY", by using one of the main advantages of CBVs: inheritance! [role="sourcecode"] .lists/views.py (ch31l007) ==== [source,python] ---- class NewListView(CreateView, HomePageView): def form_valid(self, form): list_ = List.objects.create() form.save(for_list=list_) return redirect(list_) ---- ==== And all the tests would still pass: ---- OK ---- WARNING: This is not really good object-oriented practice. Inheritance implies an "is-a" relationship, and it's probably not meaningful to say that our new list view "is-a" home page view...so, probably best not to do this. With or without that last step, how does it compare to the old version? I'd say that's not bad. We save some boilerplate code, and the view is still fairly legible. So far, I'd say we've got one point for CBGVs, and one draw.((("", startref="CBGVcreate28"))) A More Complex View to Handle Both Viewing and Adding to a List ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("class-based generic views (CBGVs)", "duplicate views", id="CBGVduplicate28")))This took me 'several' attempts. And I have to say that, although the tests told me when I got it right, they didn't really help me to figure out the steps to get there...mostly it was just trial and error, hacking about in functions like `get_context_data`, `get_form_kwargs`, and so on. One thing it did made me realise was the value of having lots of individual tests, each testing one thing. I went back and rewrote some of Chapters pass:[#chapter_11_server_prep–#chapter_13_organising_test_files] as a result. The Tests Guide Us, for a While ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Here's how things might go. Start by thinking we want a `DetailView`, something that shows you the detail of an object: [role="sourcecode dofirst-ch31l008"] .lists/views.py (ch31l009) ==== [source,python] ---- from django.views.generic import FormView, CreateView, DetailView [...] class ViewAndAddToList(DetailView): model = List ---- ==== And wiring it up in 'urls.py': [role="sourcecode"] .lists/urls.py (ch31l010) ==== [source,python] ---- url(r'^(\d+)/$', views.ViewAndAddToList.as_view(), name='view_list'), ---- ==== That gives: ---- [...] AttributeError: Generic detail view ViewAndAddToList must be called with either an object pk or a slug. FAILED (failures=5, errors=6) ---- Not totally obvious, but a bit of Googling around led me to understand that I needed to use a "named" regex capture group: [role="sourcecode"] .lists/urls.py (ch31l011) ==== [source,diff] ---- @@ -3,6 +3,6 @@ from lists import views urlpatterns = [ url(r'^new$', views.NewListView.as_view(), name='new_list'), - url(r'^(\d+)/$', views.view_list, name='view_list'), + url(r'^(?P\d+)/$', views.ViewAndAddToList.as_view(), name='view_list') ] ---- ==== The next set of errors had one that was fairly helpful: ---- [...] django.template.exceptions.TemplateDoesNotExist: lists/list_detail.html FAILED (failures=5, errors=6) ---- That's easily solved: [role="sourcecode"] .lists/views.py (ch31l012) ==== [source,python] ---- class ViewAndAddToList(DetailView): model = List template_name = 'list.html' ---- ==== That takes us down five and two: ---- [...] ERROR: test_displays_item_form (lists.tests.test_views.ListViewTest) KeyError: 'form' FAILED (failures=5, errors=2) ---- Until We're Left with Trial and Error ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ So I figured, our view doesn't just show us the detail of an object, it also allows us to create new ones. Let's make it both a `DetailView` 'and' a `CreateView`, and maybe add the `form_class`: [role="sourcecode"] .lists/views.py (ch31l013) ==== [source,python] ---- class ViewAndAddToList(DetailView, CreateView): model = List template_name = 'list.html' form_class = ExistingListItemForm ---- ==== But that gives us a lot of errors saying: ---- [...] TypeError: __init__() missing 1 required positional argument: 'for_list' ---- And the `KeyError: 'form'` was still there too! At this point the errors stopped being quite as helpful, and it was no longer obvious what to do next. I had to resort to trial and error. Still, the tests did at least tell me when I was getting things more right or more wrong. My first attempts to use `get_form_kwargs` didn't really work, but I found that I could use `get_form`: [role="sourcecode"] .lists/views.py (ch31l014) ==== [source,python] ---- def get_form(self): self.object = self.get_object() return self.form_class(for_list=self.object, data=self.request.POST) ---- ==== But it would only work if I also assigned to `self.object`, as a side effect, along the way, which was a bit upsetting. Still, that takes us down to just three errors, but we're still apparently not quite there! ---- django.core.exceptions.ImproperlyConfigured: No URL to redirect to. Either provide a url or define a get_absolute_url method on the Model. ---- Back on Track ^^^^^^^^^^^^^ And for this final failure, the tests are being helpful again. It's quite easy to define a `get_absolute_url` on the `Item` class, such that items point to their parent list's page: [role="sourcecode"] .lists/models.py (ch31l015) ==== [source,python] ---- class Item(models.Model): [...] def get_absolute_url(self): return reverse('view_list', args=[self.list.id]) ---- ==== Is That Your Final Answer? ^^^^^^^^^^^^^^^^^^^^^^^^^^ ((("", startref="CBGVduplicate28")))We end up with a view class that looks like this: [role="sourcecode currentcontens"] .lists/views.py ==== [source,python] ---- class ViewAndAddToList(DetailView, CreateView): model = List template_name = 'list.html' form_class = ExistingListItemForm def get_form(self): self.object = self.get_object() return self.form_class(for_list=self.object, data=self.request.POST) ---- ==== Compare Old and New ~~~~~~~~~~~~~~~~~~~ ((("class-based generic views (CBGVs)", "comparing old and new versions")))Let's see the old version for comparison? [role="sourcecode currentcontents"] .lists/views.py ==== [source,python] ---- def view_list(request, list_id): list_ = List.objects.get(id=list_id) form = ExistingListItemForm(for_list=list_) if request.method == 'POST': form = ExistingListItemForm(for_list=list_, data=request.POST) if form.is_valid(): form.save() return redirect(list_) return render(request, 'list.html', {'list': list_, "form": form}) ---- ==== Well, it has reduced the number of lines of code from nine to seven. Still, I find the function-based version a little easier to understand, in that it has a little bit less magic—"explicit is better than implicit", as the Zen of Python would have it. I mean...[keep-together]#`SingleObjectMixin`#? What? And, more offensively, the whole thing falls apart if we don't assign to `self.object` inside `get_form`? Yuck. Still, I guess some of it is in the eye of the beholder. Best Practices for Unit Testing CBGVs? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("class-based generic views (CBGVs)", "best practices for")))As I was working through this, I felt like my "unit" tests were sometimes a little too high-level. This is no surprise, since tests for views that involve the Django Test Client are probably more properly called integrated tests. They told me whether I was getting things right or wrong, but they didn't always offer enough clues on exactly how to fix things. I occasionally wondered whether there might be some mileage in a test that was closer to the implementation--something like this: [role="sourcecode skipme"] .lists/tests/test_views.py ==== [source,python] ---- def test_cbv_gets_correct_object(self): our_list = List.objects.create() view = ViewAndAddToList() view.kwargs = dict(pk=our_list.id) self.assertEqual(view.get_object(), our_list) ---- ==== But the problem is that it requires a lot of knowledge of the internals of Django CBVs to be able to do the right test setup for these kinds of tests. And you still end up getting very confused by the complex inheritance hierarchy. Take-Home: Having Multiple, Isolated View Tests with Single Assertions Helps ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ One thing I definitely did conclude from this appendix was that having many short unit tests for views was much more helpful than having a few tests with a narrative series of assertions. Consider this monolithic test: [role="sourcecode skipme"] .lists/tests/test_views.py ==== [source,python] ---- def test_validation_errors_sent_back_to_home_page_template(self): response = self.client.post('/lists/new', data={'text': ''}) self.assertEqual(List.objects.all().count(), 0) self.assertEqual(Item.objects.all().count(), 0) self.assertTemplateUsed(response, 'home.html') expected_error = escape("You can't have an empty list item") self.assertContains(response, expected_error) ---- ==== That is definitely less useful than having three individual tests, like this: [role="sourcecode skipme"] .lists/tests/test_views.py ==== [source,python] ---- def test_invalid_input_means_nothing_saved_to_db(self): self.post_invalid_input() self.assertEqual(List.objects.all().count(), 0) self.assertEqual(Item.objects.all().count(), 0) def test_invalid_input_renders_list_template(self): response = self.post_invalid_input() self.assertTemplateUsed(response, 'list.html') def test_invalid_input_renders_form_with_errors(self): response = self.post_invalid_input() self.assertIsinstance(response.context['form'], ExistingListItemForm) self.assertContains(response, escape(empty_list_error)) ---- ==== The reason is that, in the first case, an early failure means not all the assertions are checked. So, if the view was accidentally saving to the database on invalid POST, you would get an early fail, and so you wouldn't find out whether it was using the right template or rendering the form. The second formulation makes it much easier to pick out exactly what was or wasn't working. [role="pagebreak-before"] .Lessons Learned from CBGVs ******************************************************************************* Class-based generic views can do anything:: It might not always be clear what's going on, but you can do just about anything with class-based generic views. Single-assertion unit tests help refactoring:: ((("single-assertion unit tests")))((("unit tests", "testing only one thing")))((("testing best practices")))With each unit test providing individual guidance on what works and what doesn't, it's much easier to change the implementation of our views to using this fundamentally different paradigm.((("", startref="DJFclass28"))) ******************************************************************************* ================================================ FILE: appendix_IV_testing_migrations.asciidoc ================================================ [[data-migrations-appendix]] [appendix] Testing Database Migrations --------------------------- ((("database migrations", id="dbmig30")))((("database testing", "migrations", id="DBTmig30")))Django-migrations and its predecessor South have been around for ages, so it's not usually necessary to test database migrations. But it just so happens that we're introducing a dangerous type of migration--that is, one that introduces a new integrity constraint on our data. When I first ran the migration script against staging, I saw an error. On larger projects, where you have sensitive data, you may want the additional confidence that comes from testing your migrations in a safe environment before applying them to production data, so this toy example will hopefully be a useful rehearsal. Another common reason to want to test migrations is for speed--migrations often involve downtime, and sometimes, when they're applied to very large datasets, they can take time. It's good to know in advance how long that might be. An Attempted Deploy to Staging ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Here's what happened to me when I first tried to deploy our new validation constraints in <>: [role="skipme"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*cd deploy_tools*] $ pass:quotes[*fab deploy:host=elspeth@staging.ottg.co.uk*] [...] Running migrations: Applying lists.0005_list_item_unique_together...Traceback (most recent call last): File "/usr/local/lib/python3.7/dist-packages/django/db/backends/utils.py", line 61, in execute return self.cursor.execute(sql, params) File "/usr/local/lib/python3.7/dist-packages/django/db/backends/sqlite3/base.py", line 475, in execute return Database.Cursor.execute(self, query, params) sqlite3.IntegrityError: columns list_id, text are not unique [...] ---- What happened was that some of the existing data in the database violated the integrity constraint, so the database was complaining when I tried to apply it. In order to deal with this sort of problem, we'll need to build a "data migration". Let's first set up a local environment to test against. Running a Test Migration Locally ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ We'll use a copy of the live database to test our migration against. WARNING: Be very, very, very careful when using real data for testing. For example, you may have real customer email addresses in there, and you don't want to accidentally send them a bunch of test emails. Ask me how I know this. Entering Problematic Data ^^^^^^^^^^^^^^^^^^^^^^^^^ Start a list with some duplicate items on your live site, as shown in <>. [[dupe-data]] .A list with duplicate items image::images/twp2_ad01.png["This list has 3 identical items"] Copying Test Data from the Live Site ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Copy the database down from live: [subs="specialcharacters,quotes"] ---- $ *scp elspeth@superlists.ottg.co.uk:\ /home/elspeth/sites/superlists.ottg.co.uk/database/db.sqlite3 .* $ *mv ../database/db.sqlite3 ../database/db.sqlite3.bak* $ *mv db.sqlite3 ../database/db.sqlite3* ---- Confirming the Error ^^^^^^^^^^^^^^^^^^^^ We now have a local database that has not been migrated, and that contains some problematic data. We should see an error if we try to run `migrate`: [subs="specialcharacters,quotes"] ---- $ *python manage.py migrate --migrate* python manage.py migrate Operations to perform: [...] Running migrations: [...] Applying lists.0005_list_item_unique_together...Traceback (most recent call last): [...] return Database.Cursor.execute(self, query, params) sqlite3.IntegrityError: columns list_id, text are not unique ---- Inserting a Data Migration ~~~~~~~~~~~~~~~~~~~~~~~~~~ https://docs.djangoproject.com/en/5.2/topics/migrations/#data-migrations[Data migrations] are a special type of migration that modifies data in the database rather than changing the schema. We need to create one that will run before we apply the integrity constraint, to preventively remove any duplicates. Here's how we can do that: [subs="specialcharacters,macros"] ---- $ pass:quotes[*git rm lists/migrations/0005_list_item_unique_together.py*] $ pass:quotes[*python manage.py makemigrations lists --empty*] Migrations for 'lists': 0005_auto_20140414_2325.py: $ pass:[mv lists/migrations/0005_*.py lists/migrations/0005_remove_duplicates.py] ---- Check out https://docs.djangoproject.com/en/5.2/topics/migrations/#data-migrations[the Django docs on data migrations] for more info, but here's how we add some instructions to change existing data: [role="sourcecode"] .lists/migrations/0005_remove_duplicates.py ==== [source,python] ---- # encoding: utf8 from django.db import models, migrations def find_dupes(apps, schema_editor): List = apps.get_model("lists", "List") for list_ in List.objects.all(): items = list_.item_set.all() texts = set() for ix, item in enumerate(items): if item.text in texts: item.text = '{} ({})'.format(item.text, ix) item.save() texts.add(item.text) class Migration(migrations.Migration): dependencies = [ ('lists', '0004_item_list'), ] operations = [ migrations.RunPython(find_dupes), ] ---- ==== Re-creating the Old Migration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We re-create the old migration using `makemigrations`, which will ensure it is now the sixth migration and has an explicit dependency on `0005`, the data migration: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py makemigrations*] Migrations for 'lists': 0006_auto_20140415_0018.py: - Alter unique_together for item (1 constraints) $ pass:[mv lists/migrations/0006_* lists/migrations/0006_unique_together.py] ---- Testing the New Migrations Together ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ We're now ready to run our test against the live data: [subs="specialcharacters,macros"] ---- $ pass:quotes[*cd deploy_tools*] $ pass:quotes[*fab deploy:host=elspeth@staging.ottg.co.uk*] [...] ---- We'll need to restart the live Gunicorn job too: [role="server-commands skipme"] [subs="specialcharacters,quotes"] ---- elspeth@server:$ *sudo systemctl restart gunicorn-superlists.ottg.co.uk* ---- And we can now run our FTs against staging: [role="skipme small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*STAGING_SERVER=staging.ottg.co.uk python manage.py test functional_tests*] [...] .... --------------------------------------------------------------------- Ran 4 tests in 17.308s OK ---- Everything seems in order! Let's do it against live: [role="skipme"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*fab deploy:host=superlists.ottg.co.uk*] [superlists.ottg.co.uk] Executing task 'deploy' [...] ---- And that's a wrap. `git add lists/migrations`, `git commit`, and so on. Conclusions ~~~~~~~~~~~ This exercise was primarily aimed at building a data migration and testing it against some real data. Inevitably, this is only a drop in the ocean of the possible testing you could do for a migration. You could imagine building automated tests to check that all your data was preserved, comparing the database contents before and after. You could write individual unit tests for the helper functions in a data migration. You could spend more time measuring the time taken for migrations, and experiment with ways to speed it up by, for example, breaking up migrations into more or fewer component steps. Remember that this should be a relatively rare case. In my experience, I haven't felt the need to test 99% of the migrations I've worked on. But, should you ever feel the need on your project, I hope you've found a few pointers here to get started with.((("", startref="dbmig30")))((("", startref="DBTmig30"))) [role="pagebreak-before less_space"] .On Testing Database Migrations ****************************************************************************** Be wary of migrations which introduce constraints:: 99% of migrations happen without a hitch, but be wary of any situations, like this one, where you are introducing a new constraint on columns that already exist. Test migrations for speed:: Once you have a larger project, you should think about testing how long your migrations are going to take. Database migrations typically involve downtime, as, depending on your database, the schema update operation may lock the table it's working on until it completes. It's a good idea to use your staging site to find out how long a migration will take. Be extremely careful if using a dump of production data:: In order to do so, you'll want fill your staging site's database with an amount of data that's commensurate to the size of your production data. Explaining how to do that is outside of the scope of this book, but I will say this: if you're tempted to just take a dump of your production database and load it into staging, be 'very' careful. Production data contains real customer details, and I've personally been responsible for accidentally sending out a few hundred incorrect invoices after an automated process on my staging server started processing the copied production data I'd just loaded into it. Not a fun afternoon. ****************************************************************************** ================================================ FILE: appendix_IX_cheat_sheet.asciidoc ================================================ [[cheat-sheet]] [appendix] == Cheat Sheet By popular demand, this "cheat sheet" is loosely based on the recap/summary boxes from the end of each chapter. The idea is to provide a few reminders, and links to the chapters where you can find out more to jog your memory. I hope you find it useful! === Initial Project Setup * Start with a _user story_ and map it to a first _functional test_.((("cheat sheet", "project setup")))((("Django framework", "set up", "project creation"))) * Pick a test framework—`unittest` is fine, and options like `py.test`, `nose`, or `Green` can also offer some advantages. * Run the functional test and see your first 'expected failure'. * Pick a web framework such as Django, and find out how to run _unit tests_ against it. * Create your first _unit test_ to address the current FT failure, and see it fail. * Do your _first commit_ to a VCS like _Git_. Relevant chapters: <>, <>, <>. [role="pagebreak-before less_space"] === The Basic TDD Workflow: Red/Green/Refactor [role="two-col"] * Red, Green, Refactor((("cheat sheet", "TDD workflow")))((("Test-Driven Development (TDD)", "overall process of"))) * Double-loop TDD (<>) * Triangulation * The scratchpad * "3 Strikes and Refactor" * "Working State to Working State" * "YAGNI" [[Double-Loop-TDD-diagram2]] .Double-loop TDD image::images/tdd3_0405.png["An inner red/green/refactor loop surrounded by an outer red/green of FTs"] Relevant chapters: <>, <>, <>. === Moving Beyond Dev-Only Testing * Start system testing early. Ensure your components work together: web server, static content, database.((("cheat sheet", "moving beyond dev-only testing"))) * Build a production environment early, and automate deployment to it. - PaaS versus VPS - Docker - Ansible versus Terraform * Think through deployment pain points: the database, static files, dependencies, how to customise settings, and so on. * Build a CI server as soon as possible, so that you don't have to rely on self-discipline to see the tests run. Relevant chapters: <>, <>. [role="pagebreak-before less_space"] === General Testing Best Practices * Each test should test one thing.((("cheat sheet", "testing best practices")))((("testing best practices"))) * Test behaviour rather than implementation. * "Don't test constants". * Try to think beyond the charmed path through the code, and think through edge cases and error cases. * Balance the "test desiderata". Relevant chapters: <>, <>, <>, <>. === Selenium/Functional Testing Best Practices * Use explicit rather than implicit waits, and the interaction/wait pattern. * Avoid duplication of test code--helper methods in a base class and the page pattern are possible solutions. * Avoid double-testing functionality. If you have a test that covers a time-consuming process (e.g., login), consider ways of skipping it in other tests (but be aware of unexpected interactions between seemingly unrelated bits of functionality). * Look into BDD tools as another way of structuring your FTs. Relevant chapters: <>, <>, <>. === Outside-In Default to working outside-in. Use double-loop TDD to drive your development, start at the UI/outside layers, and work your way down to the infrastructure layers. This helps ensure that you write only the code you need, and flushes out integration issues early. Relevant chapter: <>. === The Test Pyramid Be aware that integration tests will get slower and slower over time. Find ways to shift the bulk of your testing to unit tests as your project grows in size and complexity. Relevant chapter: <>. ================================================ FILE: appendix_X_what_to_do_next.asciidoc ================================================ [[appendix4]] [appendix] == What to Do Next ((("Test-Driven Development (TDD)", "future investigations", id="TDDfuture35"))) Here I offer a few suggestions for things to investigate next, to develop your testing skills, and to apply them to some of the cool new technologies in web development (at the time of writing!). I might write an article about some of these in the future. But why not try to beat me to it, and write your own blog post chronicling your attempt at any one of these? ((("getting help"))) I'm very happy to answer questions and provide tips and guidance on all these topics, so if you find yourself attempting one and getting stuck, please don't hesitate to get in touch at obeythetestinggoat@gmail.com! === Switch to Postgres SQLite is a wonderful little database, but it won't deal well once you have more than one web worker process fielding your site's requests. Postgres is everyone's favourite database these days, so find out how to install and configure it. You'll need to figure out a place to store the usernames and passwords for your local, staging, and production Postgres servers. Take a look at <> for inspiration. Experiment with keeping your unit tests running with SQLite, and compare how much faster they are than running against Postgres. Set it up so that your local machine uses SQLite for testing, but your CI server uses Postgres. Does any of your functionality actually depend on Postgres-specific features? What should you do then? === Run Your Tests Against Different Browsers Selenium supports all sorts of different browsers, including Chrome, Safari, and Internet Exploder. Try them all out and see if your FT suite behaves any differently. In my experience, switching browsers tends to expose all sorts of race conditions in Selenium tests, and you will probably need to use the interaction/wait pattern a lot more. === The Django Admin Site Imagine a story where a user emails you wanting to "claim" an anonymous list. Let's say we implement a manual solution to this, involving the site administrator manually changing the record using the Django admin site. Find out how to switch on the admin site, and have a play with it. Write an FT that shows a normal, non–logged-in user creating a list, then have an admin user log in, go to the admin site, and assign the list to the user. The user can then see it in their "My Lists" page. === Write Some Security Tests Expand on the login, my lists, and sharing tests--what do you need to write to assure yourself that users can only do what they're authorized to? === Test for Graceful Degradation What would happen if our email server goes down? Can we at least show an apologetic error message to our users? === Caching and Performance Testing Find out how to install and configure `memcached`. Find out how to use Apache's `ab` to run a performance test. How does it perform with and without caching? Can you write an automated test that will fail if caching is not enabled? What about the dreaded problem of cache invalidation? Can tests help you to make sure your cache invalidation logic is solid? === JavaScript Frameworks Check out React, Vue.js, or perhaps my old favourite, Elm. === Async and Websockets Supposing two users are working on the same list at the same time. Wouldn't it be nice to see real-time updates, so if the other person adds an item to the list, you see it immediately? A persistent connection between client and server using websockets is the way to get this to work. Check out Django's async features and see if you can use them to implement dynamic notifications. To test it, you'll need two browser instances (like we used for the list sharing tests), and check that notifications of the actions from one appear in the other, without needing to refresh the page... === Switch to Using pytest `pytest` lets you write unit tests with less boilerplate. Try converting some of your unit tests to using 'py.test'. You may need to use a plugin to get it to play nicely with Django. === Check Out coverage.py Ned Batchelder's `coverage.py` will tell you what your 'test coverage' is--what percentage of your code is covered by tests. Now, in theory, because we've been using rigorous TDD, we should always have 100% coverage. But it's nice to know for sure, and it's also a very useful tool for working on projects that didn't have tests from the beginning. === Client-Side Encryption Here's a fun one: what if our users are paranoid about the NSA, and decide they no longer want to trust their lists to The Cloud? Can you build a JavaScript encryption system, where the user can enter a password to encypher their list item text before it gets sent to the server? One way of testing it might be to have an "administrator" user that goes to the Django admin view to inspect users' lists, and checks that they are stored encrypted in the database. === Your Suggestion Here What do you think I should put here? Suggestions, please! ((("", startref="TDDfuture35"))) ================================================ FILE: appendix_bdd.asciidoc ================================================ [[appendix_bdd]] [appendix] == Behaviour-Driven Development (BDD) Tools .Warning, Content From Second Edition ******************************************************************************* This appendix is from the second edition of the book, so the listings have not been updated for the latest versions of Django and Python. As always, feedback is welcome, but especially-especially since this stuff is all so new. Let me know how you get on :) ******************************************************************************* Now I haven't used the BDD tools in this appendix for more than a few weeks in a production project, so I can't claim any deep expertise. But, I did like what I have seen of it, and I thought that you deserved at least a whirlwind tour. In this appendix, we'll take some of the tests we wrote in a "normal" FT, and convert them to using BDD tools. === What Is BDD and What are BDD Tools? ((("behavior-driven development (BDD)", "defined"))) ((("behavior-driven development (BDD)", id="bdd31"))) BDD itself is a practice rather than a toolset--it's the approach of testing your application by testing the _behaviour_ that we expect it to have, from the point of view of a user (the https://en.wikipedia.org/wiki/Behavior-driven_development[Wikipedia entry] has quite a good overview). Essentially, whenever you've seen me say "it's better to test behaviour rather than implementation", I've been advocating for BDD. ==== Gherkin and Cucumber ((("behavior-driven development (BDD)", "tools for"))) ((("Gherkin", id="gherkin31"))) ((("Cucumber"))) But the term has become closely associated with a particular set of tools for doing BDD, and particularly the https://github.com/cucumber/cucumber/wiki/Gherkin[Gherkin syntax], which is a human-readable DSL for writing functional (or acceptance) tests. Gherkin originally came out of the Ruby world, where it's associated with a test runner called https://cucumber.io/[Cucumber]. We'll be talking about these tools in this appendix. TIP: BDD as a practice is not the same as the toolset and the Gherkin syntax ((("Lettuce"))) ((("Behave"))) In the Python world, we have a couple of equivalent test running tools, http://lettuce.it/[Lettuce] and http://pythonhosted.org/behave/[Behave]. Of these, only Behave was compatible with Python 3 at the time of writing, so that's what we'll use. We'll also use a plugin called https://pythonhosted.org/behave-django/[behave-django]. [role="pagebreak-before"] .Getting the Code for These Examples ********************************************************************** ((("code examples, obtaining and using"))) I'm going to use the example from <>. We have a basic to-do lists site, and we want to add a new feature: logged-in users should be able to view the lists they've authored in one place. Up until this point, all lists are effectively anonymous. If you've been following along with the book, I'm going to assume you can skip back to the code for that point. If you want to pull it from my repo, the place to go is the https://github.com/hjwp/book-example/tree/chapter_17[chapter_17 branch]. ********************************************************************** === Basic Housekeeping ((("behavior-driven development (BDD)", "directory creation")))We make a directory for our BDD "features," add a _steps_ directory (we'll find out what these are shortly!), and placeholder for our first feature: [subs="specialcharacters,quotes"] ---- $ *mkdir -p features/steps* $ *touch features/my_lists.feature* $ *touch features/steps/my_lists.py* $ *tree features* features ├── my_lists.feature └── steps └── my_lists.py ---- We install `behave-django`, and add it to _settings.py_: [role="dofirst-ch35l000"] [subs="specialcharacters,quotes"] ---- $ *pip install behave-django* ---- [role="sourcecode"] .superlists/settings.py ==== [source,diff] ---- --- a/superlists/settings.py +++ b/superlists/settings.py @@ -40,6 +40,7 @@ INSTALLED_APPS = [ 'lists', 'accounts', 'functional_tests', + 'behave_django', ] ---- ==== And then run `python manage.py behave` as a sanity check: [subs=""] ---- $ python manage.py behave Creating test database for alias 'default'... 0 features passed, 0 failed, 0 skipped 0 scenarios passed, 0 failed, 0 skipped 0 steps passed, 0 failed, 0 skipped, 0 undefined Took 0m0.000s Destroying test database for alias 'default'... ---- === Writing an FT as a "Feature" Using Gherkin Syntax ((("behavior-driven development (BDD)", "functional test using Gherkin syntax"))) ((("functional tests (FTs)", "using Gherkin syntax", secondary-sortas="Gherkin syntax"))) Up until now, we've been writing our FTs using human-readable comments that describe the new feature in terms of a user story, interspersed with the Selenium code required to execute each step in the story. BDD enforces a distinction between those two--we write our human-readable story using a human-readable (if occasionally somewhat awkward) syntax called "Gherkin", and that is called the "Feature". Later, we'll map each line of Gherkin to a function that contains the Selenium code necessary to implement that "step." Here's what a Feature for our new "My lists" page could look like: [role="sourcecode"] .features/my_lists.feature ==== [source,gherkin] ---- Feature: My Lists As a logged-in user I want to be able to see all my lists in one page So that I can find them all after I've written them Scenario: Create two lists and see them on the My Lists page Given I am a logged-in user When I create a list with first item "Reticulate Splines" And I add an item "Immanentize Eschaton" And I create a list with first item "Buy milk" Then I will see a link to "My lists" When I click the link to "My lists" Then I will see a link to "Reticulate Splines" And I will see a link to "Buy milk" When I click the link to "Reticulate Splines" Then I will be on the "Reticulate Splines" list page ---- ==== [role="pagebreak-before"] ==== As-a /I want to/So that At the top you'll notice the As-a/I want to/So that clause. This is optional, and it has no executable counterpart--it's just a slightly formalised way of capturing the "who and why?" aspects of a user story, gently encouraging the team to think about the justifications for each feature. ==== Given/When/Then Given/When/Then is the real core of a BDD test. This trilobite formulation matches the setup/exercise/assert pattern we've seen in our unit tests, and it represents the setup and assumptions phase, an exercise/action phase, and a subsequent assertion/observation phase. There's more info on the https://github.com/cucumber/cucumber/wiki/Given-When-Then[Cucumber wiki]. ==== Not Always a Perfect Fit! As you can see, it's not always easy to shoe-horn a user story into exactly three steps! We can use the `And` clause to expand on a step, and I've added multiple `When` steps and subsequent `Then`'s to illustrate further aspects of our "My lists" page.((("", startref="gherkin31"))) === Coding the Step Functions ((("behavior-driven development (BDD)", "step functions"))) We now build the counterpart to our Gherkin-syntax feature, which are the "step" functions that will actually implement them in code. ==== Generating Placeholder Steps When we run `behave`, it helpfully tells us about all the steps we need to implement: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *python manage.py behave* Feature: My Lists # features/my_lists.feature:1 As a logged-in user I want to be able to see all my lists in one page So that I can find them all after I've written them Scenario: Create two lists and see them on the My Lists page # features/my_lists.feature:6 Given I am a logged-in user # None Given I am a logged-in user # None When I create a list with first item "Reticulate Splines" # None And I add an item "Immanentize Eschaton" # None And I create a list with first item "Buy milk" # None Then I will see a link to "My lists" # None When I click the link to "My lists" # None Then I will see a link to "Reticulate Splines" # None And I will see a link to "Buy milk" # None When I click the link to "Reticulate Splines" # None Then I will be on the "Reticulate Splines" list page # None Failing scenarios: features/my_lists.feature:6 Create two lists and see them on the My Lists page 0 features passed, 1 failed, 0 skipped 0 scenarios passed, 1 failed, 0 skipped 0 steps passed, 0 failed, 0 skipped, 10 undefined Took 0m0.000s You can implement step definitions for undefined steps with these snippets: @given(u'I am a logged-in user') def step_impl(context): raise NotImplementedError(u'STEP: Given I am a logged-in user') @when(u'I create a list with first item "Reticulate Splines"') def step_impl(context): [...] ---- And you'll notice all this output is nicely coloured, as shown in <>. [[behave-output]] .Behave with coloured console ouptut image::images/twp2_ae01.png["Colourful console output"] It's encouraging us to copy and paste these snippets, and use them as starting points to build our steps. === First Step Definition Here's a first stab at making a step for our "Given I am a logged-in user" step. I started by stealing the code for `self.create_pre_authenticated_session` from 'functional_tests/test_my_lists.py', and adapting it slightly (removing the server-side version, for example, although it would be easy to re-add later). [role="sourcecode small-code"] .features/steps/my_lists.py ==== [source,python] ---- from behave import given, when, then from functional_tests.management.commands.create_session import \ create_pre_authenticated_session from django.conf import settings @given('I am a logged-in user') def given_i_am_logged_in(context): session_key = create_pre_authenticated_session(email='edith@example.com') ## to set a cookie we need to first visit the domain. ## 404 pages load the quickest! context.browser.get(context.get_url("/404_no_such_url/")) context.browser.add_cookie(dict( name=settings.SESSION_COOKIE_NAME, value=session_key, path='/', )) ---- ==== //ch35l004 The 'context' variable needs a little explaining—it's a sort of global variable, in the sense that it's passed to each step that's executed, and it can be used to store information that we need to share between steps. Here we've assumed we'll be storing a browser object on it, and the `server_url`. We end up using it a lot like we used `self` when we were writing `unittest` FTs. === setUp and tearDown Equivalents in environment.py Steps can make changes to state in the `context`, but the place to do preliminary set-up, the equivalent of `setUp`, is in a file called _environment.py_: [role="sourcecode"] .features/environment.py ==== [source,python] ---- from selenium import webdriver def before_all(context): context.browser = webdriver.Firefox() def after_all(context): context.browser.quit() def before_feature(context, feature): pass ---- ==== //ch35l005 === Another Run As a sanity check, we can do another run, to see if the new step works and that we really can start a browser: [subs="specialcharacters,quotes"] ---- $ *python manage.py behave* [...] 1 step passed, 0 failed, 0 skipped, 9 undefined ---- The usual reams of output, but we can see that it seems to have made it through the first step; let's define the rest of them. === Capturing Parameters in Steps ((("behavior-driven development (BDD)", "capturing parameters in steps")))We'll see how Behave allows you to capture parameters from step descriptions. Our next step says: [role="sourcecode currentcontents"] .features/my_lists.feature ==== [source,gherkin] ---- When I create a list with first item "Reticulate Splines" ---- ==== And the autogenerated step definition looked like this: [role="sourcecode small-code skipme"] .features/steps/my_lists.py ==== [source,python] ---- @given('I create a list with first item "Reticulate Splines"') def step_impl(context): raise NotImplementedError( u'STEP: When I create a list with first item "Reticulate Splines"' ) ---- ==== We want to be able to create lists with arbitrary first items, so it would be nice to somehow capture whatever is between those quotes, and pass them in as an argument to a more generic function. That's a common requirement in BDD, and Behave has a nice syntax for it, reminiscent of the new-style Python string formatting syntax: [role="sourcecode"] .features/steps/my_lists.py (ch35l006) ==== [source,python] ---- [...] @when('I create a list with first item "{first_item_text}"') def create_a_list(context, first_item_text): context.browser.get(context.get_url('/')) context.browser.find_element(By.ID, 'id_text').send_keys(first_item_text) context.browser.find_element(By.ID, 'id_text').send_keys(Keys.ENTER) wait_for_list_item(context, first_item_text) ---- ==== Neat, huh? NOTE: Capturing parameters for steps is one of the most powerful features of the BDD syntax. As usual with Selenium tests, we will need an explicit wait. Let's re-use our `@wait` decorator from 'base.py': [role="sourcecode"] .features/steps/my_lists.py (ch35l007) ==== [source,python] ---- from functional_tests.base import wait [...] @wait def wait_for_list_item(context, item_text): context.test.assertIn( item_text, context.browser.find_element_by_css_selector('#id_list_table').text ) ---- ==== Similarly, we can add to an existing list, and see or click on links: [role="sourcecode"] .features/steps/my_lists.py (ch35l008) ==== [source,python] ---- from selenium.webdriver.common.keys import Keys [...] @when('I add an item "{item_text}"') def add_an_item(context, item_text): context.browser.find_element(By.ID, 'id_text').send_keys(item_text) context.browser.find_element(By.ID, 'id_text').send_keys(Keys.ENTER) wait_for_list_item(context, item_text) @then('I will see a link to "{link_text}"') @wait def see_a_link(context, link_text): context.browser.find_element_by_link_text(link_text) @when('I click the link to "{link_text}"') def click_link(context, link_text): context.browser.find_element_by_link_text(link_text).click() ---- ==== Notice we can even use our `@wait` decorator on steps themselves. And finally the slightly more complex step that says I am on the page for a particular list: [role="sourcecode"] .features/steps/my_lists.py (ch35l009) ==== [source,python] ---- @then('I will be on the "{first_item_text}" list page') @wait def on_list_page(context, first_item_text): first_row = context.browser.find_element_by_css_selector( '#id_list_table tr:first-child' ) expected_row_text = '1: ' + first_item_text context.test.assertEqual(first_row.text, expected_row_text) ---- ==== [role="pagebreak-before"] Now we can run it and see our first expected failure: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py behave*] Feature: My Lists # features/my_lists.feature:1 As a logged-in user I want to be able to see all my lists in one page So that I can find them all after I've written them Scenario: Create two lists and see them on the My Lists page # features/my_lists.feature:6 Given I am a logged-in user # features/steps/my_lists.py:19 When I create a list with first item "Reticulate Splines" # features/steps/my_lists.py:31 And I add an item "Immanentize Eschaton" # features/steps/my_lists.py:39 And I create a list with first item "Buy milk" # features/steps/my_lists.py:31 Then I will see a link to "My lists" # functional_tests/base.py:12 Traceback (most recent call last): [...] File "features/steps/my_lists.py", line 49, in see_a_link context.browser.find_element_by_link_text(link_text) [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: My lists [...] Failing scenarios: features/my_lists.feature:6 Create two lists and see them on the My Lists page 0 features passed, 1 failed, 0 skipped 0 scenarios passed, 1 failed, 0 skipped 4 steps passed, 1 failed, 5 skipped, 0 undefined ---- You can see how the output really gives you a sense of how far through the "story" of the test we got: we manage to create our two lists successfully, but the "My lists" link does not appear. === Comparing the Inline-Style FT ((("behavior-driven development (BDD)", "comparing inline-style FT"))) I'm not going to run through the implementation of the feature, but you can see how the test will drive development just as well as the inline-style FT would have. Let's have a look at it, for comparison: [role="sourcecode skipme"] .functional_tests/test_my_lists.py ==== [source,python] ---- def test_logged_in_users_lists_are_saved_as_my_lists(self): # Edith is a logged-in user self.create_pre_authenticated_session('edith@example.com') # She goes to the homepage and starts a list self.browser.get(self.live_server_url) self.add_list_item('Reticulate splines') self.add_list_item('Immanentize eschaton') first_list_url = self.browser.current_url # She notices a "My lists" link, for the first time. self.browser.find_element_by_link_text('My lists').click() # She sees that her list is in there, named according to its # first list item self.wait_for( lambda: self.browser.find_element_by_link_text('Reticulate splines') ) self.browser.find_element_by_link_text('Reticulate splines').click() self.wait_for( lambda: self.assertEqual(self.browser.current_url, first_list_url) ) # She decides to start another list, just to see self.browser.get(self.live_server_url) self.add_list_item('Click cows') second_list_url = self.browser.current_url # Under "my lists", her new list appears self.browser.find_element_by_link_text('My lists').click() self.wait_for( lambda: self.browser.find_element_by_link_text('Click cows') ) self.browser.find_element_by_link_text('Click cows').click() self.wait_for( lambda: self.assertEqual(self.browser.current_url, second_list_url) ) # She logs out. The "My lists" option disappears self.browser.find_element_by_link_text('Log out').click() self.wait_for(lambda: self.assertEqual( self.browser.find_elements_by_link_text('My lists'), [] )) ---- ==== It's not entirely an apples-to-apples comparison, but we can look at the number of lines of code in <>. [[table-code-compare]] .Lines of code comparison [options="header"] |============================================================================== |BDD |Standard FT |Feature file: 20 (3 optional) |test function body: 45 |Steps file: 56 lines |helper functions: 23 |============================================================================== The comparison isn't perfect, but you might say that the feature file and the body of a "standard FT" test function are equivalent in that they present the main "story" of a test, while the steps and helper functions represent the "hidden" implementation details. If you add them up, the total numbers are pretty similar, but notice that they're spread out differently: the BDD tests have made the story more concise, and pushed more work out into the hidden implementation details. === BDD Encourages Structured Test Code ((("behavior-driven development (BDD)", "structured test code encouraged by"))) ((("functional tests (FTs)", "structuring test code"))) This is the real appeal, for me: the BDD tool has _forced_ us to structure our test code. In the inline-style FT, we're free to use as many lines as we want to implement a step, as described by its comment line. It's very hard to resist the urge to just copy-and-paste code from elsewhere, or just from earlier on in the test. You can see that, by this point in the book, I've built just a couple of helper functions (like `get_item_input_box`). In contrast, the BDD syntax has immediately forced me to have a separate function for each step, so I've already built some very reusable code to: * Start a new list * Add an item to an existing list * Click on a link with particular text * Assert that I'm looking at a particular list's page BDD really encourages you to write test code that seems to match well with the business domain, and to use a layer of abstraction between the story of your FT and its implementation in code. The ultimate expression of this is that, theoretically, if you wanted to change programming languages, you could keep all your features in Gherkin syntax exactly as they are, and throw away the Python steps and replace them with steps implemented in another language. === The Page Pattern as an Alternative ((("behavior-driven development (BDD)", "page pattern"))) ((("page pattern"))) In <> of the book, I present an example of the "Page pattern", which is an object-oriented approach to structuring your Selenium tests. Here's a reminder of what it looks like: [role="sourcecode skipme"] .functional_tests/test_sharing.py ==== [source,python] ---- from .my_lists_page import MyListsPage [...] class SharingTest(FunctionalTest): def test_can_share_a_list_with_another_user(self): # [...] self.browser.get(self.live_server_url) list_page = ListPage(self).add_list_item('Get help') # She notices a "Share this list" option share_box = list_page.get_share_box() self.assertEqual( share_box.get_attribute('placeholder'), 'your-friend@example.com' ) # She shares her list. # The page updates to say that it's shared with Oniciferous: list_page.share_list_with('oniciferous@example.com') ---- ==== //TODO: all these skipmes could be tested by doing a checkout of the page_pattern branch And the +Page+ class looks like this: [role="sourcecode small-code skipme"] .functional_tests/lists_pages.py ==== [source,python] ---- class ListPage(object): def __init__(self, test): self.test = test def get_table_rows(self): return self.test.browser.find_elements_by_css_selector('#id_list_table tr') @wait def wait_for_row_in_list_table(self, item_text, item_number): row_text = '{}: {}'.format(item_number, item_text) rows = self.get_table_rows() self.test.assertIn(row_text, [row.text for row in rows]) def get_item_input_box(self): return self.test.browser.find_element(By.ID, 'id_text') ---- ==== So it's definitely possible to implement a similar layer of abstraction, and a sort of DSL, in inline-style FTs, whether it's by using the Page pattern or whatever structure you prefer--but now it's a matter of self-discipline, rather than having a framework that pushes you towards it. NOTE: In fact, you can actually use the Page pattern with BDD as well, as a resource for your steps to use when navigating the pages of your site. === BDD Might Be Less Expressive than Inline Comments ((("behavior-driven development (BDD)", "vs. inline comments", secondary-sortas="inline comments"))) ((("inline comments"))) On the other hand, I can also see potential for the Gherkin syntax to feel somewhat restrictive. Compare how expressive and readable the inline-style comments are, with the slightly awkward BDD feature: [role="sourcecode skipme"] .functional_tests/test_my_lists.py ==== [source,python] ---- # Edith is a logged-in user # She goes to the homepage and starts a list # She notices a "My lists" link, for the first time. # She sees that her list is in there, named according to its # first list item # She decides to start another list, just to see # Under "my lists", her new list appears # She logs out. The "My lists" option disappears [...] ---- ==== That's much more readable and natural than our slightly forced Given/Then/When incantations, and, in a way, might encourage more user-centric thinking. (There is a syntax in Gherkin for including "comments" in a feature file, which would mitigate this somewhat, but I gather that it's not widely used.) === Will Nonprogrammers Write Tests? ((("behavior-driven development (BDD)", "benefits and drawbacks of")))I haven't touched on one of the original promises of BDD, which is that nonprogrammers--business or client representatives perhaps--might actually write the Gherkin syntax. I'm quite skeptical about whether this would actually work in the real world, but I don't think that detracts from the other potential benefits of BDD. === Some Tentative Conclusions I've only dipped my toes into the BDD world, so I'm hesitant to draw any firm conclusions. I find the "forced" structuring of FTs into steps very appealing though--in that it looks like it has the potential to encourage a lot of reuse in your FT code, and that it neatly separates concerns between describing the story and implementing it, and that it forces us to think about things in terms of the business domain, rather than in terms of "what we need to do with Selenium." But there's no free lunch. The Gherkin syntax is restrictive, compared to the total freedom offered by inline FT comments. I also would like to see how BDD scales once you have not just one or two features, and four or five steps, but several dozen features and hundreds of lines of steps code. Overall, I would say it's definitely worth investigating, and I will probably use BDD for my next personal project. My thanks to Daniel Pope, Rachel Willmer, and Jared Contrascere for their feedback on this chapter. .BDD Conclusions ******************************************************************************* Encourages structured, reusable test code:: By separating concerns, breaking your FTs out into the human-readable, Gherkin syntax "feature" file and a separate implementation of steps functions, BDD has the potential to encourage more reusable and manageable test code. It may come at the expense of readability:: The Gherkin syntax, for all its attempt to be human-readable, is ultimately a constraint on human language, and so it may not capture nuance and intention as well as inline comments do. Try it! I will:: As I keep saying, I haven't used BDD on a real project, so you should take my words with a heavy pinch of salt, but I'd like to give it a hearty endorsement. I'm going to try it out on the next project I can, and I'd encourage you to do so as well.((("", startref="bdd31"))) ******************************************************************************* ================================================ FILE: appendix_fts_for_external_dependencies.asciidoc ================================================ [[appendix_fts_for_external_dependencies]] [appendix] == The Subtleties of Functionally Testing External Dependencies You might remember from <> a point at which we wanted to test sending email from the server. Here were the options we considered: 1. We could build a "real" end-to-end test, and have our tests log in to an email server, and retrieve the email from there. That's what I did in the first and second edition. 2. You can use a service like Mailinator or Mailsac, which give you an email account to send to, and some APIs for checking what mail has been delivered. 3. We can use an alternative, fake email backend, whereby Django will save the emails to a file on disk for example, and we can inspect them there. 4. Or we could give up on testing email on the server. If we have a minimal smoke test that the server _can_ send emails, then we don't need to test that they are _actually_ delivered. In the end we decided not to bother, but let's spend a bit of time in this appendix trying out options 1 and 3, just to see some of the fiddliness and trade-offs involved. === How to Test Email End-To-End with POP3 Here's an example helper function that can retrieve a real email from a real POP3 email server, using the horrifically tortuous Python standard library POP3 client. To make it work, we'll need an email address to receive the email. I signed up for a Yahoo account for testing, but you can use any email service you like, as long as it offers POP3 access. You will need to set the `RECEIVER_EMAIL_PASSWORD` environment variable in the console that's running the FT. [subs="specialcharacters,quotes"] ---- $ *export RECEIVER_EMAIL_PASSWORD=otheremailpasswordhere* ---- [role="sourcecode skipme"] .src/functional_tests/test_login.py (ch23l001) ==== [source,python] ---- import os import poplib import re impot time [...] def retrieve_pop3_email(receiver_email, subject, pop3_server, pop3_password): email_id = None start = time.time() inbox = poplib.POP3_SSL(pop3_server) try: inbox.user(receiver_email) inbox.pass_(pop3_password) while time.time() - start < POP3_TIMEOUT: # get 10 newest messages count, _ = inbox.stat() for i in reversed(range(max(1, count - 10), count + 1)): print("getting msg", i) _, lines, __ = inbox.retr(i) lines = [l.decode("utf8") for l in lines] print(lines) if f"Subject: {subject}" in lines: email_id = i body = "\n".join(lines) return body time.sleep(5) finally: if email_id: inbox.dele(email_id) inbox.quit() ---- ==== If you're curious, I'd encourage you to try this out in your FTs. It definitely _can_ work. But, having tried it in the first couple of editions of the book. I have to say it's fiddly to get right, and often flaky, which is a highly undesirable property for a testing tool. So let's leave that there for now. === Using a Fake Email Backend For Django Next let's investigate using a filesystem-based email backend. As we'll see, although it definitely has the advantage that everything stays local on our own machine (there are no calls over the internet), there are quite a few things to watch out for. Let's say that, if we detect an environment variable `EMAIL_FILE_PATH`, we switch to Django's file-based backend: .src/superlists/settings.py (ch23l002) ==== [source,python] ---- EMAIL_HOST = "smtp.gmail.com" EMAIL_HOST_USER = "obeythetestinggoat@gmail.com" EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_PASSWORD") EMAIL_PORT = 587 EMAIL_USE_TLS = True # Use fake file-based backend if EMAIL_FILE_PATH is set if "EMAIL_FILE_PATH" in os.environ: EMAIL_BACKEND = "django.core.mail.backends.filebased.EmailBackend" EMAIL_FILE_PATH = os.environ["EMAIL_FILE_PATH"] ---- ==== Here's how we can adapt our tests to conditionally use the email file, instead of Django's `mail.outbox`, if the env var is set when running our tests: [role="sourcecode"] .src/functional_tests/test_login.py (ch23l003) ==== [source,python] ---- class LoginTest(FunctionalTest): def retrieve_email_from_file(self, sent_to, subject, emails_dir): # <1> latest_emails_file = sorted(Path(emails_dir).iterdir())[-1] # <2> latest_email = latest_emails_file.read_text().split("-" * 80)[-1] # <3> self.assertIn(subject, latest_email) self.assertIn(sent_to, latest_email) return latest_email def retrieve_email_from_django_outbox(self, sent_to, subject): # <4> email = mail.outbox.pop() self.assertIn(sent_to, email.to) self.assertEqual(email.subject, subject) return email.body def wait_for_email(self, sent_to, subject): # <5> """ Retrieve email body, from a file if the right env var is set, or get it from django.mail.outbox by default """ if email_file_path := os.environ.get("EMAIL_FILE_PATH"): # <6> return self.wait_for( # <7> lambda: self.retrieve_email_from_file(sent_to, subject, email_file_path) ) else: return self.retrieve_email_from_django_outbox(sent_to, subject) def test_login_using_magic_link(self): [...] ---- ==== <1> Here's our helper method for getting email contents from a file. It takes the configured email directory as an argument, as well as the sent-to address and expected subject. <2> Django saves a new file with emails every time you restart the server. The filename has a timestamp in it, so we can get the latest one by sorting the files in our test directory. Check out the https://docs.python.org/3/library/pathlib.html[Pathlib] docs if you haven't used it before, it's a nice, relatively new way of working with files in Python. <3> The emails in the file are separated by a line of 80 hyphens. <4> This is the matching helper for getting the email from `mail.outbox`. <5> Here's where we dispatch to the right helper based on whether the env var is set. <6> Checking whether an environment variable is set, and using its value if so, is one of the (relatively few) places where it's nice to use the walrus operator. <7> I'm using a `wait_for()` here because anything involving reading and writing from files, especially across the filesystem mounts inside and outside of Docker, has a potential race condition. We'll need a couple more minor changes to the FT, to use the helper: [role="sourcecode"] .src/functional_tests/test_login.py (ch23l004) ==== [source,diff] ---- @@ -59,15 +59,12 @@ class LoginTest(FunctionalTest): ) # She checks her email and finds a message - email = mail.outbox.pop() - self.assertIn(TEST_EMAIL, email.to) - self.assertEqual(email.subject, SUBJECT) + email_body = self.wait_for_email(TEST_EMAIL, SUBJECT) # It has a URL link in it - self.assertIn("Use this link to log in", email.body) - url_search = re.search(r"http://.+/.+$", email.body) - if not url_search: - self.fail(f"Could not find url in email body:\n{email.body}") + self.assertIn("Use this link to log in", email_body) + if not (url_search := re.search(r"http://.+/.+$", email_body, re.MULTILINE)): + self.fail(f"Could not find url in email body:\n{email_body}") url = url_search.group(0) self.assertIn(self.live_server_url, url) ---- ==== // TODO backport that walrus Now let's set that file path, and mount it inside our docker container, so that it's available both inside and outside the container: [subs="attributes+,specialcharacters,quotes"] ---- # set a local env var for our path to the emails file $ *export EMAIL_FILE_PATH=/tmp/superlists-emails* # make sure the file exists $ *mkdir -p $EMAIL_FILE_PATH* # re-run our container, with the EMAIL_FILE_PATH as an env var, and mounted. $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source=./src/db.sqlite3,target=/src/db.sqlite3 \ --mount type=bind,source=$EMAIL_FILE_PATH,target=$EMAIL_FILE_PATH \ <1> -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -e EMAIL_PASSWORD \ -e EMAIL_FILE_PATH \ <2> -it superlists* ---- <1> Here's where we mount the emails file so we can see it both inside and outside the container <2> And here's where we pass the path as an env var, once again re-exporting the variable from the current shell. And we can rerun our FT, first without using Docker or the EMAIL_FILE_PATH, just to check we didn't break anything: [subs="specialcharacters,macros"] ---- $ pass:quotes[*./src/manage.py test functional_tests.test_login*] [...] OK ---- And now _with_ Docker and the EMAIL_FILE_PATH: [subs="specialcharacters,quotes"] ---- $ *TEST_SERVER=localhost:8888 EMAIL_FILE_PATH=/tmp/superlists-emails \ python src/manage.py test functional_tests* [...] OK ---- It works! Hooray. === Double-Checking our Test and Our Fix As always, we should be suspicious of any test that we've only ever seen pass! Let's see if we can make this test fail. Before we do--we've been in the detail for a bit, it's worth reminding ourselves of what the actual bug was, and how we're fixing it! The bug was, the server was crashing when it tried to send an email. The reason was, we hadn't set the `EMAIL_PASSWORD` environment variable. We managed to repro the bug in Docker. The actual _fix_ is to set that env var, both in Docker and eventually on the server. Now we want to have a _test_ that our fix works, and we looked in to a few different options, settling on using the `filebased.EmailBackend" `EMAIL_BACKEND` setting using the `EMAIL_FILE_PATH` environment variable. Now, I say we haven't seen the test fail, but actually we have, when we repro'd the bug. If we unset the `EMAIL_PASSWORD` env var, it will fail again. I'm more worried about the new parts of our tests, the bits where we go and read from the file at `EMAIL_FILE_PATH`. How can we make that part fail? Well, how about if we deliberately break our email-sending code? [role="sourcecode"] .src/accounts/views.py (ch23l005) ==== [source,python] ---- def send_login_email(request): email = request.POST["email"] token = Token.objects.create(email=email) url = request.build_absolute_uri( reverse("login") + "?token=" + str(token.uid), ) message_body = f"Use this link to log in:\n\n{url}" # send_mail( <1> # "Your login link for Superlists", # message_body, # "noreply@superlists", # [email], # ) messages.success( request, "Check your email, we've sent you a link you can use to log in.", ) return redirect("/") ---- ==== <1> We just comment out the entire send_email block. We rebuild our docker image: [subs="specialcharacters,quotes"] ---- # check our env var is set $ *echo $EMAIL_FILE_PATH* /tmp/superlists-emails $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source=./src/db.sqlite3,target=/src/db.sqlite3 \ --mount type=bind,source=$EMAIL_FILE_PATH,target=$EMAIL_FILE_PATH \ -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -e EMAIL_PASSWORD \ -e EMAIL_FILE_PATH \ -it superlists* ---- // TODO: aside on moujnting /src/? And we rerun our test: [subs="specialcharacters,quotes"] ---- $ *TEST_SERVER=localhost:8888 EMAIL_FILE_PATH=/tmp/superlists-emails \ ./src/manage.py test functional_tests.test_login [...] Ran 1 test in 2.513s OK ---- Eh? How did that pass? === Testing side-effects is fiddly! We've run into an example of the kinds of problems you often encounter when our tests involve side-effects. Let's have a look in our test emails directory: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *ls $EMAIL_FILE_PATH* 20241120-153150-262004991022080.log 20241120-153154-262004990980688.log 20241120-153301-272143941669888.log ---- Every time we restart the server, it opens a new file, but only when it first tries to send an email. Because we've commented out the whole email-sending block, our test instead picks up on an old email, which still has a valid url in it, because the token is still in the database. NOTE: You'll run into a similar issue if you test with "real" emails in POP3. How do you make sure you're not picking up an email from a previous test run? Let's clear out the db: [subs="specialcharacters,quotes"] ---- $ *rm src/db.sqlite3 && ./src/manage.py migrate* Operations to perform: Apply all migrations: accounts, auth, contenttypes, lists, sessions Running migrations: Applying accounts.0001_initial... OK Applying accounts.0002_token... OK Applying contenttypes.0001_initial... OK Applying contenttypes.0002_remove_content_type_name... OK Applying auth.0001_initial... OK ---- And... cmdgg [subs="specialcharacters,quotes"] ---- $ *TEST_SERVER=localhost:8888 ./src/manage.py test functional_tests.test_login* [...] ERROR: test_login_using_magic_link (functional_tests.test_login.LoginTest.test_login_using_magic_link) self.wait_to_be_logged_in(email=TEST_EMAIL) ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: #id_logout; [...] ---- OK sure enough, the `wait_to_be_logged_in()` helper is failing, because now, although we have found an email, its token is invalid. Here's another way to make the tests fail: [subs="specialcharacters,macros"] ---- $ pass:[rm $EMAIL_FILE_PATH/*] ---- Now when we run the FT: [subs="specialcharacters,quotes"] ---- $ *TEST_SERVER=localhost:8888 ./src/manage.py test functional_tests.test_login* ERROR: test_login_using_magic_link (functional_tests.test_login.LoginTest.test_login_using_magic_link) [...] email_body = self.wait_for_email(TEST_EMAIL, SUBJECT) [...] return self.wait_for( ~~~~~~~~~~~~~^ lambda: self.retrieve_email_from_file(sent_to, subject, email_file_path) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [...] latest_emails_file = sorted(Path(emails_dir).iterdir())[-1] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^ IndexError: list index out of range ---- We see there are no email files, because we're not sending one. NOTE: In this configuration of Docker + `filebase.EmailBackend`, we now have to manage side effects in two locations: the database at _src/db.sqlite3_, and the email files in _/tmp_. What Django used to do for us thanks to LiveServerTestCase is now all our responsibility, and as you can see, it's hard to get right. This is a tradeoff to be aware of when writing tests against "real" systems. Still, this isn't quite satisfactory. Let's try a different way to make our tests fail, where we _will_ send an email, but we'll give it the wrong contents: [role="sourcecode"] .src/accounts/views.py (ch23l006) ==== [source,python] ---- def send_login_email(request): email = request.POST["email"] token = Token.objects.create(email=email) url = request.build_absolute_uri( reverse("login") + "?token=" + str(token.uid), ) message_body = f"Use this link to log in:\n\n{url}" send_mail( "Your login link for Superlists", "HAHA NO LOGIN URL FOR U", # <1> "noreply@superlists", [email], ) messages.success( request, "Check your email, we've sent you a link you can use to log in.", ) return redirect("/") ---- ==== <1> We _do_ send an email, but it won't contain a login URL. Let's rebuild again: [subs="specialcharacters,quotes"] ---- # check our env var is set $ *echo $EMAIL_FILE_PATH* /tmp/superlists-emails $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source=./src/db.sqlite3,target=/src/db.sqlite3 \ --mount type=bind,source=$EMAIL_FILE_PATH,target=$EMAIL_FILE_PATH \ -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -e EMAIL_PASSWORD \ -e EMAIL_FILE_PATH \ -it superlists* ---- Now how do our tests look? [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests*] FAIL: test_login_using_magic_link (functional_tests.test_login.LoginTest.test_login_using_magic_link) [...] email_body = self.wait_for_email(TEST_EMAIL, SUBJECT) [...] self.assertIn("Use this link to log in", email_body) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'Use this link to log in' not found in 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nSubject: Your login link for Superlists\nFrom: noreply@superlists\nTo: edith@example.com\nDate: Wed, 13 Nov 2024 18:00:55 -0000\nMessage-ID: [...]\n\nHAHA NO LOGIN URL FOR U\n-------------------------------------------------------------------------------\n' ---- OK good, that's the error we wanted! I think we can be fairly confident that this testing setup can genuinely test that emails are sent properly. Let's revert our temporarily-broken _views.py_, rebuild, and make sure the tests pass once again. [subs="specialcharacters,quotes"] ---- $ *git stash* $ *docker build [...]* # separate terminal $ *TEST_SERVER=localhost:8888 EMAIL_FILE_PATH=/tmp/superlists-emails [...] [...] OK ---- NOTE: It may seem like I've gone through a lot of back-and-forth, but I wanted to give you a flavour of the fiddliness involved in these kinds of tests that involve a lot of side-effects. === Decision Time: Which Test Strategy Will We Keep Let's recap our three options: .Testing Strategy Tradeoffs [cols="1,1,1"] |======= | Strategy | Pros | Cons | End-to-end with POP3 | Maximally realistic, tests the whole system | Slow, fiddly, unreliable | File-based fake email backend | Faster, more reliable, no network calls, tests end-to-end (albeit with fake components) | Still Fiddly, requires managing db & filesystem side-effects | Give up on testing email on the server/Docker | Fast, simple | Less confidence that things work "for real" |======= This is a common problem in testing integration with external systems, how far should we go? How realistic should we make our tests? In the book in the end, I suggested we go for the last option, ie give up. Email itself is a well-understood protocol (reader, it's been around since _before I was born_, and that's a whiles ago now) and Django has supported sending email for more than a decade, so I think we can afford to say, in this case, that the costs of building testing tools for email outweigh the benefits. But not all external dependencies are as well-understood as email. If you're working with a new API, or a new service, you may well decide it's worth putting in the effort to get a "real" end-to-end functional test to work. As always, it's tradeoffs all the way down, folks. ================================================ FILE: appendix_github_links.asciidoc ================================================ [[appendix_github_links]] [appendix] == Source Code Examples ((("code examples, obtaining and using"))) All of the code examples I've used in the book are available in https://github.com/hjwp/book-example/[my repo on GitHub]. So, if you ever want to compare your code against mine, you can take a look at it there. Each chapter has its own branch named after it, like so: https://github.com/hjwp/book-example/tree/chapter_01. Be aware that each branch contains all of the commits for that chapter, so its state represents the code at the 'end' of the chapter. === Full List of Links for Each Chapter |=== | Chapter | GitHub branch name & hyperlink | <> | https://github.com/hjwp/book-example/tree/chapter_01[chapter_01] | <> | https://github.com/hjwp/book-example/tree/chapter_02_unittest[chapter_02_unittest] | <> | https://github.com/hjwp/book-example/tree/chapter_03_unit_test_first_view[chapter_03_unit_test_first_view] | <> | https://github.com/hjwp/book-example/tree/chapter_04_philosophy_and_refactoring[chapter_04_philosophy_and_refactoring] | <> | https://github.com/hjwp/book-example/tree/chapter_05_post_and_database[chapter_05_post_and_database] | <> | https://github.com/hjwp/book-example/tree/chapter_06_explicit_waits_1[chapter_06_explicit_waits_1] | <> | https://github.com/hjwp/book-example/tree/chapter_07_working_incrementally[chapter_07_working_incrementally] | <> | https://github.com/hjwp/book-example/tree/chapter_08_prettification[chapter_08_prettification] | <> | https://github.com/hjwp/book-example/tree/chapter_09_docker[chapter_09_docker] | <> | https://github.com/hjwp/book-example/tree/chapter_10_production_readiness[chapter_10_production_readiness] | <> | https://github.com/hjwp/book-example/tree/chapter_11_server_prep[chapter_11_server_prep] | <> | https://github.com/hjwp/book-example/tree/chapter_13_organising_test_files[chapter_13_organising_test_files] | <> | https://github.com/hjwp/book-example/tree/chapter_14_database_layer_validation[chapter_14_database_layer_validation] | <> | https://github.com/hjwp/book-example/tree/chapter_15_simple_form[chapter_15_simple_form] | <> | https://github.com/hjwp/book-example/tree/chapter_16_advanced_forms[chapter_16_advanced_forms] | <> | https://github.com/hjwp/book-example/tree/chapter_17_javascript[chapter_17_javascript] | <> | https://github.com/hjwp/book-example/tree/chapter_18_second_deploy[chapter_18_second_deploy] | <> | https://github.com/hjwp/book-example/tree/chapter_19_spiking_custom_auth[chapter_19_spiking_custom_auth] | <> | https://github.com/hjwp/book-example/tree/chapter_20_mocking_1[chapter_20_mocking_1] | <> | https://github.com/hjwp/book-example/tree/chapter_21_mocking_2[chapter_21_mocking_2] | <> | https://github.com/hjwp/book-example/tree/chapter_22_fixtures_and_wait_decorator[chapter_22_fixtures_and_wait_decorator] | <> | https://github.com/hjwp/book-example/tree/chapter_23_debugging_prod[chapter_23_debugging_prod] | <> | https://github.com/hjwp/book-example/tree/chapter_24_outside_in[chapter_24_outside_in] | <> | https://github.com/hjwp/book-example/tree/chapter_25_CI[chapter_25_CI] | <> | https://github.com/hjwp/book-example/tree/chapter_26_page_pattern[chapter_26_page_pattern] | Online Appendix: Test Isolation, and Listening to Your Tests | https://github.com/hjwp/book-example/tree/appendix_purist_unit_tests[appendix_purist_unit_tests] | Online Appendix: BDD | https://github.com/hjwp/book-example/tree/appendix_bdd[appendix_bdd] | Online Apendix: Building a REST API | https://github.com/hjwp/book-example/tree/appendix_rest_api[appendix_rest_api] |=== === Using Git to Check Your Progress If you feel like developing your Git-Fu a little further, you can add my repo as a 'remote': [role="skipme"] ----- git remote add harry https://github.com/hjwp/book-example.git git fetch harry ----- And then, to check your difference from the 'end' of <>: [role="skipme"] ---- git diff harry/chapter_04_philosophy_and_refactoring ---- Git can handle multiple remotes, so you can still do this even if you're already pushing your code up to GitHub or Bitbucket. Be aware that the precise order of, say, methods in a class may differ between your version and mine. It may make diffs hard to read. === Downloading a ZIP File for a Chapter If, for whatever reason, you want to "start from scratch" for a chapter, or skip ahead,footnote:[ I don't recommend skipping ahead. I haven't designed the chapters to stand on their own; each relies on the previous ones, so it may be more confusing than anything else...] and/or you're just not comfortable with Git, you can download a version of my code as a ZIP file, from URLs following this pattern: https://github.com/hjwp/book-example/archive/chapter_01.zip https://github.com/hjwp/book-example/archive/chapter_04_philosophy_and_refactoring.zip === Don't Let it Become a Crutch! Try not to sneak a peek at the answers unless you're really, really stuck. Like I said at the beginning of <>, there's a lot of value in debugging errors all by yourself, and in real life, there's no "harrys repo" to check against and find all the answers. Happy coding! ================================================ FILE: appendix_logging.asciidoc ================================================ [[appendix_logging]] [apendix] Logging ~~~~~~~ Using Hierarchical Logging Config ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ NOTE: this content is left over from the first edition, and has not been integrated into the new edition When we hacked in the `logging.warning` earlier, we were using the root logger. That's not normally a good idea, since any third-party package can mess with the root logger. The normal pattern is to use a logger named after the file you're in, by using: [role="skipme"] [source,python] ---- logger = logging.getLogger(__name__) ---- Logging configuration is hierarchical, so you can define "parent" loggers for top-level modules, and all the Python modules inside them will inherit that config. Here's how we add a logger for both our apps into 'settings.py': [role="sourcecode skipme"] .superlists/settings.py ==== [source,python] ---- LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'handlers': { 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', }, }, 'loggers': { 'django': { 'handlers': ['console'], }, 'accounts': { 'handlers': ['console'], }, 'lists': { 'handlers': ['console'], }, }, 'root': {'level': 'INFO'}, } ---- ==== Now 'accounts.models', 'accounts.views', 'accounts.authentication', and all the others will inherit the `logging.StreamHandler` from the parent 'accounts' logger. Unfortunately, because of Django's project structure, there's no way of defining a top-level logger for your whole project (aside from using the root logger), so you have to define one logger per app. Here's how to write a test for logging behaviour: [role="sourcecode skipme"] .accounts/tests/test_authentication.py (ch18l023) ==== [source,python] ---- import logging [...] @patch('accounts.authentication.requests.post') class AuthenticateTest(TestCase): [...] def test_logs_non_okay_responses_from_persona(self, mock_post): response_json = { 'status': 'not okay', 'reason': 'eg, audience mismatch' } mock_post.return_value.ok = True mock_post.return_value.json.return_value = response_json #<1> logger = logging.getLogger('accounts.authentication') #<2> with patch.object(logger, 'warning') as mock_log_warning: #<3> self.backend.authenticate('an assertion') mock_log_warning.assert_called_once_with( 'Persona says no. Json was: {}'.format(response_json) #<4> ) ---- ==== <1> We set up our test with some data that should cause some logging. <2> We retrieve the actual logger for the module we're testing. <3> We use `patch.object` to temporarily mock out its warning function, by using `with` to make it a 'context manager' around the function we're testing. <4> And then it's available for us to make assertions against. That gives us: [role="skipme"] [subs="specialcharacters,macros"] ---- AssertionError: Expected 'warning' to be called once. Called 0 times. ---- Let's just try it out, to make sure we really are testing what we think we are: [role="sourcecode skipme"] .accounts/authentication.py (ch18l024) ==== [source,python] ---- import logging logger = logging.getLogger(__name__) [...] if response.ok and response.json()['status'] == 'okay': [...] else: logger.warning('foo') ---- ==== We get the expected failure: [role="skipme"] [subs="specialcharacters,macros"] ---- AssertionError: Expected call: warning("Persona says no. Json was: {'status': 'not okay', 'reason': 'eg, audience mismatch'}") Actual call: warning('foo') ---- And so we settle in with our real implementation: [role="sourcecode skipme"] .accounts/authentication.py (ch18l025) ==== [source,python] ---- else: logger.warning( 'Persona says no. Json was: {}'.format(response.json()) ) ---- ==== [role="skipme"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test accounts*] [...] Ran 15 tests in 0.033s OK ---- You can easily imagine how you could test more combinations at this point, if you wanted different error messages for `response.ok != True`, and so on. .More notes ******************************************************************************* Use loggers named after the module you're in:: The root logger is a single global object, available to any library that's loaded in your Python process, so you're never quite in control of it. Instead, follow the `logging.getLogger(__name__)` pattern to get one that's unique to your module, but that inherits from a top-level configuration you control. Test important log messages:: As we saw, log messages can be critical to debugging issues in production. If a log message is important enough to keep in your codebase, it's probably important enough to test. We follow the rule of thumb that anything above `logging.INFO` definitely needs a test. Using `patch.object` on the logger for the module you're testing is one convenient way of unit testing it. ******************************************************************************* ================================================ FILE: appendix_purist_unit_tests.asciidoc ================================================ [[appendix_purist_unit_tests]] [appendix] == Test Isolation, and "Listening to Your Tests" .Warning, Appendix Not Updated ******************************************************************************* 🚧 Warning, this appendix is the 2e version, and uses Django 1.11 This appendix and all the following ones are the second edition versions, so they still use Django 1.11, Python 3.8, and so on. To follow along with this appendix, it’s probably easiest to reset your code to match my example code as it was in the 2e, by resetting to: https://github.com/hjwp/book-example/tree/chapter_outside_in And you should also probably delete and re-create your virtualenv with * Python 3.8 or 3.9 * and Django 1.11 (pip install "django <2") Alternatively, you can muddle through and try and figure out how to make things work with Django 5 etc, but be aware that the listings below won’t be quite right. ******************************************************************************* ((("functional tests (FTs)", "ensuring isolation", id="FTisolat23"))) This appendix picks up from a point about half-way through <>, when we made the decision to leave a unit test failing in the views layer while we proceeded to write more tests and more code at the models layer to get it to pass: <>. We got away with it because our app is simple, but in a more complex application, this would feel more risky. Is there a way to "finish" a higher level, even when the lower levels don't exist yet?footnote:[ I'm grateful to Gary Bernhardt, who took a look at an early draft of the <>, and encouraged me to get into a longer discussion of test isolation.] In this appendix we'll explore the using mocks to stand in for parts of the code we haven't written yet, enabling a form of outside-in development with isolated tests at each layer. WARNING: This is an example of "London-school" TDD, which is not the style I usually use, which means I'm not necessarily the best guide to the topic. If you're intrigued, the seminal book on the topic is "GOOSGBT", aka http://www.growing-object-oriented-software.com/[Growing Object-Oriented Software Guided by Tests] by Steve Freeman and Nat Pryce, and I enthusiastically recommend you read that as a better guide to London-style TDD. ((("isolation, ensuring", "benefits and drawbacks of"))) As we'll see, using mocks in this way can be a lot of work, but it can be a way to use our tests to give us feedback on design, and thus encourage us to write better code. NOTE: I revisited some of the tradeoffs outlined here in my my https://www.cosmicpython.com[second book on architecture patterns]. === Revisiting Our Decision Point: The Views Layer Depends on Unwritten Models Code ((("isolation, ensuring", "failed test example"))) Let's revisit the point we were at halfway through the outside-in chapter, when we couldn't get the `new_list` view to work because lists didn't have the `.owner` attribute yet. We'll actually go back in time and check out the old codebase using the tag we saved earlier, so that we can see how things would have worked if we'd used more isolated tests: [subs="specialcharacters,quotes"] ---- $ *git switch -c more-isolation* # a branch for this experiment $ *git reset --hard revisit_this_point_with_isolated_tests* ---- Here's what our failing test looks like: [role="sourcecode currentcontents"] .lists/tests/test_views.py ==== [source,python] ---- class NewListTest(TestCase): [...] def test_list_owner_is_saved_if_user_is_authenticated(self): user = User.objects.create(email='a@b.com') self.client.force_login(user) self.client.post('/lists/new', data={'text': 'new item'}) list_ = List.objects.first() self.assertEqual(list_.owner, user) ---- ==== And here's what our attempted solution looked like: [role="sourcecode currentcontents"] .lists/views.py ==== [source,python] ---- def new_list(request): form = ItemForm(data=request.POST) if form.is_valid(): list_ = List() list_.owner = request.user list_.save() form.save(for_list=list_) return redirect(list_) else: return render(request, 'home.html', {"form": form}) ---- ==== And at this point, the view test is failing because we don't have the model layer yet: ---- self.assertEqual(list_.owner, user) AttributeError: 'List' object has no attribute 'owner' ---- NOTE: You won't see this error unless you actually check out the old code and revert 'lists/models.py'. You should definitely do this; part of the objective of this appendix is to see whether we really can write tests for a models layer that doesn't exist yet. A First Attempt at Using Mocks for Isolation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("isolation, ensuring", "using mocks for", secondary-sortas="mocks for", id="IEmock23")))((("mocks", "isolating tests using", id="Misolate23")))Lists don't have owners yet, but we can let the views layer tests pretend they do by using a bit of mocking: //IDEA: rename all "mockList" to "mockListClass"... [role="sourcecode"] .lists/tests/test_views.py (ch20l003) ==== [source,python] ---- from unittest.mock import patch [...] @patch('lists.views.List') #<1> @patch('lists.views.ItemForm') #<2> def test_list_owner_is_saved_if_user_is_authenticated( self, mockItemFormClass, mockListClass #<3> ): user = User.objects.create(email='a@b.com') self.client.force_login(user) self.client.post('/lists/new', data={'text': 'new item'}) mock_list = mockListClass.return_value #<4> self.assertEqual(mock_list.owner, user) #<5> ---- ==== <1> We mock out the `List` class to be able to get access to any lists that might be created by the view. <2> We also mock out the `ItemForm`. Otherwise, our form will raise an error when we call `form.save()`, because it can't use a mock object as the foreign key for the +Item+ it wants to create. Once you start mocking, it can be hard to stop! <3> The mock objects are injected into the test's arguments in the opposite order to which they're declared. Tests with lots of mocks often have this strange signature, with the dangling `):`. You get used to it! <4> The list instance that the view will have access to will be the return value of the mocked `List` class. <5> And we can make assertions about whether the `.owner` attribute is set on it. If we try to run this test now, it should pass: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] Ran 37 tests in 0.145s OK ---- If you don't see a pass, make sure that your views code in 'views.py' is exactly as I've shown it, using `List()`, not `List.objects.create`. NOTE: Using mocks does tie you to specific ways of using an API. This is one of the many trade-offs involved in the use of mock objects. Using Mock side_effects to Check the Sequence of Events ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The trouble with this test is that it can still let us get away with writing the wrong code by mistake. Imagine if we accidentally call +save+ before we we assign the owner: [role="sourcecode"] .lists/views.py ==== [source,python] ---- if form.is_valid(): list_ = List() list_.save() list_.owner = request.user form.save(for_list=list_) return redirect(list_) ---- ==== The test, as it's written now, still passes: ---- OK ---- So strictly speaking, we need to check not just that the owner is assigned, but that it's assigned 'before' we call +save+ on our list object. Here's how we could test the sequence of events using mocks--you can mock out a function, and use it as a spy to check on the state of the world at the moment it's called: [role="sourcecode"] .lists/tests/test_views.py (ch20l005) ==== [source,python] ---- @patch('lists.views.List') @patch('lists.views.ItemForm') def test_list_owner_is_saved_if_user_is_authenticated( self, mockItemFormClass, mockListClass ): user = User.objects.create(email='a@b.com') self.client.force_login(user) mock_list = mockListClass.return_value def check_owner_assigned(): #<1> self.assertEqual(mock_list.owner, user) mock_list.save.side_effect = check_owner_assigned #<2> self.client.post('/lists/new', data={'text': 'new item'}) mock_list.save.assert_called_once_with() #<3> ---- ==== <1> We define a function that makes the assertion about the thing we want to happen first: checking that the list's owner has been set. <2> We assign that check function as a `side_effect` to the thing we want to check happened second. When the view calls our mocked save function, it will go through this assertion. We make sure to set this up before we actually call the function we're testing. <3> Finally, we make sure that the function with the `side_effect` was actually triggered--that is, that we did `.save()`. Otherwise, our assertion may actually never have been run. TIP: Two common mistakes when you're using mock side effects are assigning the side effect too late (i.e., 'after' you call the function under test), and forgetting to check that the side-effect function was actually called. And by common, I mean, "I made both these mistakes several times _while writing this chapter_.” At this point, if you've still got the "broken" code from earlier, where we assign the owner but call +save+ in the wrong order, you should now see a fail: ---- FAIL: test_list_owner_is_saved_if_user_is_authenticated (lists.tests.test_views.NewListTest) [...] File "...goat-book/lists/views.py", line 17, in new_list list_.save() [...] File "...goat-book/lists/tests/test_views.py", line 74, in check_owner_assigned self.assertEqual(mock_list.owner, user) AssertionError: != ---- Notice how the failure happens when we try to save, and then go inside our `side_effect` function. We can get it passing again like this: [role="sourcecode"] .lists/views.py ==== [source,python] ---- if form.is_valid(): list_ = List() list_.owner = request.user list_.save() form.save(for_list=list_) return redirect(list_) ---- ==== //006 ... ---- OK ---- ((("", startref="IEmock23")))((("", startref="Misolate23")))But, boy, that's getting to be an ugly test! Listen to Your Tests: Ugly Tests Signal a Need to Refactor ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("isolation, ensuring", "refactoring ugly tests")))((("refactoring")))Whenever you find yourself having to write a test like this, and you're finding it hard work, it's likely that your tests are trying to tell you something. Eight lines of setup (two lines for mocks, three to set up a user, and three more for our side-effect function) is way too many. What this test is trying to tell us is that our view is doing too much work, dealing with creating a form, creating a new list object, 'and' deciding whether or not to save an owner for the list. We've already seen that we can make our views simpler and easier to understand by pushing some of the work down to a form class. Why does the view need to create the list object? Perhaps our `ItemForm.save` could do that? And why does the view need to make decisions about whether or not to save the `request.user`? Again, the form could do that. While we're giving this form more responsibilities, it feels like it should probably get a new name too. We could call it `NewListForm` instead, since that's a better representation of what it does...something like this? [role="sourcecode skipme"] .lists/views.py ==== [source,python] ---- # don't enter this code yet, we're only imagining it. def new_list(request): form = NewListForm(data=request.POST) if form.is_valid(): list_ = form.save(owner=request.user) # creates both List and Item return redirect(list_) else: return render(request, 'home.html', {"form": form}) ---- ==== That would be neater! Let's see how we'd get to that state by using fully isolated tests. Rewriting Our Tests for the View to Be Fully Isolated ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("isolation, ensuring", "view layer", id="IEviews23")))Our first attempt at a test suite for this view was highly 'integrated'. It needed the database layer and the forms layer to be fully functional in order for it to pass. We've started trying to make it more isolated, so let's now go all the way. Keep the Old Integrated Test Suite Around as a Sense Check ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Let's rename our old `NewListTest` class to `NewListViewIntegratedTest`, and throw away our attempt at a mocky test for saving the owner, putting back the integrated version, with a skip on it for now: [role="sourcecode"] .lists/tests/test_views.py (ch20l008) ==== [source,python] ---- import unittest [...] class NewListViewIntegratedTest(TestCase): def test_can_save_a_POST_request(self): [...] @unittest.skip def test_list_owner_is_saved_if_user_is_authenticated(self): user = User.objects.create(email='a@b.com') self.client.force_login(user) self.client.post('/lists/new', data={'text': 'new item'}) list_ = List.objects.first() self.assertEqual(list_.owner, user) ---- ==== TIP: Have you heard the term "integration test" and are wondering what the difference is from an "integrated test"? Go and take a peek at the definitions box in <>. [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] Ran 37 tests in 0.139s OK ---- A New Test Suite with Full Isolation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Let's start with a blank slate, and see if we can use isolated tests to drive a replacement of our `new_list` view. We'll call it `new_list2`, build it alongside the old view, and when we're ready, swap it in and see if the old integrated tests all still pass: [role="sourcecode"] .lists/views.py (ch20l009) ==== [source,python] ---- def new_list(request): [...] def new_list2(request): pass ---- ==== Thinking in Terms of Collaborators ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In order to rewrite our tests to be fully isolated, we need to throw out our old way of thinking about the tests in terms of the "real" effects of the view on things like the database, and instead think of it in terms of the objects it collaborates with, and how it interacts with them. In the new world, the view's main collaborator will be a form object, so we mock that out in order to be able to fully control it, and in order to be able to define, by wishful thinking, the way we want our form to work: [role="sourcecode"] .lists/tests/test_views.py (ch20l010) ==== [source,python] ---- from unittest.mock import patch from django.http import HttpRequest from lists.views import new_list2 [...] @patch('lists.views.NewListForm') #<2> class NewListViewUnitTest(unittest.TestCase): #<1> def setUp(self): self.request = HttpRequest() self.request.POST['text'] = 'new list item' #<3> def test_passes_POST_data_to_NewListForm(self, mockNewListForm): new_list2(self.request) mockNewListForm.assert_called_once_with(data=self.request.POST) #<4> ---- ==== <1> The Django `TestCase` class makes it too easy to write integrated tests. As a way of making sure we're writing "pure", isolated unit tests, we'll only use `unittest.TestCase`. <2> We mock out the +NewListForm+ class (which doesn't even exist yet). It's going to be used in all the tests, so we mock it out at the class level. <3> We set up a basic POST request in `setUp`, building up the request by hand rather than using the (overly integrated) Django Test Client. <4> And we check the first thing about our new view: it initialises its collaborator, the `NewListForm`, with the correct constructor--the data from the request. That will start with a failure, saying we don't have a `NewListForm` in our view yet: ---- AttributeError: does not have the attribute 'NewListForm' ---- Let's create a placeholder for it: [role="sourcecode"] .lists/views.py (ch20l011) ==== [source,python] ---- from lists.forms import ExistingListItemForm, ItemForm, NewListForm [...] ---- ==== and: [role="sourcecode"] .lists/forms.py (ch20l012) ==== [source,python] ---- class ItemForm(forms.models.ModelForm): [...] class NewListForm(object): pass class ExistingListItemForm(ItemForm): [...] ---- ==== Next we get a real failure: ---- AssertionError: Expected 'NewListForm' to be called once. Called 0 times. ---- And we implement like this: [role="sourcecode"] .lists/views.py (ch20l012-2) ==== [source,python] ---- def new_list2(request): NewListForm(data=request.POST) ---- ==== [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] Ran 38 tests in 0.143s OK ---- Let's continue. If the form is valid, we want to call +save+ on it: [role="sourcecode"] .lists/tests/test_views.py (ch20l013) ==== [source,python] ---- from unittest.mock import patch, Mock [...] @patch('lists.views.NewListForm') class NewListViewUnitTest(unittest.TestCase): def setUp(self): self.request = HttpRequest() self.request.POST['text'] = 'new list item' self.request.user = Mock() def test_passes_POST_data_to_NewListForm(self, mockNewListForm): new_list2(self.request) mockNewListForm.assert_called_once_with(data=self.request.POST) def test_saves_form_with_owner_if_form_valid(self, mockNewListForm): mock_form = mockNewListForm.return_value mock_form.is_valid.return_value = True new_list2(self.request) mock_form.save.assert_called_once_with(owner=self.request.user) ---- ==== [role="pagebreak-before"] That takes us to this: [role="sourcecode"] .lists/views.py (ch20l014) ==== [source,python] ---- def new_list2(request): form = NewListForm(data=request.POST) form.save(owner=request.user) ---- ==== In the case where the form is valid, we want the view to return a redirect, to send us to see the object that the form has just created. So we mock out another of the view's collaborators, the `redirect` function: [role="sourcecode"] .lists/tests/test_views.py (ch20l015) ==== [source,python] ---- @patch('lists.views.redirect') #<1> def test_redirects_to_form_returned_object_if_form_valid( self, mock_redirect, mockNewListForm #<2> ): mock_form = mockNewListForm.return_value mock_form.is_valid.return_value = True #<3> response = new_list2(self.request) self.assertEqual(response, mock_redirect.return_value) #<4> mock_redirect.assert_called_once_with(mock_form.save.return_value) #<5> ---- ==== <1> We mock out the `redirect` function, this time at the method level. <2> `patch` decorators are applied innermost first, so the new mock is injected to our method before the `mockNewListForm`. <3> We specify that we're testing the case where the form is valid. <4> We check that the response from the view is the result of the `redirect` function. <5> And we check that the redirect function was called with the object that the form returns on save. That takes us to here: [role="sourcecode"] .lists/views.py (ch20l016) ==== [source,python] ---- def new_list2(request): form = NewListForm(data=request.POST) list_ = form.save(owner=request.user) return redirect(list_) ---- ==== [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] Ran 40 tests in 0.163s OK ---- And now the failure case--if the form is invalid, we want to render the home page template: [role="sourcecode"] .lists/tests/test_views.py (ch20l017) ==== [source,python] ---- @patch('lists.views.render') def test_renders_home_template_with_form_if_form_invalid( self, mock_render, mockNewListForm ): mock_form = mockNewListForm.return_value mock_form.is_valid.return_value = False response = new_list2(self.request) self.assertEqual(response, mock_render.return_value) mock_render.assert_called_once_with( self.request, 'home.html', {'form': mock_form} ) ---- ==== That gives us: ---- AssertionError: != ---- TIP: When using assert methods on mocks, like +assert_called_​once_with+, it's doubly important to make sure you run the test and see it fail. It's all too easy to make a typo in your assert function name and end up calling a mock method that does nothing (mine was to write `asssert_called_once_with` with three essses; try it!). //TODO: this is now a duplicate warning compared to mocking chapter. // replace all assert_calleds with self.assertEquals? We make a deliberate mistake, just to make sure our tests are comprehensive: [role="sourcecode"] .lists/views.py (ch20l018) ==== [source,python] ---- def new_list2(request): form = NewListForm(data=request.POST) list_ = form.save(owner=request.user) if form.is_valid(): return redirect(list_) return render(request, 'home.html', {'form': form}) ---- ==== That passes, but it shouldn't! One more test then: [role="sourcecode"] .lists/tests/test_views.py (ch20l019) ==== [source,python] ---- def test_does_not_save_if_form_invalid(self, mockNewListForm): mock_form = mockNewListForm.return_value mock_form.is_valid.return_value = False new_list2(self.request) self.assertFalse(mock_form.save.called) ---- ==== Which fails: ---- self.assertFalse(mock_form.save.called) AssertionError: True is not false ---- ((("", startref="IEviews23")))And we get to to our neat, small finished view: [role="sourcecode"] .lists/views.py ==== [source,python] ---- def new_list2(request): form = NewListForm(data=request.POST) if form.is_valid(): list_ = form.save(owner=request.user) return redirect(list_) return render(request, 'home.html', {'form': form}) ---- ==== ... [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] Ran 42 tests in 0.163s OK ---- Moving Down to the Forms Layer ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("isolation, ensuring", "forms layer", id="IEforms23")))So we've built up our view function based on a "wishful thinking" version of a form called `NewListForm`, which doesn't even exist yet. We'll need the form's save method to create a new list, and a new item based on the text from the form's validated POST data. If we were to just dive in and use the ORM, the code might look something a bit like this: [role="skipme"] [source,python] ---- class NewListForm(forms.models.ModelForm): def save(self, owner): list_ = List() if owner: list_.owner = owner list_.save() item = Item() item.list = list_ item.text = self.cleaned_data['text'] item.save() ---- This implementation depends on two classes from the model layer, `Item` and `List`. So, what would a well-isolated test look like? [role="skipme"] [source,python] ---- class NewListFormTest(unittest.TestCase): @patch('lists.forms.List') #<1> @patch('lists.forms.Item') #<1> def test_save_creates_new_list_and_item_from_post_data( self, mockItem, mockList #<1> ): mock_item = mockItem.return_value mock_list = mockList.return_value user = Mock() form = NewListForm(data={'text': 'new item text'}) form.is_valid() #<2> def check_item_text_and_list(): self.assertEqual(mock_item.text, 'new item text') self.assertEqual(mock_item.list, mock_list) self.assertTrue(mock_list.save.called) mock_item.save.side_effect = check_item_text_and_list #<3> form.save(owner=user) self.assertTrue(mock_item.save.called) #<4> ---- <1> We mock out the two collaborators for our form from the models layer below. <2> We need to call `is_valid()` so that the form populates the `.cleaned_data` dictionary where it stores validated data. <3> We use the `side_effect` method to make sure that, when we save the new item object, we're doing so with a saved `List` and with the correct item text. <4> As always, we double-check that our side-effect function was actually called. Yuck! What an ugly test! Let's not even bother saving that to disk, we can do better. Keep Listening to Your Tests: Removing ORM Code from Our Application ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ((("object-relational mapper (ORM)")))Again, these tests are trying to tell us something: the Django ORM is hard to mock out, and our form class needs to know too much about how it works. Programming by wishful thinking again, what would be a simpler API that our form could use? How about something like this: [role="skipme"] [source,python] ---- def save(self): List.create_new(first_item_text=self.cleaned_data['text']) ---- Our wishful thinking says: how about a helper method that would live on the `List` classfootnote:[It could easily just be a standalone function, but hanging it on the model class is a nice way to keep track of where it lives, and gives a bit more of a hint as to what it will do.] and encapsulate all the logic of saving a new list object and its associated first item? So let's write a test for that instead: [role="sourcecode"] .lists/tests/test_forms.py (ch20l021) ==== [source,python] ---- import unittest from unittest.mock import patch, Mock from django.test import TestCase from lists.forms import ( DUPLICATE_ITEM_ERROR, EMPTY_ITEM_ERROR, ExistingListItemForm, ItemForm, NewListForm ) from lists.models import Item, List [...] class NewListFormTest(unittest.TestCase): @patch('lists.forms.List.create_new') def test_save_creates_new_list_from_post_data_if_user_not_authenticated( self, mock_List_create_new ): user = Mock(is_authenticated=False) form = NewListForm(data={'text': 'new item text'}) form.is_valid() form.save(owner=user) mock_List_create_new.assert_called_once_with( first_item_text='new item text' ) ---- ==== [role="pagebreak-before"] And while we're at it, we can test the case where the user is an authenticated user too: [role="sourcecode"] .lists/tests/test_forms.py (ch20l022) ==== [source,python] ---- @patch('lists.forms.List.create_new') def test_save_creates_new_list_with_owner_if_user_authenticated( self, mock_List_create_new ): user = Mock(is_authenticated=True) form = NewListForm(data={'text': 'new item text'}) form.is_valid() form.save(owner=user) mock_List_create_new.assert_called_once_with( first_item_text='new item text', owner=user ) ---- ==== You can see this is a much more readable test. Let's start implementing our new form. We start with the import: [role="sourcecode"] .lists/forms.py (ch20l023) ==== [source,python] ---- from lists.models import Item, List ---- ==== Now mock tells us to create a placeholder for our `create_new` method: [subs="specialcharacters,macros"] ---- AttributeError: does not have the attribute 'create_new' ---- [role="sourcecode"] .lists/models.py ==== [source,python] ---- class List(models.Model): def get_absolute_url(self): return reverse('view_list', args=[self.id]) def create_new(): pass ---- ==== //24 And after a few steps, we should end up with a form save method like this: [role="sourcecode small-code"] .lists/forms.py (ch20l025) ==== [source,python] ---- class NewListForm(ItemForm): def save(self, owner): if owner.is_authenticated: List.create_new(first_item_text=self.cleaned_data['text'], owner=owner) else: List.create_new(first_item_text=self.cleaned_data['text']) ---- ==== And passing tests: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] Ran 44 tests in 0.192s OK ---- .Hiding ORM Code Behind Helper Methods ******************************************************************************* ((("helper methods")))One of the techniques that emerged from our use of isolated tests was the "ORM helper method". Django's ORM lets you get things done quickly with a reasonably readable syntax (it's certainly much nicer than raw SQL!). But some people like to try to minimise the amount of ORM code in the application--particularly removing it from the views and forms layers. One reason is that it makes it much easier to test those layers. But another is that it forces us to build helper functions that express our domain logic more clearly. [keep-together]#Compare#: [role="skipme"] [source,python] ---- list_ = List() list_.save() item = Item() item.list = list_ item.text = self.cleaned_data['text'] item.save() ---- With: [role="skipme"] [source,python] ---- List.create_new(first_item_text=self.cleaned_data['text']) ---- This applies to read queries as well as write. Imagine something like this: [role="skipme"] [source,python] ---- Book.objects.filter(in_print=True, pub_date__lte=datetime.today()) ---- Versus a helper method, like: [role="skipme"] [source,python] ---- Book.all_available_books() ---- When we build helper functions, we can give them names that express what we are doing in terms of the business domain, which can actually make our code more legible, as well as giving us the benefit of keeping all ORM calls at the model layer, and thus making our whole application more loosely coupled.((("", startref="IEforms23"))) ******************************************************************************* Finally, Moving Down to the Models Layer ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("isolation, ensuring", "models layer", id="IEmodels23")))At the models layer, we no longer need to write isolated tests--the whole point of the models layer is to integrate with the database, so it's appropriate to write integrated tests: [role="sourcecode"] .lists/tests/test_models.py (ch20l026) ==== [source,python] ---- class ListModelTest(TestCase): def test_get_absolute_url(self): list_ = List.objects.create() self.assertEqual(list_.get_absolute_url(), f'/lists/{list_.id}/') def test_create_new_creates_list_and_first_item(self): List.create_new(first_item_text='new item text') new_item = Item.objects.first() self.assertEqual(new_item.text, 'new item text') new_list = List.objects.first() self.assertEqual(new_item.list, new_list) ---- ==== Which gives: [subs="specialcharacters,macros"] ---- TypeError: create_new() got an unexpected keyword argument 'first_item_text' ---- And that will take us to a first cut implementation that looks like this: [role="sourcecode"] .lists/models.py (ch20l027) ==== [source,python] ---- class List(models.Model): def get_absolute_url(self): return reverse('view_list', args=[self.id]) @staticmethod def create_new(first_item_text): list_ = List.objects.create() Item.objects.create(text=first_item_text, list=list_) ---- ==== Notice we've been able to get all the way down to the models layer, driving a nice design for the views and forms layers, and the `List` model still doesn't support having an owner! Now let's test the case where the list should have an owner, and add: [role="sourcecode"] .lists/tests/test_models.py (ch20l028) ==== [source,python] ---- from django.contrib.auth import get_user_model User = get_user_model() [...] def test_create_new_optionally_saves_owner(self): user = User.objects.create() List.create_new(first_item_text='new item text', owner=user) new_list = List.objects.first() self.assertEqual(new_list.owner, user) ---- ==== And while we're at it, we can write the tests for the new owner attribute: [role="sourcecode"] .lists/tests/test_models.py (ch20l029) ==== [source,python] ---- class ListModelTest(TestCase): [...] def test_lists_can_have_owners(self): List(owner=User()) # should not raise def test_list_owner_is_optional(self): List().full_clean() # should not raise ---- ==== These two are almost exactly the same tests we used in the outside-in chapter, but I've re-written them slightly so they don't actually save objects--just having them as in-memory objects is enough for this test. TIP: Use in-memory (unsaved) model objects in your tests whenever you can; it makes your tests faster. That gives: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] ERROR: test_create_new_optionally_saves_owner TypeError: create_new() got an unexpected keyword argument 'owner' [...] ERROR: test_lists_can_have_owners (lists.tests.test_models.ListModelTest) TypeError: 'owner' is an invalid keyword argument for this function [...] Ran 48 tests in 0.204s FAILED (errors=2) ---- We implement, just like we did in the chapter: [role="sourcecode"] .lists/models.py (ch20l030-1) ==== [source,python] ---- from django.conf import settings [...] class List(models.Model): owner = models.ForeignKey(settings.AUTH_USER_MODEL, blank=True, null=True) [...] ---- ==== That will give us the usual integrity failures, until we do a migration: ---- django.db.utils.OperationalError: no such column: lists_list.owner_id ---- Building the migration will get us down to three failures: [role="dofirst-ch20l030-2"] [subs="specialcharacters,macros"] ---- ERROR: test_create_new_optionally_saves_owner TypeError: create_new() got an unexpected keyword argument 'owner' [...] ValueError: Cannot assign ">": "List.owner" must be a "User" instance. ValueError: Cannot assign ">": "List.owner" must be a "User" instance. ---- Let's deal with the first one, which is for our `create_new` method: [role="sourcecode"] .lists/models.py (ch20l030-3) ==== [source,python] ---- @staticmethod def create_new(first_item_text, owner=None): list_ = List.objects.create(owner=owner) Item.objects.create(text=first_item_text, list=list_) ---- ==== Back to Views ^^^^^^^^^^^^^ Two of our old integrated tests for the views layer are failing. What's happening? ---- ValueError: Cannot assign ">": "List.owner" must be a "User" instance. ---- Ah, the old view isn't discerning enough about what it does with list owners yet: [role="sourcecode currentcontents"] .lists/views.py ==== [source,python] ---- if form.is_valid(): list_ = List() list_.owner = request.user list_.save() ---- ==== This is the point at which we realise that our old code wasn't fit for purpose. Let's fix it to get all our tests passing: [role="sourcecode"] .lists/views.py (ch20l031) ==== [source,python] ---- def new_list(request): form = ItemForm(data=request.POST) if form.is_valid(): list_ = List() if request.user.is_authenticated: list_.owner = request.user list_.save() form.save(for_list=list_) return redirect(list_) else: return render(request, 'home.html', {"form": form}) def new_list2(request): [...] ---- ==== NOTE: ((("", startref="IEmodels23")))((("integrated tests", "benefits and drawbacks of")))One of the benefits of integrated tests is that they help you to catch less predictable interactions like this. We'd forgotten to write a test for the case where the user is not authenticated, but because the integrated tests use the stack all the way down, errors from the model layer came up to let us know we'd forgotten something: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] Ran 48 tests in 0.175s OK ---- The Moment of Truth (and the Risks of Mocking) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("mocks", "benefits and drawbacks of")))((("isolation, ensuring", "risks of mocking")))So let's try switching out our old view, and activating our new view. We can make the swap in 'urls.py': [role="sourcecode"] .lists/urls.py ==== [source,python] ---- [...] url(r'^new$', views.new_list2, name='new_list'), ---- ==== We should also remove the `unittest.skip` from our integrated test class, to see if our new code for list owners really works: [role="sourcecode"] .lists/tests/test_views.py (ch20l033) ==== [source,python] ---- class NewListViewIntegratedTest(TestCase): def test_can_save_a_POST_request(self): [...] def test_list_owner_is_saved_if_user_is_authenticated(self): [...] self.assertEqual(list_.owner, user) ---- ==== So what happens when we run our tests? Oh no! ---- ERROR: test_list_owner_is_saved_if_user_is_authenticated [...] ERROR: test_can_save_a_POST_request [...] ERROR: test_redirects_after_POST (lists.tests.test_views.NewListViewIntegratedTest) File "...goat-book/lists/views.py", line 30, in new_list2 return redirect(list_) [...] TypeError: argument of type 'NoneType' is not iterable FAILED (errors=3) ---- Here's an important lesson to learn about test isolation: it might help you to drive out good design for individual layers, but it won't automatically verify the integration 'between' your layers. What's happened here is that the view was expecting the form to return a list item: [role="sourcecode currentcontents"] .lists/views.py ==== [source,python] ---- list_ = form.save(owner=request.user) return redirect(list_) ---- ==== But we forgot to make it return anything: [role="sourcecode currentcontents small-code"] .lists/forms.py ==== [source,python] ---- def save(self, owner): if owner.is_authenticated: List.create_new(first_item_text=self.cleaned_data['text'], owner=owner) else: List.create_new(first_item_text=self.cleaned_data['text']) ---- ==== Thinking of Interactions Between Layers as "Contracts" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("isolation, ensuring", "layer interactions as contracts", id="IEinteract23")))Ultimately, even if we had been writing nothing but isolated unit tests, our functional tests would have picked up this particular slip-up. But ideally we'd want our feedback cycle to be quicker--functional tests may take a couple of minutes to run, or even a few hours once your app starts to grow. Is there any way to avoid this sort of problem before it happens? Methodologically, the way to do it is to think about the interaction between your layers in terms of contracts. Whenever we mock out the behaviour of one layer, we have to make a mental note that there is now an implicit contract between the layers, and that a mock on one layer should probably translate into a test at the layer below. Here's the part of the contract that we missed: [role="sourcecode currentcontents"] .lists/tests/test_views.py ==== [source,python] ---- @patch('lists.views.redirect') def test_redirects_to_form_returned_object_if_form_valid( self, mock_redirect, mockNewListForm ): mock_form = mockNewListForm.return_value mock_form.is_valid.return_value = True response = new_list2(self.request) self.assertEqual(response, mock_redirect.return_value) mock_redirect.assert_called_once_with(mock_form.save.return_value) #<1> ---- ==== <1> The mocked `form.save` function is returning an object, which we expect our view to be able to use. Identifying Implicit Contracts ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It's worth reviewing each of the tests in `NewListViewUnitTest` and seeing what each mock is saying about the implicit contract: [role="sourcecode currentcontents"] .lists/tests/test_views.py ==== [source,python] ---- def test_passes_POST_data_to_NewListForm(self, mockNewListForm): [...] mockNewListForm.assert_called_once_with(data=self.request.POST) #<1> def test_saves_form_with_owner_if_form_valid(self, mockNewListForm): mock_form = mockNewListForm.return_value mock_form.is_valid.return_value = True #<2> new_list2(self.request) mock_form.save.assert_called_once_with(owner=self.request.user) #<3> def test_does_not_save_if_form_invalid(self, mockNewListForm): [...] mock_form.is_valid.return_value = False #<2> [...] @patch('lists.views.redirect') def test_redirects_to_form_returned_object_if_form_valid( self, mock_redirect, mockNewListForm ): [...] mock_redirect.assert_called_once_with(mock_form.save.return_value) #<4> @patch('lists.views.render') def test_renders_home_template_with_form_if_form_invalid( [...] ---- ==== <1> We need to be able to initialise our form by passing it a POST request as data. <2> It should have an `is_valid()` function which returns +True+ or +False+ appropriately, based on the input data. <3> The form should have a `.save` method which will accept a `request.user`, which may or may not be a logged-in user, and deal with it appropriately. <4> The form's `.save` method should return a new list object, for our view to redirect the user to. If we have a look through our form tests, we'll see that, actually, only item (3) is tested explicitly. On items (1) and (2) we were lucky--they're default features of a Django `ModelForm`, and they are actually covered by our tests for the parent `ItemForm` class. But contract clause number (4) managed to slip through the net. NOTE: When doing Outside-In TDD with isolated tests, you need to keep track of each test's implicit assumptions about the contract which the next layer should implement, and remember to test each of those in turn later. You could use our scratchpad for this, or create a placeholder test with a `self.fail`. Fixing the Oversight ^^^^^^^^^^^^^^^^^^^^ Let's add a new test that our form should return the new saved list: [role="sourcecode"] .lists/tests/test_forms.py (ch20l038-1) ==== [source,python] ---- @patch('lists.forms.List.create_new') def test_save_returns_new_list_object(self, mock_List_create_new): user = Mock(is_authenticated=True) form = NewListForm(data={'text': 'new item text'}) form.is_valid() response = form.save(owner=user) self.assertEqual(response, mock_List_create_new.return_value) ---- ==== And, actually, this is a good example--we have an implicit contract with the `List.create_new`; we want it to return the new list object. Let's add a placeholder test for that: [role="sourcecode"] .lists/tests/test_models.py (ch20l038-2) ==== [source,python] ---- class ListModelTest(TestCase): [...] def test_create_returns_new_list_object(self): self.fail() ---- ==== So, we have one test failure that's telling us to fix the form save: ---- AssertionError: None != FAILED (failures=2, errors=3) ---- Like this: [role="sourcecode small-code"] .lists/forms.py (ch20l039-1) ==== [source,python] ---- class NewListForm(ItemForm): def save(self, owner): if owner.is_authenticated: return List.create_new(first_item_text=self.cleaned_data['text'], owner=owner) else: return List.create_new(first_item_text=self.cleaned_data['text']) ---- ==== That's a start; now we should look at our placeholder test: ---- [...] FAIL: test_create_returns_new_list_object self.fail() AssertionError: None FAILED (failures=1, errors=3) ---- We flesh it out: [role="sourcecode"] .lists/tests/test_models.py (ch20l039-2) ==== [source,python] ---- def test_create_returns_new_list_object(self): returned = List.create_new(first_item_text='new item text') new_list = List.objects.first() self.assertEqual(returned, new_list) ---- ==== ... ---- AssertionError: None != ---- And we add our return value: [role="sourcecode"] .lists/models.py (ch20l039-3) ==== [source,python] ---- @staticmethod def create_new(first_item_text, owner=None): list_ = List.objects.create(owner=owner) Item.objects.create(text=first_item_text, list=list_) return list_ ---- ==== ((("", startref="IEinteract23")))And that gets us to a fully passing test suite: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] Ran 50 tests in 0.169s OK ---- One More Test ~~~~~~~~~~~~~ That's our code for saving list owners, test-driven all the way down and working. But our functional test isn't passing quite yet: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests.test_my_lists*] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: Reticulate splines ---- It's because we have one last feature to implement, the `.name` attribute on list objects. Again, we can grab the test and code from the outside-in chapter: [role="sourcecode"] .lists/tests/test_models.py (ch20l040) ==== [source,python] ---- def test_list_name_is_first_item_text(self): list_ = List.objects.create() Item.objects.create(list=list_, text='first item') Item.objects.create(list=list_, text='second item') self.assertEqual(list_.name, 'first item') ---- ==== (Again, since this is a model-layer test, it's OK to use the ORM. You could conceivably write this test using mocks, but there wouldn't be much point.) [role="sourcecode"] .lists/models.py (ch20l041) ==== [source,python] ---- @property def name(self): return self.item_set.first().text ---- ==== And that gets us to a passing FT! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests.test_my_lists*] Ran 1 test in 21.428s OK ---- Tidy Up: What to Keep from Our Integrated Test Suite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ((("isolation, ensuring", "removing redundant code", id="IEredund23")))Now everything is working, we can remove some redundant tests, and decide whether we want to keep any of our old integrated tests. Removing Redundant Code at the Forms Layer ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We can get rid of the test for the old save method on the `ItemForm`: [role="sourcecode"] .lists/tests/test_forms.py ==== [source,diff] ---- --- a/lists/tests/test_forms.py +++ b/lists/tests/test_forms.py @@ -23,14 +23,6 @@ class ItemFormTest(TestCase): self.assertEqual(form.errors['text'], [EMPTY_ITEM_ERROR]) - def test_form_save_handles_saving_to_a_list(self): - list_ = List.objects.create() - form = ItemForm(data={'text': 'do me'}) - new_item = form.save(for_list=list_) - self.assertEqual(new_item, Item.objects.first()) - self.assertEqual(new_item.text, 'do me') - self.assertEqual(new_item.list, list_) - ---- ==== And in our actual code, we can get rid of two redundant save methods in 'forms.py': [role="sourcecode"] .lists/forms.py ==== [source,diff] ---- --- a/lists/forms.py +++ b/lists/forms.py @@ -22,11 +22,6 @@ class ItemForm(forms.models.ModelForm): self.fields['text'].error_messages['required'] = EMPTY_ITEM_ERROR - def save(self, for_list): - self.instance.list = for_list - return super().save() - - class NewListForm(ItemForm): @@ -52,8 +47,3 @@ class ExistingListItemForm(ItemForm): e.error_dict = {'text': [DUPLICATE_ITEM_ERROR]} self._update_errors(e) - - - def save(self): - return forms.models.ModelForm.save(self) - ---- ==== Removing the Old Implementation of the View ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We can now completely remove the old `new_list` view, and rename `new_list2` to `new_list`: [role="sourcecode skipme"] .lists/tests/test_views.py ==== [source,diff] ---- -from lists.views import new_list, new_list2 +from lists.views import new_list class HomePageTest(TestCase): @@ -75,7 +75,7 @@ class NewListViewIntegratedTest(TestCase): request = HttpRequest() request.user = User.objects.create(email='a@b.com') request.POST['text'] = 'new list item' - new_list2(request) + new_list(request) list_ = List.objects.first() self.assertEqual(list_.owner, request.user) @@ -91,21 +91,21 @@ class NewListViewUnitTest(unittest.TestCase): def test_passes_POST_data_to_NewListForm(self, mockNewListForm): - new_list2(self.request) + new_list(self.request) [.. several more] ---- ==== [role="sourcecode dofirst-ch20l045"] .lists/urls.py ==== [source,diff] ---- --- a/lists/urls.py +++ b/lists/urls.py @@ -3,7 +3,7 @@ from django.conf.urls import url from lists import views urlpatterns = [ - url(r'^new$', views.new_list2, name='new_list'), + url(r'^new$', views.new_list, name='new_list'), url(r'^(\d+)/$', views.view_list, name='view_list'), url(r'^users/(.+)/$', views.my_lists, name='my_lists'), ] ---- ==== [role="sourcecode"] .lists/views.py (ch20l047) ==== [source,python] ---- def new_list(request): form = NewListForm(data=request.POST) if form.is_valid(): list_ = form.save(owner=request.user) [...] ---- ==== And a quick check that all the tests still pass: ---- OK ---- [role="pagebreak-before less_space"] Removing Redundant Code at the Forms Layer ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Finally, we have to decide what (if anything) to keep from our integrated test suite. One option is to throw them all away, and decide that the FTs will pick up any integration problems. That's perfectly valid. On the other hand, we saw how integrated tests can warn you when you've made small mistakes in integrating your layers. We could keep just a couple of tests around as "sense checks", to give us a quicker feedback cycle. How about these three: [role="sourcecode"] .lists/tests/test_views.py (ch20l048) ==== [source,python] ---- class NewListViewIntegratedTest(TestCase): def test_can_save_a_POST_request(self): self.client.post('/lists/new', data={'text': 'A new list item'}) self.assertEqual(Item.objects.count(), 1) new_item = Item.objects.first() self.assertEqual(new_item.text, 'A new list item') def test_for_invalid_input_doesnt_save_but_shows_errors(self): response = self.client.post('/lists/new', data={'text': ''}) self.assertEqual(List.objects.count(), 0) self.assertContains(response, escape(EMPTY_ITEM_ERROR)) def test_list_owner_is_saved_if_user_is_authenticated(self): user = User.objects.create(email='a@b.com') self.client.force_login(user) self.client.post('/lists/new', data={'text': 'new item'}) list_ = List.objects.first() self.assertEqual(list_.owner, user) ---- ==== If you're going to keep any intermediate-level tests at all, I like these three because they feel like they're doing the most "integration" jobs: they test the full stack, from the request down to the actual database, and they cover the three most important use cases of our view.((("", startref="IEredund23"))) Conclusions: When to Write Isolated Versus Integrated Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ TIP: I explored some of these issues in more detail in my https://www.cosmicpython.com[second book] ((("isolation, ensuring", "vs. integrated tests", secondary-sortas="integrated tests", id="IEinteg23")))((("integrated tests", "vs. isolated", secondary-sortas="isolated", id="IEisol23")))Django's testing tools make it very easy to quickly put together integrated tests. The test runner helpfully creates a fast, in-memory version of your database and resets it for you in between each test. The `TestCase` class and the test client make it easy to test your views, from checking whether database objects are modified, confirming that your URL mappings work, and inspecting the rendering of the templates. This lets you get started with testing very easily and get good coverage across your whole stack. On the other hand, these kinds of integrated tests won't necessarily deliver the full benefit that rigorous unit testing and Outside-In TDD are meant to confer in terms of design. If we look at the example in this appendix, compare the code we had before and after: [role="sourcecode skipme"] .Before [source,python] ---- def new_list(request): form = ItemForm(data=request.POST) if form.is_valid(): list_ = List() if not isinstance(request.user, AnonymousUser): list_.owner = request.user list_.save() form.save(for_list=list_) return redirect(list_) else: return render(request, 'home.html', {"form": form}) ---- [role="sourcecode skipme"] .After [source,python] ---- def new_list(request): form = NewListForm(data=request.POST) if form.is_valid(): list_ = form.save(owner=request.user) return redirect(list_) return render(request, 'home.html', {'form': form}) ---- If we hadn't bothered to go down the isolation route, would we have bothered to refactor the view function? I know I didn't in the first draft of this book. I'd like to think I would have "in real life", but it's hard to be sure. But writing isolated tests does make you very aware of where the complexities in your code lie. Let Complexity Be Your Guide ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ I'd say the point at which isolated tests start to become worth it is to do with complexity. The example in this book is extremely simple, so it's not usually been worth it so far. Even in the example in this appendix, I can convince myself I didn't really 'need' to write those isolated tests. But once an application gains a little more complexity--if it starts growing any more layers between views and models, if you find yourself writing helper methods, or if you're writing your own classes, then you will probably gain from writing more isolated tests. Should You Do Both? ^^^^^^^^^^^^^^^^^^^ We already have our suite of functional tests, which will serve the purpose of telling us if we ever make any mistakes in integrating the different parts of our code together. Writing isolated tests can help us to drive out better design for our code, and to verify correctness in finer detail. Would a middle layer of integration tests serve any additional purpose? I think the answer is potentially yes, if they can provide a faster feedback cycle, and help you identify more clearly what integration problems you suffer from--their tracebacks may provide you with better debug information than you would get from a functional test, for example. There may even be a case for building them as a separate test suite--you could have one suite of fast, isolated unit tests that don't even use `manage.py`, because they don't need any of the database cleanup and teardown that the Django test runner gives you, and then the intermediate layer that uses Django, and finally the functional tests layer that, say, talks to a staging server. It may be worth it if each layer delivers incremental benefits. It's a judgement call. I hope that, by going through this appendix, I've given you a feel for what the trade-offs are. There's more discussion on this in <>.((("", startref="IEinteg23")))((("", startref="IEisol23"))) Onwards! ^^^^^^^^ We're happy with our new version, so let's bring it across to master: [subs="specialcharacters,quotes"] ---- $ *git add .* $ *git commit -m "add list owners via forms. more isolated tests"* $ *git switch master* $ *git switch -c master-noforms-noisolation-bak* # optional backup $ *git switch -* $ *git reset --hard more-isolation* # reset master to our branch. ---- In the meantime--those FTs are taking an annoyingly long time to run. I wonder if there's something we can do about that? [role="pagebreak-before less_space"] .On the Pros and Cons of Different Types of Tests, pass:[
]and Decoupling ORM Code **** Functional tests:: * ((("functional tests (FTs)", "benefits and drawbacks of")))Provide the best guarantee that your application really works correctly, from the point of view of the user * But: it's a slower feedback cycle * And they don't necessarily help you write clean code Integrated tests (reliant on, for example, the ORM or the Django Test Client):: * ((("integrated tests", "benefits and drawbacks of")))Are quick to write * Are easy to understand * Will warn you of any integration issues * But: may not always drive good design (that's up to you!) * And are usually slower than isolated tests Isolated ("mocky") tests:: * ((("mocks", "benefits and drawbacks of")))((("isolation, ensuring", "benefits and drawbacks of")))Involve the most hard work * Can be harder to read and understand * But: are the best ones for guiding you towards better design * And run the fastest Decoupling our application from ORM code:: ((("object-relational mapper (ORM)")))One of the consequences of striving to write isolated tests is that we find ourselves forced to remove ORM code from places like views and forms, by hiding it behind helper functions or methods. This can be beneficial in terms of decoupling your application from the ORM, but also just because it makes your code more readable. As with all things, it's a judgement call as to whether the additional effort is worth it in particular circumstances.((("", startref="FTisolat23"))) **** ================================================ FILE: appendix_rest_api.asciidoc ================================================ [[appendix_rest_api]] [appendix] Building a REST API: JSON, Ajax, and Mocking with JavaScript ------------------------------------------------------------ ((("Representational State Transfer (REST)", "defined")))Representational State Transfer (REST) is an approach to designing a web service to allow a user to retrieve and update information about "resources". It's become the dominant approach when designing APIs for use over the web. We've built a working web app without needing an API so far. Why might we want one? One motivation might be to improve the user experience by making the site more dynamic. Rather than waiting for the page to refresh after each addition to a list, we can use JavaScript to fire off those requests asynchronously to our API, and give the user a more interactive feeling. Perhaps more interestingly, once we've built an API, we can interact with our back-end application via other mechanisms than the browser. A mobile app might be one new candidate client application, another might be some sort of command-line application, or other developers might be able to build libraries and tools around your backend. In this chapter we'll see how to build an API "by hand". In the next, I'll give an overview of how to use a popular tool from the Django ecosystem called Django-Rest-Framework. Our Approach for This Appendix ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ I won't convert the entirety of the app for now; we'll start by assuming we have an existing list. REST defines a relationship between URLs and the HTTP methods (GET and POST, but also the more funky ones like PUT and DELETE) which will guide us in our design.((("Representational State Transfer (REST)", "additional resources"))) http://bit.ly/2u6qeYw[The Wikipedia entry on REST] has a good overview. In brief: * Our new URL structure will be '/api/lists/{id}/' * GET will give you details of a list (including all its items) in JSON format * POST lets you add an item We'll take the code from its state at the end of <>. Choosing Our Test Approach ~~~~~~~~~~~~~~~~~~~~~~~~~~ If we ((("Representational State Transfer (REST)", "building a REST API", id="RESTbuild32")))were building an API that was entirely agnostic about its clients, we might want to think about what levels to test it at. The equivalent of functional tests would perhaps spin up a real server (maybe using `LiveServerTestCase`) and interact with it using the `requests` library. We'd have to think carefully about how to set up fixtures (if we use the API itself, that introduces a lot of dependencies between tests) and what additional layer of lower-level/unit tests might be most useful to us. Or we might decide that a single layer of tests using the Django Test Client would be enough. As it is, we're building an API in the context of a browser-based client side. We want to start using it on our production site, and have the app continue to provide the same functionality as it did before. So our functional tests will continue to serve the role of being the highest-level tests, and of checking the integration between our JavaScript and our API. That leaves the Django Test Client as a natural place to site our lower-level tests. Let's start there. Basic Piping ~~~~~~~~~~~~ We start with a unit test that just checks that our new URL structure returns a 200 response to GET requests, and that it uses the JSON format (instead of HTML): [role="sourcecode"] .lists/tests/test_api.py ==== [source,python] ---- import json from django.test import TestCase from lists.models import List, Item class ListAPITest(TestCase): base_url = '/api/lists/{}/' #<1> def test_get_returns_json_200(self): list_ = List.objects.create() response = self.client.get(self.base_url.format(list_.id)) self.assertEqual(response.status_code, 200) self.assertEqual(response['content-type'], 'application/json') ---- ==== <1> Using a class-level constant for the URL under test is a new pattern we'll introduce for this appendix. It'll help us to remove duplication of hardcoded URLs. You could even use a call to `reverse` to reduce duplication even further. First we wire up a couple of 'urls' files: [role="sourcecode"] .superlists/urls.py ==== [source,python] ---- from django.conf.urls import include, url from accounts import urls as accounts_urls from lists import views as list_views from lists import api_urls from lists import urls as list_urls urlpatterns = [ url(r'^$', list_views.home_page, name='home'), url(r'^lists/', include(list_urls)), url(r'^accounts/', include(accounts_urls)), url(r'^api/', include(api_urls)), ] ---- ==== and: [role="sourcecode"] .lists/api_urls.py ==== [source,python] ---- from django.conf.urls import url from lists import api urlpatterns = [ url(r'^lists/(\d+)/$', api.list, name='api_list'), ] ---- ==== And the actual core of our API can live in a file called 'api.py'. Just three lines should be enough: [role="sourcecode"] .lists/api.py ==== [source,python] ---- from django.http import HttpResponse def list(request, list_id): return HttpResponse(content_type='application/json') ---- ==== The tests should pass, and we have the basic piping together: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] .................................................. --------------------------------------------------------------------- Ran 50 tests in 0.177s OK ---- Actually Responding with Something ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Our next step is to get our API to actually respond with some content--specifically, a JSON representation of our list items: [role="sourcecode"] .lists/tests/test_api.py (ch36l002) ==== [source,python] ---- def test_get_returns_items_for_correct_list(self): other_list = List.objects.create() Item.objects.create(list=other_list, text='item 1') our_list = List.objects.create() item1 = Item.objects.create(list=our_list, text='item 1') item2 = Item.objects.create(list=our_list, text='item 2') response = self.client.get(self.base_url.format(our_list.id)) self.assertEqual( json.loads(response.content.decode('utf8')), #<1> [ {'id': item1.id, 'text': item1.text}, {'id': item2.id, 'text': item2.text}, ] ) ---- ==== <1> This is the main thing to notice about this test. We expect our response to be in JSON format; we use `json.loads()` because testing Python objects is easier than messing about with raw JSON strings. And the implementation, conversely, uses `json.dumps()`: [role="sourcecode"] .lists/api.py ==== [source,python] ---- import json from django.http import HttpResponse from lists.models import List, Item def list(request, list_id): list_ = List.objects.get(id=list_id) item_dicts = [ {'id': item.id, 'text': item.text} for item in list_.item_set.all() ] return HttpResponse( json.dumps(item_dicts), content_type='application/json' ) ---- ==== A nice opportunity to use a list comprehension! Adding POST ~~~~~~~~~~~ The second thing we need from our API is the ability to add new items to our list by using a POST request. We'll start with the "happy path": [role="sourcecode"] .lists/tests/test_api.py (ch36l004) ==== [source,python] ---- def test_POSTing_a_new_item(self): list_ = List.objects.create() response = self.client.post( self.base_url.format(list_.id), {'text': 'new item'}, ) self.assertEqual(response.status_code, 201) new_item = list_.item_set.get() self.assertEqual(new_item.text, 'new item') ---- ==== And the implementation is similarly simple--basically the same as what we do in our normal view, but we return a 201 rather than a redirect: [role="sourcecode"] .lists/api.py (ch36l005) ==== [source,python] ---- def list(request, list_id): list_ = List.objects.get(id=list_id) if request.method == 'POST': Item.objects.create(list=list_, text=request.POST['text']) return HttpResponse(status=201) item_dicts = [ [...] ---- ==== //ch36l005 And that should get us started: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] Ran 52 tests in 0.177s OK ---- NOTE: One of the fun things about building a REST API is that you get to use a few more of the full range of https://en.wikipedia.org/wiki/List_of_HTTP_status_codes[HTTP status codes]. Testing the Client-Side Ajax with Sinon.js ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Don't even 'think' of doing Ajax testing without a mocking library. Different test frameworks and tools have their own; 'Sinon' is generic. It also provides JavaScript mocks, as we'll see... Start by downloading it from its site, http://sinonjs.org/, and putting it into our 'lists/static/tests/' folder. Then we can write our first Ajax test: [role="sourcecode dofirst-ch36l006"] .lists/static/tests/tests.html (ch36l007) ==== [source,html] ----
Error text
<1>
<2> ---- ==== <1> We add a new item to the fixture `div` to represent our list table. <2> We import 'sinon.js' (you'll need to download it and put it in the right folder). <3> `testStart` and `testDone` are the QUnit equivalents of `setUp` and `tearDown`. We use them to tell Sinon to start up its Ajax testing tool, the `fakeServer`, and make it available via a globally scoped variable called `server`. <4> That lets us make assertions about any Ajax requests that were made by our code. In this case, we test what URL the request went to, and what HTTP method it used. To actually make our Ajax request, we'll use the https://api.jquery.com/jQuery.get/[jQuery Ajax helpers], which are 'much' easier than trying to use the low-level browser standard `XMLHttpRequest` objects: [role="sourcecode"] .lists/static/list.js ==== [source,diff] ---- @@ -1,6 +1,10 @@ window.Superlists = {}; -window.Superlists.initialize = function () { +window.Superlists.initialize = function (url) { $('input[name="text"]').on('keypress', function () { $('.has-error').hide(); }); + + $.get(url); + }; + ---- ==== That should get our test passing: [role="qunit-output"] ---- 5 assertions of 5 passed, 0 failed. 1. errors should be hidden on keypress (1) 2. errors aren't hidden if there is no keypress (1) 3. should get items by ajax on initialize (3) ---- Well, we might be pinging out a GET request to the server, but what about actually 'doing' something? How do we test the actual "async" part, where we deal with the (eventual) response? Sinon and Testing the Asynchronous Part of Ajax ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This is a major reason to love Sinon. `server.respond()` allows us to exactly control the flow of the asynchronous code. [role="sourcecode"] .lists/static/tests/tests.html (ch36l009) ==== [source,javascript] ---- QUnit.test("should fill in lists table from ajax response", function (assert) { var url = '/getitems/'; var responseData = [ {'id': 101, 'text': 'item 1 text'}, {'id': 102, 'text': 'item 2 text'}, ]; server.respondWith('GET', url, [ 200, {"Content-Type": "application/json"}, JSON.stringify(responseData) //<1> ]); window.Superlists.initialize(url); //<2> server.respond(); //<3> var rows = $('#id_list_table tr'); //<4> assert.equal(rows.length, 2); var row1 = $('#id_list_table tr:first-child td'); assert.equal(row1.text(), '1: item 1 text'); var row2 = $('#id_list_table tr:last-child td'); assert.equal(row2.text(), '2: item 2 text'); }); ---- ==== <1> We set up some response data for Sinon to use, telling it what status code, headers, and importantly what kind of response JSON we want to simulate coming from the server. <2> Then we call the function under test. <3> Here's the magic. 'Then' we can call `server.respond()`, whenever we like, and that will kick off all the async part of the Ajax loop—that is, any callback we've assigned to deal with the response. <4> Now we can quietly check whether our Ajax callback has actually populated our table with the new list rows... The implementation might look something like this: [role="sourcecode"] .lists/static/list.js (ch36l010) ==== [source,javascript] ---- if (url) { $.get(url).done(function (response) { //<1> var rows = ''; for (var i=0; i var item = response[i]; rows += '\n' + (i+1) + ': ' + item.text + ''; } $('#id_list_table').html(rows); }); } ---- ==== TIP: We're lucky because of the way jQuery registers its callbacks for Ajax when we use the `.done()` function. If you want to switch to the more standard JavaScript Promise `.then()` callback, we get one more "level" of async. QUnit does have a way of dealing with that. Check out the docs for the http://api.qunitjs.com/async/[async] function. Other test frameworks have something similar. Wiring It All Up in the Template to See If It Really Works ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ We break it first, by removing the list table `{% for %}` loop from the _lists.html_ [keep-together]#template#: [role="sourcecode"] .lists/templates/list.html ==== [source,diff] ---- @@ -6,9 +6,6 @@ {% block table %} - {% for item in list.item_set.all %} - - {% endfor %}
{{ forloop.counter }}: {{ item.text }}
{% if list.owner %} ---- ==== NOTE: This will cause one of the unit tests to fail. It's OK to delete that test at this point. .Graceful Degradation and Progressive Enhancement ******************************************************************************* By removing the non-Ajax version of the lists page, I've removed the option of https://www.w3.org/wiki/Graceful_degradation_versus_progressive_enhancement[graceful degradation]—that is, keeping a version of the site that will still work without [keep-together]#JavaScript#. This used to be an accessibility issue: "screen reader" browsers for visually impaired people used not to have JavaScript, so relying entirely on JavaScript would exclude those users. That's not so much of an issue any more, as I understand it. But some users will block JavaScript for security reasons. Another common problem is differing levels of JavaScript support in different browsers. This is a particular issue if you start adventuring off in the direction of "modern" frontend development and ES2015. [role="pagebreak-before"] In short, it's always nice to have a non-JavaScript "backup". Particularly if you've built a site that works fine without it, don't throw away your working "plain old" HTML version too hastily. I'm just doing it because it's convenient for what I want to [keep-together]#demonstrate#. ******************************************************************************* That causes our basic FT to fail: [role="dofirst-ch36l015"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests.test_simple_list_creation*] [...] FAIL: test_can_start_a_list_for_one_user [...] File "...goat-book/functional_tests/test_simple_list_creation.py", line 32, in test_can_start_a_list_for_one_user self.wait_for_row_in_list_table('1: Buy peacock feathers') [...] AssertionError: '1: Buy peacock feathers' not found in [] [...] FAIL: test_multiple_users_can_start_lists_at_different_urls FAILED (failures=2) ---- Let's add a block called `{% scripts %}` to the base template, which we can selectively override later in our lists page: [role="sourcecode"] .lists/templates/base.html ==== [source,html] ---- {% block scripts %} {% endblock scripts %} ---- ==== And now in 'list.html' we add a slightly different call to `initialize`, with the correct URL: [role="sourcecode"] .lists/templates/list.html (ch36l016) ==== [source,html] ---- {% block scripts %} {% endblock scripts %} ---- ==== And guess what? The test passes! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests.test_simple_list_creation*] [...] Ran 2 test in 11.730s OK ---- That's a pretty good start! Now if you run all the FTs you'll see we've got some failures in other FTs, so we'll have to deal with them. Also, we're using an old-fashioned POST from the form, with page refresh, so we're not at our trendy hipster single-page app yet. But we'll get there! //TODO: which FTs fail exactly? Implementing Ajax POST, Including the CSRF Token ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ First we give our list form an `id` so we can pick it up easily in our JS: [role="sourcecode small-code"] .lists/templates/base.html ==== [source,html] ----

{% block header_text %}{% endblock %}

{% block list_form %}
{{ form.text }} [...] ---- ==== Next tweak the fixture in our JavaScript test to reflect that ID, as well as the CSRF token that's currently on the page: [role="sourcecode"] .lists/static/tests/tests.html ==== [source,diff] ---- @@ -9,9 +9,14 @@
- + -
Error text
+ +
+
+ Error text +
+
---- ==== And here's our test: [role="sourcecode"] .lists/static/tests/tests.html (ch36l019) ==== [source,javascript] ---- QUnit.test("should intercept form submit and do ajax post", function (assert) { var url = '/listitemsapi/'; window.Superlists.initialize(url); $('#id_item_form input[name="text"]').val('user input'); //<1> $('#id_item_form input[name="csrfmiddlewaretoken"]').val('tokeney'); //<1> $('#id_item_form').submit(); //<1> assert.equal(server.requests.length, 2); //<2> var request = server.requests[1]; assert.equal(request.url, url); assert.equal(request.method, "POST"); assert.equal( request.requestBody, 'text=user+input&csrfmiddlewaretoken=tokeney' //<3> ); }); ---- ==== <1> We simulate the user filling in the form and hitting Submit. <2> We now expect that there should be a second Ajax request (the first one is the GET for the list items table). <3> We check our POST `requestBody`. As you can see, it's URL-encoded, which isn't the most easy value to test, but it's still just about readable. And here's how we implement it: [role="sourcecode"] .lists/static/list.js ==== [source,javascript] ---- [...] $('#id_list_table').html(rows); }); var form = $('#id_item_form'); form.on('submit', function(event) { event.preventDefault(); $.post(url, { 'text': form.find('input[name="text"]').val(), 'csrfmiddlewaretoken': form.find('input[name="csrfmiddlewaretoken"]').val(), }); }); ---- ==== That gets our JavaScript tests passing but it breaks our FTs, because, although we're doing our POST all right, we're not updating the page after the POST to show the new list item: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests.test_simple_list_creation*] [...] AssertionError: '2: Use peacock feathers to make a fly' not found in ['1: Buy peacock feathers'] ---- Mocking in JavaScript ~~~~~~~~~~~~~~~~~~~~~ We want our client side to update the table of items after the Ajax POST completes. Essentially it'll do the same work as we do as soon as the page loads, retrieving the current list of items from the server, and filling in the item table. Sounds like a helper function is in order! [role="sourcecode"] .lists/static/list.js ==== [source,javascript] ---- window.Superlists = {}; window.Superlists.updateItems = function (url) { $.get(url).done(function (response) { var rows = ''; for (var i=0; i'; } $('#id_list_table').html(rows); }); }; window.Superlists.initialize = function (url) { $('input[name="text"]').on('keypress', function () { $('.has-error').hide(); }); if (url) { window.Superlists.updateItems(url); var form = $('#id_item_form'); [...] ---- ==== That was just a refactor; now we check that the JavaScript tests all still pass: [role="qunit-output"] ---- 12 assertions of 12 passed, 0 failed. 1. errors should be hidden on keypress (1) 2. errors aren't hidden if there is no keypress (1) 3. should get items by ajax on initialize (3) 4. should fill in lists table from ajax response (3) 5. should intercept form submit and do ajax post (4) ---- Now how to test that our Ajax POST calls `updateItems` on POST success? We don't want to dumbly duplicate the code that simulates a server response and checks the items table manually...how about a mock? First we set up a thing called a "sandbox". It will keep track of all the mocks we create, and make sure to un-monkeypatch all the things that have been mocked after each test: [role="sourcecode"] .lists/static/tests/tests.html (ch36l023) ==== [source,html] ---- var server, sandbox; QUnit.testStart(function () { server = sinon.fakeServer.create(); sandbox = sinon.sandbox.create(); }); QUnit.testDone(function () { server.restore(); sandbox.restore(); //<1> }); ---- ==== <1> This `.restore()` is the important part; it undoes all the mocking we've done in each test. [role="sourcecode"] .lists/static/tests/tests.html (ch36l024) ==== [source,html] ---- QUnit.test("should call updateItems after successful post", function (assert) { var url = '/listitemsapi/'; window.Superlists.initialize(url); //<1> var response = [ 201, {"Content-Type": "application/json"}, JSON.stringify({}), ]; server.respondWith('POST', url, response); //<1> $('#id_item_form input[name="text"]').val('user input'); $('#id_item_form input[name="csrfmiddlewaretoken"]').val('tokeney'); $('#id_item_form').submit(); sandbox.spy(window.Superlists, 'updateItems'); //<2> server.respond(); //<2> assert.equal( window.Superlists.updateItems.lastCall.args, //<3> url ); }); ---- ==== <1> First important thing to notice: We only set up our server response 'after' we do the initialize. We want this to be the response to the POST request that happens on form submit, not the response to the initial GET request. (Remember our lesson from <>? One of the most challenging things about JavaScript testing is controlling the order of execution.) <2> Similarly, we only start mocking our helper function 'after' we know the first call for the initial GET has already happened. The `sandbox.spy` call is what does the job that `patch` does in Python tests. It replaces the given object with a mock [keep-together]#version#. <3> Our `updateItems` function has now grown some mocky extra attributes, like `lastCall` and `lastCall.args`, which are like the Python mock's `call_args`. To get it passing, we first make a deliberate mistake, to check that our tests really do test what we think they do: [role="sourcecode"] .lists/static/list.js ==== [source,javascript] ---- $.post(url, { 'text': form.find('input[name="text"]').val(), 'csrfmiddlewaretoken': form.find('input[name="csrfmiddlewaretoken"]').val(), }).done(function () { window.Superlists.updateItems(); }); ---- ==== Yep, we're almost there but not quite: [role="qunit-output"] ---- 12 assertions of 13 passed, 1 failed. [...] 6. should call updateItems after successful post (1, 0, 1) 1. failed Expected: "/listitemsapi/" Result: [] Diff: "/listitemsapi/"[] Source: file://...goat-book/lists/static/tests/tests.html:124:15 ---- And we fix it thusly: [role="sourcecode"] .lists/static/list.js ==== [source,javascript] ---- }).done(function () { window.Superlists.updateItems(url); }); ---- ==== And our FT passes! Or at least one of them does. The others have problems, and we'll come back to them shortly. Finishing the Refactor: Getting the Tests to Match the Code ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ First, I'm not happy until we've seen through this refactor, and made our unit tests match the code a little more: //TODO: fix long lines in this listing [role="sourcecode small-code"] .lists/static/tests/tests.html ==== [source,diff] ---- @@ -50,9 +50,19 @@ QUnit.testDone(function () { }); -QUnit.test("should get items by ajax on initialize", function (assert) { +QUnit.test("should call updateItems on initialize", function (assert) { var url = '/getitems/'; + sandbox.spy(window.Superlists, 'updateItems'); window.Superlists.initialize(url); + assert.equal( + window.Superlists.updateItems.lastCall.args, + url + ); +}); + +QUnit.test("updateItems should get correct url by ajax", function (assert) { + var url = '/getitems/'; + window.Superlists.updateItems(url); assert.equal(server.requests.length, 1); var request = server.requests[0]; @@ -60,7 +70,7 @@ QUnit.test("should get items by ajax on initialize", function (assert) { assert.equal(request.method, 'GET'); }); -QUnit.test("should fill in lists table from ajax response", function (assert) { +QUnit.test("updateItems should fill in lists table from ajax response", function (assert) { var url = '/getitems/'; var responseData = [ {'id': 101, 'text': 'item 1 text'}, @@ -69,7 +79,7 @@ QUnit.test("should fill in lists table from ajax response", function [...] server.respondWith('GET', url, [ 200, {"Content-Type": "application/json"}, JSON.stringify(responseData) ]); - window.Superlists.initialize(url); + window.Superlists.updateItems(url); server.respond(); ---- ==== //ch36l026 And that should give us a test run that looks like this instead: [role="qunit-output"] ---- 14 assertions of 14 passed, 0 failed. 1. errors should be hidden on keypress (1) 2. errors aren't hidden if there is no keypress (1) 3. should call updateItems on initialize (1) 4. updateItems should get correct url by ajax (3) 5. updateItems should fill in lists table from ajax response (3) 6. should intercept form submit and do ajax post (4) 7. should call updateItems after successful post (1) ---- [role="pagebreak-before less_space"] Data Validation: An Exercise for the Reader? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you do a full test run, you should find two of the validation FTs are failing: [role="dofirst-ch36l017"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] [...] ERROR: test_cannot_add_duplicate_items (functional_tests.test_list_item_validation.ItemValidationTest) [...] ERROR: test_error_messages_are_cleared_on_input (functional_tests.test_list_item_validation.ItemValidationTest) [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .has-error ---- I won't spell this all out for you, but here's at least the unit tests you'll need: [role="sourcecode dofirst-ch36l028 small-code"] .lists/tests/test_api.py (ch36l027) ==== [source,python] ---- from lists.forms import DUPLICATE_ITEM_ERROR, EMPTY_ITEM_ERROR [...] def post_empty_input(self): list_ = List.objects.create() return self.client.post( self.base_url.format(list_.id), data={'text': ''} ) def test_for_invalid_input_nothing_saved_to_db(self): self.post_empty_input() self.assertEqual(Item.objects.count(), 0) def test_for_invalid_input_returns_error_code(self): response = self.post_empty_input() self.assertEqual(response.status_code, 400) self.assertEqual( json.loads(response.content.decode('utf8')), {'error': EMPTY_ITEM_ERROR} ) def test_duplicate_items_error(self): list_ = List.objects.create() self.client.post( self.base_url.format(list_.id), data={'text': 'thing'} ) response = self.client.post( self.base_url.format(list_.id), data={'text': 'thing'} ) self.assertEqual(response.status_code, 400) self.assertEqual( json.loads(response.content.decode('utf8')), {'error': DUPLICATE_ITEM_ERROR} ) ---- ==== And on the JavaScript side: [role="sourcecode dofirst-ch36l029-1"] .lists/static/tests/tests.html (ch36l029-2) ==== [source,python] ---- QUnit.test("should display errors on post failure", function (assert) { var url = '/listitemsapi/'; window.Superlists.initialize(url); server.respondWith('POST', url, [ 400, {"Content-Type": "application/json"}, JSON.stringify({'error': 'something is amiss'}) ]); $('.has-error').hide(); $('#id_item_form').submit(); server.respond(); // post assert.equal($('.has-error').is(':visible'), true); assert.equal($('.has-error .help-block').text(), 'something is amiss'); }); QUnit.test("should hide errors on post success", function (assert) { [...] ---- ==== You'll also want some modifications to 'base.html' to make it compatible with both displaying Django errors (which the home page still uses for now) and errors from [keep-together]#JavaScript#: [role="sourcecode dofirst-ch36l030"] .lists/templates/base.html (ch36l031) ==== [source,diff] ---- @@ -51,17 +51,21 @@

{% block header_text %}{% endblock %}

+ {% block list_form %}
-
{{ form.text.errors }}
+
+
+ {% if form.errors %} + {{ form.text.errors }} + {% endif %}
- {% endif %} +
{% endblock %} +
---- ==== //ch36l031 By the end you should get to a JavaScript test run a bit like this: [role="qunit-output dofirst-ch36l033"] ---- 20 assertions of 20 passed, 0 failed. 1. errors should be hidden on keypress (1) 2. errors aren't hidden if there is no keypress (1) 3. should call updateItems on initialize (1) 4. updateItems should get correct url by ajax (3) 5. updateItems should fill in lists table from ajax response (3) 6. should intercept form submit and do ajax post (4) 7. should call updateItems after successful post (1) 8. should not intercept form submit if no api url passed in (1) 9. should display errors on post failure (2) 10. should hide errors on post success (1) 11. should display generic error if no error json (2) ---- And a full test run should pass, including all the FTs: //TODO: there's a possible race condition here, line 56 in the test_sharing // sometimes fails because oni tries to add his list before the table has // loaded [role="dofirst-ch36l032"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] [...] Ran 81 tests in 62.029s OK ---- Laaaaaahvely.footnote:[Put on your best cockney accent for this one.] And there's your hand-rolled REST API with Django. If you need a hint finishing it off yourself, check out https://github.com/hjwp/book-example/tree/appendix_rest_api[the repo]. But I would never suggest building a REST API in Django without at least checking out 'Django-Rest-Framework'. Which is the topic of the next appendix! Read on, [keep-together]#Macduff#.((("", startref="RESTbuild32"))) .REST API Tips ******************************************************************************* Dedupe URLs:: ((("Representational State Transfer (REST)", "tips for REST APIs")))URLs are more important, in a way, to an API than they are to a browser-facing app. Try to reduce the amount of times you hardcode them in your tests. Don't work with raw JSON strings:: `json.loads` and `json.dumps` are your friend. Always use an Ajax mocking library for your JavaScript tests:: Sinon is fine. Jasmine has its own, as does Angular. Bear graceful degradation and progressive enhancement in mind:: Especially if you're moving from a static site to a more JavaScript-driven one, consider keeping at least the core of your site's functionality working without JavaScript. ******************************************************************************* ================================================ FILE: appendix_tradeoffs.asciidoc ================================================ [[appendix_tradeoffs]] [appendix] == Testing Tradeoffs: Choosing the Right Place to Test From pick up in chapter 16, refactor form, move html back to template tests were in the wrong place, better to have them in test_views.py mind you, it's also tested in the FT. -> 3 places we could/do test. discuss contract between FE and BE. it's the `name=` basically where do we want to keep the logic about bootstrap css? * show moving tests from test_forms.py to test_views.py * get rid of assertions about `isinstance(... Form)` lesson: tests at higher level enable more refactoring === Deleting some FTs validation test is doing quite a lot of work, tests integration w/ backend and bootstrap, good main todos test is the key smoke test test multiple lists? less wortwhile. test list sharing? maybe not worth it. test login? maybe not adding value === FT speedup techniques `with self.subTest`. === Testing at a Lower Level imagine a new feature, "strict todos": - rule about duplicate items is relaxed for non-strict todos - strict todos must start with capital letter and end with full stop - use linguistic analysis to check for imperative mood (?) => justitify some proper unit tests? at the very least, testing for the regex _could_ happen at a lower level ================================================ FILE: asciidoc.conf ================================================ [replacements] %1%=➊ %2%=➋ %3%=➌ %4%=➍ %5%=➎ %6%=➏ ================================================ FILE: asciidoctor.css ================================================ /* Asciidoctor default stylesheet | MIT License | http://asciidoctor.org */ /* Remove comment around @import statement below when using as a custom stylesheet */ /*@import "https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400,700";*/ @import url('https://fonts.googleapis.com/css?family=Kalam'); article,aside,details,figcaption,figure,footer,header,hgroup,main,nav,section,summary{display:block} audio,canvas,video{display:inline-block} audio:not([controls]){display:none;height:0} [hidden],template{display:none} script{display:none!important} html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%} body{margin:0} a{background:transparent} a:focus{outline:thin dotted} a:active,a:hover{outline:0} h1{font-size:2em;margin:.67em 0} abbr[title]{border-bottom:1px dotted} b,strong{font-weight:bold} dfn{font-style:italic} hr{-moz-box-sizing:content-box;box-sizing:content-box;height:0} mark{background:#ff0;color:#000} code,kbd,pre,samp{font-family:monospace;font-size:1em} pre{white-space:pre-wrap} q{quotes:"\201C" "\201D" "\2018" "\2019"} small{font-size:80%} sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline} sup{top:-.5em} sub{bottom:-.25em} img{border:0} svg:not(:root){overflow:hidden} figure{margin:0} fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em} legend{border:0;padding:0} button,input,select,textarea{font-family:inherit;font-size:100%;margin:0} button,input{line-height:normal} button,select{text-transform:none} button,html input[type="button"],input[type="reset"],input[type="submit"]{-webkit-appearance:button;cursor:pointer} button[disabled],html input[disabled]{cursor:default} input[type="checkbox"],input[type="radio"]{box-sizing:border-box;padding:0} input[type="search"]{-webkit-appearance:textfield;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;box-sizing:content-box} input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-decoration{-webkit-appearance:none} button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0} textarea{overflow:auto;vertical-align:top} table{border-collapse:collapse;border-spacing:0} *,*:before,*:after{-moz-box-sizing:border-box;-webkit-box-sizing:border-box;box-sizing:border-box} html,body{font-size:100%} body{background:#fff;color:rgba(0,0,0,.8);padding:0;margin:0;font-family:"Noto Serif","DejaVu Serif",serif;font-weight:400;font-style:normal;line-height:1;position:relative;cursor:auto} a:hover{cursor:pointer} img,object,embed{max-width:100%;height:auto} object,embed{height:100%} img{-ms-interpolation-mode:bicubic} .left{float:left!important} .right{float:right!important} .text-left{text-align:left!important} .text-right{text-align:right!important} .text-center{text-align:center!important} .text-justify{text-align:justify!important} .hide{display:none} body{-webkit-font-smoothing:antialiased} img,object,svg{display:inline-block;vertical-align:middle} textarea{height:auto;min-height:50px} select{width:100%} .center{margin-left:auto;margin-right:auto} .spread{width:100%} p.lead,.paragraph.lead>p,#preamble>.sectionbody>.paragraph:first-of-type p{font-size:1.21875em;line-height:1.6} .subheader,.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{line-height:1.45;color:#7a2518;font-weight:400;margin-top:0;margin-bottom:.25em} div,dl,dt,dd,ul,ol,li,h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6,pre,form,p,blockquote,th,td{margin:0;padding:0;direction:ltr} a{color:#2156a5;text-decoration:underline;line-height:inherit} a:hover,a:focus{color:#1d4b8f} a img{border:none} p{font-family:inherit;font-weight:400;font-size:1em;line-height:1.6;margin-bottom:1.25em;text-rendering:optimizeLegibility} p aside{font-size:.875em;line-height:1.35;font-style:italic} h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{font-family:"Open Sans","DejaVu Sans",sans-serif;font-weight:300;font-style:normal;color:#ba3925;text-rendering:optimizeLegibility;margin-top:1em;margin-bottom:.5em;line-height:1.0125em} h1 small,h2 small,h3 small,#toctitle small,.sidebarblock>.content>.title small,h4 small,h5 small,h6 small{font-size:60%;color:#e99b8f;line-height:0} h1{font-size:2.125em} h2{font-size:1.6875em} h3,#toctitle,.sidebarblock>.content>.title{font-size:1.375em} h4,h5{font-size:1.125em} h6{font-size:1em} hr{border:solid #ddddd8;border-width:1px 0 0;clear:both;margin:1.25em 0 1.1875em;height:0} em,i{font-style:italic;line-height:inherit} strong,b{font-weight:bold;line-height:inherit} small{font-size:60%;line-height:inherit} code{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;color:rgba(0,0,0,.9)} ul,ol,dl{font-size:1em;line-height:1.6;margin-bottom:1.25em;list-style-position:outside;font-family:inherit} ul,ol,ul.no-bullet,ol.no-bullet{margin-left:1.5em} ul li ul,ul li ol{margin-left:1.25em;margin-bottom:0;font-size:1em} ul.square li ul,ul.circle li ul,ul.disc li ul{list-style:inherit} ul.square{list-style-type:square} ul.circle{list-style-type:circle} ul.disc{list-style-type:disc} ul.no-bullet{list-style:none} ol li ul,ol li ol{margin-left:1.25em;margin-bottom:0} dl dt{margin-bottom:.3125em;font-weight:bold} dl dd{margin-bottom:1.25em} abbr,acronym{text-transform:uppercase;font-size:90%;color:rgba(0,0,0,.8);border-bottom:1px dotted #ddd;cursor:help} abbr{text-transform:none} blockquote{margin:0 0 1.25em;padding:.5625em 1.25em 0 1.1875em;border-left:1px solid #ddd} blockquote cite{display:block;font-size:.9375em;color:rgba(0,0,0,.6)} blockquote cite:before{content:"\2014 \0020"} blockquote cite a,blockquote cite a:visited{color:rgba(0,0,0,.6)} blockquote,blockquote p{line-height:1.6;color:rgba(0,0,0,.85)} @media only screen and (min-width:768px){h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2} h1{font-size:2.75em} h2{font-size:2.3125em} h3,#toctitle,.sidebarblock>.content>.title{font-size:1.6875em} h4{font-size:1.4375em}} table{background:#fff;margin-bottom:1.25em;border:solid 1px #dedede} table thead,table tfoot{background:#f7f8f7;font-weight:bold} table thead tr th,table thead tr td,table tfoot tr th,table tfoot tr td{padding:.5em .625em .625em;font-size:inherit;color:rgba(0,0,0,.8);text-align:left} table tr th,table tr td{padding:.5625em .625em;font-size:inherit;color:rgba(0,0,0,.8)} table tr.even,table tr.alt,table tr:nth-of-type(even){background:#f8f8f7} table thead tr th,table tfoot tr th,table tbody tr td,table tr td,table tfoot tr td{display:table-cell;line-height:1.6} body{tab-size:4} h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2;word-spacing:-.05em} h1 strong,h2 strong,h3 strong,#toctitle strong,.sidebarblock>.content>.title strong,h4 strong,h5 strong,h6 strong{font-weight:400} .clearfix:before,.clearfix:after,.float-group:before,.float-group:after{content:" ";display:table} .clearfix:after,.float-group:after{clear:both} *:not(pre)>code{font-size:.9375em;font-style:normal!important;letter-spacing:0;padding:.1em .5ex;word-spacing:-.15em;background-color:#f7f7f8;-webkit-border-radius:4px;border-radius:4px;line-height:1.45;text-rendering:optimizeSpeed} pre,pre>code{line-height:1.45;color:rgba(0,0,0,.9);font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;text-rendering:optimizeSpeed} .keyseq{color:rgba(51,51,51,.8)} kbd{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;display:inline-block;color:rgba(0,0,0,.8);font-size:.65em;line-height:1.45;background-color:#f7f7f7;border:1px solid #ccc;-webkit-border-radius:3px;border-radius:3px;-webkit-box-shadow:0 1px 0 rgba(0,0,0,.2),0 0 0 .1em white inset;box-shadow:0 1px 0 rgba(0,0,0,.2),0 0 0 .1em #fff inset;margin:0 .15em;padding:.2em .5em;vertical-align:middle;position:relative;top:-.1em;white-space:nowrap} .keyseq kbd:first-child{margin-left:0} .keyseq kbd:last-child{margin-right:0} .menuseq,.menu{color:rgba(0,0,0,.8)} b.button:before,b.button:after{position:relative;top:-1px;font-weight:400} b.button:before{content:"[";padding:0 3px 0 2px} b.button:after{content:"]";padding:0 2px 0 3px} p a>code:hover{color:rgba(0,0,0,.9)} #header,#content,#footnotes,#footer{width:100%;margin-left:auto;margin-right:auto;margin-top:0;margin-bottom:0;max-width:62.5em;*zoom:1;position:relative;padding-left:.9375em;padding-right:.9375em} #header:before,#header:after,#content:before,#content:after,#footnotes:before,#footnotes:after,#footer:before,#footer:after{content:" ";display:table} #header:after,#content:after,#footnotes:after,#footer:after{clear:both} #content{margin-top:1.25em} #content:before{content:none} #header>h1:first-child{color:rgba(0,0,0,.85);margin-top:2.25rem;margin-bottom:0} #header>h1:first-child+#toc{margin-top:8px;border-top:1px solid #ddddd8} #header>h1:only-child,body.toc2 #header>h1:nth-last-child(2){border-bottom:1px solid #ddddd8;padding-bottom:8px} #header .details{border-bottom:1px solid #ddddd8;line-height:1.45;padding-top:.25em;padding-bottom:.25em;padding-left:.25em;color:rgba(0,0,0,.6);display:-ms-flexbox;display:-webkit-flex;display:flex;-ms-flex-flow:row wrap;-webkit-flex-flow:row wrap;flex-flow:row wrap} #header .details span:first-child{margin-left:-.125em} #header .details span.email a{color:rgba(0,0,0,.85)} #header .details br{display:none} #header .details br+span:before{content:"\00a0\2013\00a0"} #header .details br+span.author:before{content:"\00a0\22c5\00a0";color:rgba(0,0,0,.85)} #header .details br+span#revremark:before{content:"\00a0|\00a0"} #header #revnumber{text-transform:capitalize} #header #revnumber:after{content:"\00a0"} #content>h1:first-child:not([class]){color:rgba(0,0,0,.85);border-bottom:1px solid #ddddd8;padding-bottom:8px;margin-top:0;padding-top:1rem;margin-bottom:1.25rem} #toc{border-bottom:1px solid #efefed;padding-bottom:.5em} #toc>ul{margin-left:.125em} #toc ul.sectlevel0>li>a{font-style:italic} #toc ul.sectlevel0 ul.sectlevel1{margin:.5em 0} #toc ul{font-family:"Open Sans","DejaVu Sans",sans-serif;list-style-type:none} #toc li{line-height:1.3334;margin-top:.3334em} #toc a{text-decoration:none} #toc a:active{text-decoration:underline} #toctitle{color:#7a2518;font-size:1.2em} @media only screen and (min-width:768px){#toctitle{font-size:1.375em} body.toc2{padding-left:15em;padding-right:0} #toc.toc2{margin-top:0!important;background-color:#f8f8f7;position:fixed;width:15em;left:0;top:0;border-right:1px solid #efefed;border-top-width:0!important;border-bottom-width:0!important;z-index:1000;padding:1.25em 1em;height:100%;overflow:auto} #toc.toc2 #toctitle{margin-top:0;margin-bottom:.8rem;font-size:1.2em} #toc.toc2>ul{font-size:.9em;margin-bottom:0} #toc.toc2 ul ul{margin-left:0;padding-left:1em} #toc.toc2 ul.sectlevel0 ul.sectlevel1{padding-left:0;margin-top:.5em;margin-bottom:.5em} body.toc2.toc-right{padding-left:0;padding-right:15em} body.toc2.toc-right #toc.toc2{border-right-width:0;border-left:1px solid #efefed;left:auto;right:0}} @media only screen and (min-width:1280px){body.toc2{padding-left:20em;padding-right:0} #toc.toc2{width:20em} #toc.toc2 #toctitle{font-size:1.375em} #toc.toc2>ul{font-size:.95em} #toc.toc2 ul ul{padding-left:1.25em} body.toc2.toc-right{padding-left:0;padding-right:20em}} #content #toc{border-style:solid;border-width:1px;border-color:#e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;-webkit-border-radius:4px;border-radius:4px} #content #toc>:first-child{margin-top:0} #content #toc>:last-child{margin-bottom:0} #footer{max-width:100%;background-color:rgba(0,0,0,.8);padding:1.25em} #footer-text{color:rgba(255,255,255,.8);line-height:1.44} /* harry addition march 2025 for better contrast (a11y) in footer hyperlink text */ #footer-text a {color: lightblue} .sect1{padding-bottom:.625em} @media only screen and (min-width:768px){.sect1{padding-bottom:1.25em}} .sect1+.sect1{border-top:1px solid #efefed} #content h1>a.anchor,h2>a.anchor,h3>a.anchor,#toctitle>a.anchor,.sidebarblock>.content>.title>a.anchor,h4>a.anchor,h5>a.anchor,h6>a.anchor{position:absolute;z-index:1001;width:1.5ex;margin-left:-1.5ex;display:block;text-decoration:none!important;visibility:hidden;text-align:center;font-weight:400} #content h1>a.anchor:before,h2>a.anchor:before,h3>a.anchor:before,#toctitle>a.anchor:before,.sidebarblock>.content>.title>a.anchor:before,h4>a.anchor:before,h5>a.anchor:before,h6>a.anchor:before{content:"\00A7";font-size:.85em;display:block;padding-top:.1em} #content h1:hover>a.anchor,#content h1>a.anchor:hover,h2:hover>a.anchor,h2>a.anchor:hover,h3:hover>a.anchor,#toctitle:hover>a.anchor,.sidebarblock>.content>.title:hover>a.anchor,h3>a.anchor:hover,#toctitle>a.anchor:hover,.sidebarblock>.content>.title>a.anchor:hover,h4:hover>a.anchor,h4>a.anchor:hover,h5:hover>a.anchor,h5>a.anchor:hover,h6:hover>a.anchor,h6>a.anchor:hover{visibility:visible} #content h1>a.link,h2>a.link,h3>a.link,#toctitle>a.link,.sidebarblock>.content>.title>a.link,h4>a.link,h5>a.link,h6>a.link{color:#ba3925;text-decoration:none} #content h1>a.link:hover,h2>a.link:hover,h3>a.link:hover,#toctitle>a.link:hover,.sidebarblock>.content>.title>a.link:hover,h4>a.link:hover,h5>a.link:hover,h6>a.link:hover{color:#a53221} .audioblock,.imageblock,.literalblock,.listingblock,.stemblock,.videoblock{margin-bottom:1.25em} .admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{text-rendering:optimizeLegibility;text-align:left;font-family:"Noto Serif","DejaVu Serif",serif;font-size:1rem;font-style:italic} table.tableblock>caption.title{white-space:nowrap;overflow:visible;max-width:0} .paragraph.lead>p,#preamble>.sectionbody>.paragraph:first-of-type p{color:rgba(0,0,0,.85)} table.tableblock #preamble>.sectionbody>.paragraph:first-of-type p{font-size:inherit} .admonitionblock>table{border-collapse:separate;border:0;background:none;width:100%} .admonitionblock>table td.icon{text-align:center;width:80px} .admonitionblock>table td.icon img{max-width:none} .admonitionblock>table td.icon .title{font-weight:bold;font-family:"Open Sans","DejaVu Sans",sans-serif;text-transform:uppercase} .admonitionblock>table td.content{padding-left:1.125em;padding-right:1.25em;border-left:1px solid #ddddd8;color:rgba(0,0,0,.6)} .admonitionblock>table td.content>:last-child>:last-child{margin-bottom:0} .exampleblock>.content{ /* harry modif for new exampleblock source code listings */ /* border-style:solid; border-width:1px; border-color:#e6e6e6; */ margin-bottom:1.25em; /* padding:1.25em; */ background:#fff; -webkit-border-radius:4px; border-radius:4px } .exampleblock>.content>:first-child{margin-top:0} .exampleblock>.content>:last-child{margin-bottom:0} .sidebarblock{border-style:solid;border-width:1px;border-color:#e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;-webkit-border-radius:4px;border-radius:4px} .sidebarblock>:first-child{margin-top:0} .sidebarblock>:last-child{margin-bottom:0} .sidebarblock>.content>.title{color:#7a2518;margin-top:0;text-align:center} .exampleblock>.content>:last-child>:last-child,.exampleblock>.content .olist>ol>li:last-child>:last-child,.exampleblock>.content .ulist>ul>li:last-child>:last-child,.exampleblock>.content .qlist>ol>li:last-child>:last-child,.sidebarblock>.content>:last-child>:last-child,.sidebarblock>.content .olist>ol>li:last-child>:last-child,.sidebarblock>.content .ulist>ul>li:last-child>:last-child,.sidebarblock>.content .qlist>ol>li:last-child>:last-child{margin-bottom:0} .literalblock pre,.listingblock pre:not(.highlight),.listingblock pre[class="highlight"],.listingblock pre[class^="highlight "],.listingblock pre.CodeRay,.listingblock pre.prettyprint{background:#f7f7f8} .sidebarblock .literalblock pre,.sidebarblock .listingblock pre:not(.highlight),.sidebarblock .listingblock pre[class="highlight"],.sidebarblock .listingblock pre[class^="highlight "],.sidebarblock .listingblock pre.CodeRay,.sidebarblock .listingblock pre.prettyprint{background:#f2f1f1} div.sidebarblock.scratchpad:before { content: " "; background-image: url('images/papertop.png'); position: relative; background-repeat: no-repeat; background-size: 4in auto; display: block; height: 0.386in; margin-bottom: 0; padding-bottom: 0; width: 4in; margin-left: -25px; top: -0.386in; margin-top: 0.386in; } div.sidebarblock.scratchpad { background-image: url('images/papermiddle.png'); background-position: left top; background-repeat: repeat-y; background-size: 4in auto; padding-top: 0; padding-bottom: 0; padding-left: 25px; width: 4in; page-break-inside: avoid; border: none; font-family: 'Kalam', cursive; } div.sidebarblock.scratchpad:after { content: " "; background-image: url('images/paperbottom.png'); background-repeat: no-repeat; background-size: 4in auto; display: block; height: 0.377in; margin-top: 0; width: 4in; margin-left: -25px; margin-top: 3pt; } .literalblock pre,.literalblock pre[class],.listingblock pre,.listingblock pre[class]{-webkit-border-radius:4px;border-radius:4px;word-wrap:break-word;padding:1em;font-size:.8125em} .literalblock pre.nowrap,.literalblock pre[class].nowrap,.listingblock pre.nowrap,.listingblock pre[class].nowrap{overflow-x:auto;white-space:pre;word-wrap:normal} @media only screen and (min-width:768px){.literalblock pre,.literalblock pre[class],.listingblock pre,.listingblock pre[class]{font-size:.90625em}} @media only screen and (min-width:1280px){.literalblock pre,.literalblock pre[class],.listingblock pre,.listingblock pre[class]{font-size:1em}} .literalblock.output pre{color:#f7f7f8;background-color:rgba(0,0,0,.9)} .listingblock pre.highlightjs{padding:0} .listingblock pre.highlightjs>code{padding:1em;-webkit-border-radius:4px;border-radius:4px} .listingblock pre.prettyprint{border-width:0} .listingblock>.content{position:relative} .listingblock code[data-lang]:before{display:none;content:attr(data-lang);position:absolute;font-size:.75em;top:.425rem;right:.5rem;line-height:1;text-transform:uppercase;color:#999} .listingblock:hover code[data-lang]:before{display:block} .listingblock.terminal pre .command:before{content:attr(data-prompt);padding-right:.5em;color:#999} .listingblock.terminal pre .command:not([data-prompt]):before{content:"$"} table.pyhltable{border-collapse:separate;border:0;margin-bottom:0;background:none} table.pyhltable td{vertical-align:top;padding-top:0;padding-bottom:0;line-height:1.45} table.pyhltable td.code{padding-left:.75em;padding-right:0} pre.pygments .lineno,table.pyhltable td:not(.code){color:#999;padding-left:0;padding-right:.5em;border-right:1px solid #ddddd8} pre.pygments .lineno{display:inline-block;margin-right:.25em} table.pyhltable .linenodiv{background:none!important;padding-right:0!important} .quoteblock{margin:0 1em 1.25em 1.5em;display:table} .quoteblock>.title{margin-left:-1.5em;margin-bottom:.75em} .quoteblock blockquote,.quoteblock blockquote p{color:rgba(0,0,0,.85);font-size:1.15rem;line-height:1.75;word-spacing:.1em;letter-spacing:0;font-style:italic;text-align:justify} .quoteblock blockquote{margin:0;padding:0;border:0} .quoteblock blockquote:before{content:"\201c";float:left;font-size:2.75em;font-weight:bold;line-height:.6em;margin-left:-.6em;color:#7a2518;text-shadow:0 1px 2px rgba(0,0,0,.1)} .quoteblock blockquote>.paragraph:last-child p{margin-bottom:0} .quoteblock .attribution{margin-top:.5em;margin-right:.5ex;text-align:right} .quoteblock .quoteblock{margin-left:0;margin-right:0;padding:.5em 0;border-left:3px solid rgba(0,0,0,.6)} .quoteblock .quoteblock blockquote{padding:0 0 0 .75em} .quoteblock .quoteblock blockquote:before{display:none} .verseblock{margin:0 1em 1.25em 1em} .verseblock pre{font-family:"Open Sans","DejaVu Sans",sans;font-size:1.15rem;color:rgba(0,0,0,.85);font-weight:300;text-rendering:optimizeLegibility} .verseblock pre strong{font-weight:400} .verseblock .attribution{margin-top:1.25rem;margin-left:.5ex} .quoteblock .attribution,.verseblock .attribution{font-size:.9375em;line-height:1.45;font-style:italic} .quoteblock .attribution br,.verseblock .attribution br{display:none} .quoteblock .attribution cite,.verseblock .attribution cite{display:block;letter-spacing:-.025em;color:rgba(0,0,0,.6)} .quoteblock.abstract{margin:0 0 1.25em 0;display:block} .quoteblock.abstract blockquote,.quoteblock.abstract blockquote p{text-align:left;word-spacing:0} .quoteblock.abstract blockquote:before,.quoteblock.abstract blockquote p:first-of-type:before{display:none} table.tableblock{max-width:100%;border-collapse:separate} table.tableblock td>.paragraph:last-child p>p:last-child,table.tableblock th>p:last-child,table.tableblock td>p:last-child{margin-bottom:0} table.tableblock,th.tableblock,td.tableblock{border:0 solid #dedede} table.grid-all th.tableblock,table.grid-all td.tableblock{border-width:0 1px 1px 0} table.grid-all tfoot>tr>th.tableblock,table.grid-all tfoot>tr>td.tableblock{border-width:1px 1px 0 0} table.grid-cols th.tableblock,table.grid-cols td.tableblock{border-width:0 1px 0 0} table.grid-all *>tr>.tableblock:last-child,table.grid-cols *>tr>.tableblock:last-child{border-right-width:0} table.grid-rows th.tableblock,table.grid-rows td.tableblock{border-width:0 0 1px 0} table.grid-all tbody>tr:last-child>th.tableblock,table.grid-all tbody>tr:last-child>td.tableblock,table.grid-all thead:last-child>tr>th.tableblock,table.grid-rows tbody>tr:last-child>th.tableblock,table.grid-rows tbody>tr:last-child>td.tableblock,table.grid-rows thead:last-child>tr>th.tableblock{border-bottom-width:0} table.grid-rows tfoot>tr>th.tableblock,table.grid-rows tfoot>tr>td.tableblock{border-width:1px 0 0 0} table.frame-all{border-width:1px} table.frame-sides{border-width:0 1px} table.frame-topbot{border-width:1px 0} th.halign-left,td.halign-left{text-align:left} th.halign-right,td.halign-right{text-align:right} th.halign-center,td.halign-center{text-align:center} th.valign-top,td.valign-top{vertical-align:top} th.valign-bottom,td.valign-bottom{vertical-align:bottom} th.valign-middle,td.valign-middle{vertical-align:middle} table thead th,table tfoot th{font-weight:bold} tbody tr th{display:table-cell;line-height:1.6;background:#f7f8f7} tbody tr th,tbody tr th p,tfoot tr th,tfoot tr th p{color:rgba(0,0,0,.8);font-weight:bold} p.tableblock>code:only-child{background:none;padding:0} p.tableblock{font-size:1em} td>div.verse{white-space:pre} ol{margin-left:1.75em} ul li ol{margin-left:1.5em} dl dd{margin-left:1.125em} dl dd:last-child,dl dd:last-child>:last-child{margin-bottom:0} ol>li p,ul>li p,ul dd,ol dd,.olist .olist,.ulist .ulist,.ulist .olist,.olist .ulist{margin-bottom:.625em} ul.unstyled,ol.unnumbered,ul.checklist,ul.none{list-style-type:none} ul.unstyled,ol.unnumbered,ul.checklist{margin-left:.625em} ul.checklist li>p:first-child>.fa-square-o:first-child,ul.checklist li>p:first-child>.fa-check-square-o:first-child{width:1em;font-size:.85em} ul.checklist li>p:first-child>input[type="checkbox"]:first-child{width:1em;position:relative;top:1px} ul.inline{margin:0 auto .625em auto;margin-left:-1.375em;margin-right:0;padding:0;list-style:none;overflow:hidden} ul.inline>li{list-style:none;float:left;margin-left:1.375em;display:block} ul.inline>li>*{display:block} .unstyled dl dt{font-weight:400;font-style:normal} ol.arabic{list-style-type:decimal} ol.decimal{list-style-type:decimal-leading-zero} ol.loweralpha{list-style-type:lower-alpha} ol.upperalpha{list-style-type:upper-alpha} ol.lowerroman{list-style-type:lower-roman} ol.upperroman{list-style-type:upper-roman} ol.lowergreek{list-style-type:lower-greek} .hdlist>table,.colist>table{border:0;background:none} .hdlist>table>tbody>tr,.colist>table>tbody>tr{background:none} td.hdlist1,td.hdlist2{vertical-align:top;padding:0 .625em} td.hdlist1{font-weight:bold;padding-bottom:1.25em} .literalblock+.colist,.listingblock+.colist{margin-top:-.5em} /* harry vertical-align callout buttons top */ .colist>table tr>td:first-of-type{ padding:0.4em .75em; line-height:1; vertical-align:top; } .colist>table tr>td:last-of-type{padding:.25em 0} .thumb,.th{line-height:0;display:inline-block;border:solid 4px #fff;-webkit-box-shadow:0 0 0 1px #ddd;box-shadow:0 0 0 1px #ddd} .imageblock.left,.imageblock[style*="float: left"]{margin:.25em .625em 1.25em 0} .imageblock.right,.imageblock[style*="float: right"]{margin:.25em 0 1.25em .625em} .imageblock>.title{margin-bottom:0} .imageblock.thumb,.imageblock.th{border-width:6px} .imageblock.thumb>.title,.imageblock.th>.title{padding:0 .125em} .image.left,.image.right{margin-top:.25em;margin-bottom:.25em;display:inline-block;line-height:0} .image.left{margin-right:.625em} .image.right{margin-left:.625em} a.image{text-decoration:none;display:inline-block} a.image object{pointer-events:none} sup.footnote,sup.footnoteref{font-size:.875em;position:static;vertical-align:super} sup.footnote a,sup.footnoteref a{text-decoration:none} sup.footnote a:active,sup.footnoteref a:active{text-decoration:underline} #footnotes{padding-top:.75em;padding-bottom:.75em;margin-bottom:.625em} #footnotes hr{width:20%;min-width:6.25em;margin:-.25em 0 .75em 0;border-width:1px 0 0 0} #footnotes .footnote{padding:0 .375em 0 .225em;line-height:1.3334;font-size:.875em;margin-left:1.2em;text-indent:-1.05em;margin-bottom:.2em} #footnotes .footnote a:first-of-type{font-weight:bold;text-decoration:none} #footnotes .footnote:last-of-type{margin-bottom:0} #content #footnotes{margin-top:-.625em;margin-bottom:0;padding:.75em 0} .gist .file-data>table{border:0;background:#fff;width:100%;margin-bottom:0} .gist .file-data>table td.line-data{width:99%} div.unbreakable{page-break-inside:avoid} .big{font-size:larger} .small{font-size:smaller} .underline{text-decoration:underline} .overline{text-decoration:overline} .line-through{text-decoration:line-through} .aqua{color:#00bfbf} .aqua-background{background-color:#00fafa} .black{color:#000} .black-background{background-color:#000} .blue{color:#0000bf} .blue-background{background-color:#0000fa} .fuchsia{color:#bf00bf} .fuchsia-background{background-color:#fa00fa} .gray{color:#606060} .gray-background{background-color:#7d7d7d} .green{color:#006000} .green-background{background-color:#007d00} .lime{color:#00bf00} .lime-background{background-color:#00fa00} .maroon{color:#600000} .maroon-background{background-color:#7d0000} .navy{color:#000060} .navy-background{background-color:#00007d} .olive{color:#606000} .olive-background{background-color:#7d7d00} .purple{color:#600060} .purple-background{background-color:#7d007d} .red{color:#bf0000} .red-background{background-color:#fa0000} .silver{color:#909090} .silver-background{background-color:#bcbcbc} .teal{color:#006060} .teal-background{background-color:#007d7d} .white{color:#bfbfbf} .white-background{background-color:#fafafa} .yellow{color:#bfbf00} .yellow-background{background-color:#fafa00} span.icon>.fa{cursor:default} .admonitionblock td.icon [class^="fa icon-"]{font-size:2.5em;text-shadow:1px 1px 2px rgba(0,0,0,.5);cursor:default} .admonitionblock td.icon .icon-note:before{content:"\f05a";color:#19407c} .admonitionblock td.icon .icon-tip:before{content:"\f0eb";text-shadow:1px 1px 2px rgba(155,155,0,.8);color:#111} .admonitionblock td.icon .icon-warning:before{content:"\f071";color:#bf6900} .admonitionblock td.icon .icon-caution:before{content:"\f06d";color:#bf3400} .admonitionblock td.icon .icon-important:before{content:"\f06a";color:#bf0000} .conum[data-value]{display:inline-block;color:#fff!important;background-color:rgba(0,0,0,.8);-webkit-border-radius:100px;border-radius:100px;text-align:center;font-size:.75em;width:1.67em;height:1.67em;line-height:1.67em;font-family:"Open Sans","DejaVu Sans",sans-serif;font-style:normal;font-weight:bold} .conum[data-value] *{color:#fff!important} .conum[data-value]+b{display:none} .conum[data-value]:after{content:attr(data-value)} pre .conum[data-value]{position:relative;top:-.125em} b.conum *{color:inherit!important} .conum:not([data-value]):empty{display:none} dt,th.tableblock,td.content,div.footnote{text-rendering:optimizeLegibility} h1,h2,p,td.content,span.alt{letter-spacing:-.01em} p strong,td.content strong,div.footnote strong{letter-spacing:-.005em} p,blockquote,dt,td.content,span.alt{font-size:1.0625rem} p{margin-bottom:1.25rem} .sidebarblock p,.sidebarblock dt,.sidebarblock td.content,p.tableblock{font-size:1em} .exampleblock>.content{background-color:#fffef7;border-color:#e0e0dc;-webkit-box-shadow:0 1px 4px #e0e0dc;box-shadow:0 1px 4px #e0e0dc} .print-only{display:none!important} @media print{@page{margin:1.25cm .75cm} *{-webkit-box-shadow:none!important;box-shadow:none!important;text-shadow:none!important} a{color:inherit!important;text-decoration:underline!important} a.bare,a[href^="#"],a[href^="mailto:"]{text-decoration:none!important} a[href^="http:"]:not(.bare):after,a[href^="https:"]:not(.bare):after{content:"(" attr(href) ")";display:inline-block;font-size:.875em;padding-left:.25em} abbr[title]:after{content:" (" attr(title) ")"} pre,blockquote,tr,img,object,svg{page-break-inside:avoid} thead{display:table-header-group} svg{max-width:100%} p,blockquote,dt,td.content{font-size:1em;orphans:3;widows:3} h2,h3,#toctitle,.sidebarblock>.content>.title{page-break-after:avoid} #toc,.sidebarblock,.exampleblock>.content{background:none!important} #toc{border-bottom:1px solid #ddddd8!important;padding-bottom:0!important} .sect1{padding-bottom:0!important} .sect1+.sect1{border:0!important} #header>h1:first-child{margin-top:1.25rem} body.book #header{text-align:center} body.book #header>h1:first-child{border:0!important;margin:2.5em 0 1em 0} body.book #header .details{border:0!important;display:block;padding:0!important} body.book #header .details span:first-child{margin-left:0!important} body.book #header .details br{display:block} body.book #header .details br+span:before{content:none!important} body.book #toc{border:0!important;text-align:left!important;padding:0!important;margin:0!important} body.book #toc,body.book #preamble,body.book h1.sect0,body.book .sect1>h2{page-break-before:always} .listingblock code[data-lang]:before{display:block} #footer{background:none!important;padding:0 .9375em} #footer-text{color:rgba(0,0,0,.6)!important;font-size:.9em} .hide-on-print{display:none!important} .print-only{display:block!important} .hide-for-print{display:none!important} .show-for-print{display:inherit!important}} ================================================ FILE: atlas.json ================================================ { "branch": "main", "files": [ "cover.html", "praise.html", "titlepage.html", "copyright.html", "toc.html", "preface.asciidoc", "ai_preface.asciidoc", "pre-requisite-installations.asciidoc", "acknowledgments.asciidoc", "part1.asciidoc", "chapter_01.asciidoc", "chapter_02_unittest.asciidoc", "chapter_03_unit_test_first_view.asciidoc", "chapter_04_philosophy_and_refactoring.asciidoc", "chapter_05_post_and_database.asciidoc", "chapter_06_explicit_waits_1.asciidoc", "chapter_07_working_incrementally.asciidoc", "chapter_08_prettification.asciidoc", "part2.asciidoc", "chapter_09_docker.asciidoc", "chapter_10_production_readiness.asciidoc", "chapter_11_server_prep.asciidoc", "chapter_12_ansible.asciidoc", "part3.asciidoc", "chapter_13_organising_test_files.asciidoc", "chapter_14_database_layer_validation.asciidoc", "chapter_15_simple_form.asciidoc", "chapter_16_advanced_forms.asciidoc", "part4.asciidoc", "chapter_17_javascript.asciidoc", "chapter_18_second_deploy.asciidoc", "chapter_19_spiking_custom_auth.asciidoc", "chapter_20_mocking_1.asciidoc", "chapter_21_mocking_2.asciidoc", "chapter_22_fixtures_and_wait_decorator.asciidoc", "chapter_23_debugging_prod.asciidoc", "chapter_24_outside_in.asciidoc", "chapter_25_CI.asciidoc", "chapter_26_page_pattern.asciidoc", "chapter_27_hot_lava.asciidoc", "epilogue.asciidoc", "bibliography.asciidoc", "appendix_IX_cheat_sheet.asciidoc", "appendix_X_what_to_do_next.asciidoc", "appendix_github_links.asciidoc", "ix.html", "author_bio.html", "colo.html" ], "formats": { "pdf": { "version": "print", "toc": true, "index": true, "antennahouse_version": "AHFormatterV71_64-MR2", "syntaxhighlighting": true, "show_comments": false, "color_count": "1", "trim_size": "7inx9.1875in" }, "epub": { "index": true, "toc": true, "epubcheck": true, "syntaxhighlighting": true, "show_comments": false, "downsample_images": false, "mathmlreplacement": false }, "mobi": { "index": true, "toc": true, "syntaxhighlighting": true, "show_comments": false, "downsample_images": false }, "html": { "index": true, "toc": true, "consolidated": false, "syntaxhighlighting": true, "show_comments": false } }, "theme": "oreillymedia/animal_theme_sass", "title": "Test-Driven Development with Python", "print_isbn13": "9781098148713", "templating": false, "lang": "en", "accent_color": "", "preprocessing": "none" } ================================================ FILE: author_bio.html ================================================

About the Author

After an idyllic childhood spent playing with BASIC on French 8-bit computers like the Thomson T07 whose keys go "boop" when you press them, Harry Percival spent a few years being deeply unhappy with economics and management consultancy. Soon, he rediscovered his true geek nature, and was lucky enough to fall in with a bunch of XP fanatics, working on the pioneering but sadly defunct Resolver One spreadsheet. He now works at PythonAnywhere LLP, and spreads the gospel of test-driven development worldwide at talks, workshops, and conferences, with all the passion and enthusiasm of a recent convert.

================================================ FILE: bibliography.asciidoc ================================================ [role="bibliography":"] [appendix] == Bibliography A few books about TDD and software development that I've mentioned in the book, and which I enthusiastically recommend. ++++
  • Kent Beck, Test Driven Development: By Example, Addison-Wesley
  • Martin Fowler, Refactoring, Addison-Wesley
  • Ross Anderson, Security Engineering, Third Edition, Addison-Wesley: https://www.cl.cam.ac.uk/archive/rja14/book.html
  • Steve Freeman and Nat Pryce, Growing Object-Oriented Software Guided by Tests, Addison-Wesley
  • Hal Abelson, Jerry Sussman and Julie Sussman, Structure and Interpretation of Computer Programs (SICP), MIT Press
  • Dave Farley, Modern Software Engineering, Addison-Wesley
++++ ================================================ FILE: book.asciidoc ================================================ :doctype: book :bookseries: pythonbook :source-highlighter: pygments :pygments-style: manni :icons: font = Test-Driven Development with Python :toc: :sectnums!: include::praise.forbook.asciidoc[] include::preface.asciidoc[] include::ai_preface.asciidoc[] include::pre-requisite-installations.asciidoc[] //include::video_plug.asciidoc[] include::acknowledgments.asciidoc[] include::part1.forbook.asciidoc[] :sectnums: include::chapter_01.asciidoc[] include::chapter_02_unittest.asciidoc[] include::chapter_03_unit_test_first_view.asciidoc[] include::chapter_04_philosophy_and_refactoring.asciidoc[] include::chapter_05_post_and_database.asciidoc[] include::chapter_06_explicit_waits_1.asciidoc[] include::chapter_07_working_incrementally.asciidoc[] include::chapter_08_prettification.asciidoc[] include::part2.forbook.asciidoc[] include::chapter_09_docker.asciidoc[] include::chapter_10_production_readiness.asciidoc[] include::chapter_11_server_prep.asciidoc[] include::chapter_12_ansible.asciidoc[] include::part3.forbook.asciidoc[] include::chapter_13_organising_test_files.asciidoc[] include::chapter_14_database_layer_validation.asciidoc[] include::chapter_15_simple_form.asciidoc[] include::chapter_16_advanced_forms.asciidoc[] include::part4.forbook.asciidoc[] include::chapter_17_javascript.asciidoc[] include::chapter_18_second_deploy.asciidoc[] include::chapter_19_spiking_custom_auth.asciidoc[] include::chapter_20_mocking_1.asciidoc[] include::chapter_21_mocking_2.asciidoc[] include::chapter_22_fixtures_and_wait_decorator.asciidoc[] include::chapter_23_debugging_prod.asciidoc[] include::chapter_24_outside_in.asciidoc[] include::chapter_25_CI.asciidoc[] include::chapter_26_page_pattern.asciidoc[] include::chapter_27_hot_lava.asciidoc[] include::epilogue.asciidoc[] include::appendix_fts_for_external_dependencies.asciidoc[] include::appendix_CD.asciidoc[] include::appendix_bdd.asciidoc[] include::appendix_purist_unit_tests.asciidoc[] include::appendix_rest_api.asciidoc[] include::appendix_IX_cheat_sheet.asciidoc[] include::appendix_X_what_to_do_next.asciidoc[] include::appendix_github_links.asciidoc[] include::bibliography.asciidoc[] ================================================ FILE: buy_the_book_banner.html ================================================ ================================================ FILE: chapter_01.asciidoc ================================================ [[chapter_01]] == Getting Django Set Up Using a [keep-together]#Functional Test# Test-driven development isn't something that comes naturally. It's a discipline, like a martial art, and just like in a Kung Fu movie, you need a bad-tempered and unreasonable master to force you to learn the discipline. Ours is the Testing Goat. === Obey the Testing Goat! Do Nothing Until You Have a Test ((("Testing Goat", "defined"))) The Testing Goat is the unofficial mascotfootnote:[ OK more of a minor running joke from PyCon in the mid 2010s, which I am single-handedly trying to make into a thing.] of TDD in the Python testing community. It probably means different things to different people, but, to me, the Testing Goat is a voice inside my head that keeps me on the True Path of Testing--like one of those little angels or demons that pops up by your shoulder in the cartoons, but with a very niche set of concerns. I hope, with this book, to install the Testing Goat inside your head too. So we've decided to build a web app, even if we're not quite sure what it's going to do yet. Normally, the first step in web development is getting your web framework installed and configured. __Download this, install that, configure the other, run the script__...but TDD requires a different mindset. When you're doing TDD, you always have the Testing Goat inside your head--single-minded as goats are--bleating ``Test first, test first!'' In TDD the first step is always the same: _write a test_. _First_ we write the test; _then_ we run it and check that it fails as expected. _Only then_ do we go ahead and build some of our app. Repeat that to yourself in a goat-like voice. I know I do. Another thing about goats is that they take one step at a time. That's why they seldom fall off things, see, no matter how steep they are—as you can see in <>. [[tree_goat]] .Goats are more agile than you think (source: Caitlin Stewart, on Flickr) image::images/tdd3_0101.png["A picture of a goat up a tree", scale="50"] We'll proceed with nice small steps; we're going to use _Django_, which is a popular Python web framework, to build our app. ((("Django framework", "set up", id="DJFsetup01"))) The first thing we want to do is check that we've got Django installed and that it's ready for us to work with. The _way_ we'll check is by confirming that we can spin up Django's development server and actually see it serving up a web page, in our web browser, on our local computer. We'll use the _Selenium_ browser automation tool for this.((("Selenium"))) [[first-FT]] ((("functional tests (FTs)", "creating"))) Create a new Python file called _functional_tests.py_ wherever you want to keep the code for your project, and enter the following code. If you feel like making a few little goat noises as you do it, it may help: [role="sourcecode"] .functional_tests.py ==== [source,python] ---- from selenium import webdriver browser = webdriver.Firefox() browser.get("http://localhost:8000") assert "Congratulations!" in browser.title print("OK") ---- ==== That's our first _functional test_ (FT); I'll talk more about what I mean by functional tests, and how they contrast with unit tests, in a bit. For now, it's enough to assure ourselves that we understand what it's doing: - Starting a Selenium WebDriver to pop up a real Firefox browser window.((("Selenium WebDriver")))((("Firefox"))) - Using it to open up a web page, which we're expecting to be served from the local computer. - Checking (making a test assertion) that the page has the word "Congratulations!" in its title. - If all goes well, we print OK. Let's try running it: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] Traceback (most recent call last): File "...goat-book/functional_tests.py", line 4, in browser.get("http://localhost:8000") File ".../selenium/webdriver/remote/webdriver.py", line 483, in get self.execute(Command.GET, {"url": url}) File ".../selenium/webdriver/remote/webdriver.py", line 458, in execute self.error_handler.check_response(response) File ".../selenium/webdriver/remote/errorhandler.py", line 232, in check_response raise exception_class(message, screen, stacktrace) selenium.common.exceptions.WebDriverException: Message: Reached error page: abo ut:neterror?e=connectionFailure&u=http%3A//localhost%3A8000/[...] Stacktrace: RemoteError@chrome://remote/content/shared/RemoteError.sys.mjs:8:8 WebDriverError@chrome://remote/content/shared/webdriver/Errors.sys.mjs:182:5 UnknownError@chrome://remote/content/shared/webdriver/Errors.sys.mjs:530:5 [...] ---- [role="pagebreak-before"] You should see a browser window pop up trying to open _localhost:8000_, showing the "Unable to connect" error page. If you switch back to your console, you'll see the big, ugly error message telling us that Selenium ran into an error page. And then, you will probably be irritated at the fact that it left the Firefox window lying around your desktop for you to tidy up. We'll fix that later! NOTE: If, instead, you see an error trying to import Selenium, or an error trying to find something called "geckodriver", you might need to go back and have another look at the “<>”. [[firefox_upgrade_popup_aside]] .What to Do If You Get a Firefox Upgrade Pop-up ******************************************************************************* ((("Selenium", "upgrading"))) ((("geckodriver", "upgrading"))) ((("Firefox", "upgrading"))) ((("functional tests (FTs)", "troubleshooting hung tests"))) ((("troubleshooting", "hung functional tests"))) Now and again, when running Selenium tests, you might encounter a strange pop-up window, such as the one shown in <>. [[firefox_upgrade_popup]] .Firefox wants to install a new what now? image::images/tdd3_0102.png["A pop-up window saying 'Firefox is trying to install a new helper tool.' and prompting for a username and password"] This happens when Firefox has automatically downloaded a new version, in the background. When Selenium tries to load a fresh Firefox session, it wants to install the latest version of its "geckodriver" plugin. To resolve the issue, you have to close the Selenium browser window, go back to your main browser window and tell it to install the upgrade and restart itself, and then try again. NOTE: If something strange is going on with your FTs, it's worth checking if there's a Firefox upgrade pending. ******************************************************************************* For now though, we have a _failing test_, so that means we're allowed to start building our app. === Getting Django Up and Running ((("Django framework", "set up", "project creation"))) As you've definitely read “<>” by now, you've already got Django installed (right?). The first step in getting Django up and running is to create a _project_, which will be the main container for our site. Django provides a little command-line tool for this: [subs="specialcharacters,quotes"] ---- $ *django-admin startproject superlists .* ---- //002 Don't forget that "." at the end; it's important! ((("superlists folder"))) That will create a file called _manage.py_ in((("manage.py file"))) your current folder, and a subfolder called “_superlists_”, with more stuff inside it: ---- . ├── functional_tests.py ├── manage.py └── superlists ├── __init__.py ├── asgi.py ├── settings.py ├── urls.py └── wsgi.py ---- NOTE: Make sure your project folder looks exactly like this! If you see two nested folders called "superlists", it's because you forgot the "." in the command. Delete them and try again, or there will be lots of confusion with paths and working directories. The _superlists_ folder is intended for stuff that applies to the whole project--like _settings.py_, which is used to store global configuration information for the site. [role="pagebreak-before"] But the main thing to notice is _manage.py_. That's Django's Swiss Army knife, and one of the things it can do is run((("development server", "running with manage.py")))((("manage.py file", "running a development server"))) a development server. Let's try that now: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py runserver*] Watching for file changes with StatReloader Performing system checks... System check identified no issues (0 silenced). You have 18 unapplied migration(s). Your project may not work properly until you apply the migrations for app(s): admin, auth, contenttypes, sessions. Run 'python manage.py migrate' to apply them. March 17, 2023 - 18:07:30 Django version 5.2.4, using settings 'superlists.settings' Starting development server at http://127.0.0.1:8000/ Quit the server with CONTROL-C. ---- // IDEA: get this under test That's Django's development server now up and running on our machine. NOTE: It's safe to ignore that message about "unapplied migrations" for now. We'll look at migrations in <>. Leave it there and open((("virtualenv (virtual environment)", "activating and using in functional test"))) another command shell. Navigate to your project folder, activate your virtualenv, and then try running our test again: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] OK ---- Not much action on the command line, but you should notice two things: firstly, there was no ugly `AssertionError` and, secondly, the Firefox window that Selenium popped up had a different-looking page on it.((("AssertionError"))) TIP: If you see an error saying "ModuleNotFoundError: No module named selenium", you've forgotten to activate your virtualenv. Check the “<>” section again, if you need to. Well, it may not look like much, but that was our first ever passing test! Hooray! If it all feels a bit too much like magic, like it wasn't quite real, why not go and take a look at the dev server manually, by opening a web browser yourself and visiting pass:[http://localhost:8000]? You should see something like <>. You can quit the development server now if you like, back in the original shell, using Ctrl+C. [[installed_successfully_screenshot]] .It worked! image::images/tdd3_0103.png["Screenshot of Django Installed Successfully Screen"] .Adieu to Roman Numerals! ******************************************************************************* So many introductions to TDD ((("Roman numerals in examples")))use Roman numerals in their examples that it has become a running joke--I even started writing one myself. If you're curious, you can find it on https://github.com/hjwp/tdd-roman-numeral-calculator[my GitHub page]. Roman numerals, as an example, are both good and bad. It's a nice "toy" problem, reasonably limited in scope, and you can explain the core of TDD quite well with it. The problem is that it can be hard to relate to the real world. That's why I've decided to use the building of a real web app, starting from nothing, as my example. Although it's a simple web app, my hope is that it will be easier for you to carry across to your next real project. In addition, it means we can start out using functional tests as well as unit tests, and demonstrate a TDD workflow that's more like real life, and less like that of a toy project. ******************************************************************************* [role="pagebreak-before less_space"] === Starting a Git Repository ((("Git", "starting repositories"))) ((("version control systems (VCSs)", seealso="Git"))) There's one last thing to do before we finish the chapter: start to commit our work to a _version control system_ (VCS). If you're an experienced programmer, you don't need to hear me preaching about version control. But if you're new to it, please believe me when I say that VCS is a must-have. As soon as your project gets to be more than a few weeks old and a few lines of code, having a tool available to look back over old versions of code, revert changes, explore new ideas safely, even just as a backup...It's hard to overstate how useful that is. TDD goes hand in hand with version control, so I want to make sure I impart how it fits into the workflow. .Our Working Directory Is Always the Folder That Contains manage.py ****************************************************************************** We'll be using this same folder throughout the book as our working directory--if in doubt, it's((("manage.py file", "working directory containing"))) the one that contains _manage.py_. (For simplicity, in my command listings, I'll always show it as: _...goat-book/_. Although it will probably actually be something like: _/home/kind-reader-username/my-python-projects/goat-book/_.) Whenever I show a command to type in, I will assume we're in this directory. Similarly, if I mention a path to a file, it will be relative to this directory. So, for example, _superlists/settings.py_ means the _settings.py_ inside the _superlists_ folder. ****************************************************************************** So, our first commit! If anything, it's a bit late; shame on us. We're using _Git_ as our VCS, ’cos it's the best. Let's start by doing the `git init` to start the repository: [subs="specialcharacters,quotes"] ---- $ *ls* db.sqlite3 functional_tests.py manage.py superlists $ *git init .* Initialised empty Git repository in ...goat-book/.git/ ---- [role="pagebreak-before less_space"] .Setting the Default Branch Name in Git ******************************************************************************* If you see this message: [role="skipme small-code"] [subs="specialcharacters,macros"] ---- hint: Using 'master' as the name for the initial branch. This default branch hint: name is subject to change. To configure the initial branch name to use hint: in all of your new repositories, which will suppress this warning, call: hint: hint: git config --global init.defaultBranch hint: hint: Names commonly chosen instead of 'master' are 'main', 'trunk' and hint: 'development'. The just-created branch can be renamed via this command: hint: hint: git branch -m Initialised empty Git repository in ...goat-book/.git/ ---- Consider following the advice and choosing an explicit default branch name.((("default branch name in Git")))((("Git", "default branch name, choosing"))) I chose `main`. It's a popular choice, and you might see it here and there in the book. So if you want to match that, do: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *git config --global init.defaultBranch main* # then let's re-create our git repo by deleting and starting again: $ *rm -rf .git* $ *git init .* Initialised empty Git repository in ...goat-book/.git/ ---- ******************************************************************************* ((("Git", "commits"))) Now let's take a look and see what files we want to commit: [subs="specialcharacters,quotes"] ---- $ *ls* db.sqlite3 functional_tests.py manage.py superlists ---- There are a few things in here that we _don't_ want under version control: _db.sqlite3_ is the database file, and our virtualenv shouldn't be in Git either. We'll add all of (((".gitignore file", primary-sortas="gitignore")))them to a special file called _.gitignore_ which, um, tells Git what to ignore: [subs="specialcharacters,quotes"] ---- $ *echo "db.sqlite3" >> .gitignore* $ *echo ".venv" >> .gitignore* ---- [role="pagebreak-before"] Next we can add the rest of the contents of the current "." folder: [subs="specialcharacters,macros"] ---- $ pass:quotes[*git add .*] $ pass:quotes[*git status*] On branch main No commits yet Changes to be committed: (use "git rm --cached ..." to unstage) new file: .gitignore new file: functional_tests.py new file: manage.py new file: superlists/__init__.py new file: superlists/__pycache__/__init__.cpython-314.pyc new file: superlists/__pycache__/settings.cpython-314.pyc new file: superlists/__pycache__/urls.cpython-314.pyc new file: superlists/__pycache__/wsgi.cpython-314.pyc new file: superlists/asgi.py new file: superlists/settings.py new file: superlists/urls.py new file: superlists/wsgi.py ---- Oops! We've got a bunch of '.pyc' files in there; it's pointless to commit those. Let's remove them from Git and add them to '.gitignore' too: [subs="specialcharacters,macros"] ---- $ pass:[git rm -r --cached superlists/__pycache__] rm 'superlists/__pycache__/__init__.cpython-314.pyc' rm 'superlists/__pycache__/settings.cpython-314.pyc' rm 'superlists/__pycache__/urls.cpython-314.pyc' rm 'superlists/__pycache__/wsgi.cpython-314.pyc' $ pass:[echo "__pycache__" >> .gitignore] $ pass:[echo "*.pyc" >> .gitignore] ---- [role="pagebreak-before"] Now let's see where we are... [subs="specialcharacters,macros"] ---- $ pass:[git status] On branch main Initial commit Changes to be committed: (use "git rm --cached ..." to unstage) new file: .gitignore new file: functional_tests.py new file: manage.py new file: superlists/__init__.py new file: superlists/asgi.py new file: superlists/settings.py new file: superlists/urls.py new file: superlists/wsgi.py Changes not staged for commit: (use "git add ..." to update what will be committed) (use "git restore ..." to discard changes in working directory) modified: .gitignore ---- TIP: You'll see I'm using `git status` a lot--so much so that I often alias it to `git st`...I'm not telling you how to do that though; I leave you to discover the secrets of Git aliases on your own!((("aliases in Git"))) Looking good--we're ready to do our first commit! [subs="specialcharacters,quotes"] ---- $ *git add .gitignore* $ *git commit* ---- [role="pagebreak-before"] When you type `git commit`, it will pop up an editor window for you to write your commit message in. Mine looked like <>.footnote:[ Did a strange terminal-based editor (the dreaded Vim) pop up and you had no idea what to do? Or did you see a message about account identity and `git config --global user.username`? Check out the Git manual and its http://git-scm.com/book/en/Customizing-Git-Git-Configuration[basic configuration section]. PS: To quit Vim, it's Esc, then `:q!`] [[first_git_commit]] .First Git commit image::images/tdd3_0104.png["Screenshot of git commit vi window"] NOTE: If you want to really go to town on Git, this is the time to also learn about how to push your work to a cloud-based VCS hosting service like GitHub or GitLab. They'll be useful if you think you want to follow along with this book on different computers.((("GitHub or GitLab VCS, cloud-based, pushing work to"))) I leave it to you to find out how they work; they have excellent documentation. Alternatively, you can wait until <>, where we'll use one. That's it for the VCS lecture. Congratulations! You've written a functional test using Selenium, and you've gotten Django installed and running, in a certifiable, test-first, goat-approved TDD way. Give yourself a well-deserved pat on the back before moving on to <>.((("", startref="DJFsetup01"))) ================================================ FILE: chapter_02_unittest.asciidoc ================================================ [[chapter_02_unittest]] == Extending Our Functional Test Using [keep-together]#the unittest Module# ((("functional tests (FTs)", "using unittest module", id="FTunittest02"))) ((("unittest module", "basic functional test creation", id="UTMbasic02"))) Let's adapt our test, which currently checks for the default Django "it worked" page, and check instead for some of the things we want to see on the real front page of our site. Time to reveal what kind of web app we're building: a to-do lists site!((("to-do lists website, building"))) I know, I know, every other web dev tutorial online is also a to-do lists app, or maybe a blog or a polls app. I'm very much following fashion. The reason is that a to-do list is a really nice example. At its most basic, it is very simple indeed--just a list of text strings--so it's easy to get a "minimum viable" list app up and running. But it can be extended in all sorts of ways--different persistence models, adding deadlines, reminders, sharing with other users, and improving the client-side UI. There's no reason to be limited to just "to-do" lists either; they could be any kind of lists. But the point is that it should enable me to demonstrate all of the main aspects of web programming, and how you apply TDD to them. [role="pagebreak-before less_space"] === Using a Functional Test to Scope Out a Minimum [keep-together]#Viable App# Tests that use Selenium let us drive a real web browser, so they really let us see how the application _functions_ from the user's point of view.((("minimum viable app, using functional tests as spec"))) That's why they're called _functional tests_. ((("user stories"))) This means that an FT can be a sort of specification for your application. It tends to track what you might call a _user story_, and follows how the user might work with a particular feature and how the app should respond to them.footnote:[ If you want to read more about user stories, check out Gojko Adzic's _Fifty Quick Ideas to Improve Your User Stories_ or Mike Cohn's _User Stories Applied: For Agile Software Development_.] .Terminology: pass:[
]Functional Test == End-to-End Test == Acceptance Test ******************************************************************************************* ((("end-to-end tests", see="functional tests"))) ((("system tests", see="functional tests"))) ((("acceptance tests", seealso="functional tests"))) ((("black box tests", see="functional tests"))) What I call functional tests, some people prefer to call _end-to-end tests_, or, slightly less commonly, _system tests_. The main point is that these kinds of tests look at how the whole application functions, from the outside. Another name is _black box test_, or _closed box test_, because the test doesn't know anything about the internals of the system under test. Others also like the name _acceptance tests_ (see <>). This distinction is less about the level of granularity of the test or the system, but more about whether the test is checking on the “acceptance criteria” for a feature (i.e., _behaviour_), as visible to the user.((("behaviour", "acceptance criteria for")))((("features"))) ******************************************************************************************* Feature tests should have a human-readable story that we can follow. We make it explicit using comments that accompany the test code.((("feature tests"))) When creating a new FT, we can write the comments first, to capture the key points of the user story. Being human-readable, you could even share them with nonprogrammers, as a way of discussing the requirements and features of your app. Test-driven development and Agile or Lean software development methodologies often go together, and one of the things we tend to talk about is the minimum viable app: what is the simplest thing we can build that is still useful? Let's start by building that, so that we can test the water as quickly as possible. A minimum viable to-do list really only needs to let the user enter some to-do items, and remember them for their next visit. [role="pagebreak-before"] Open up _functional_tests.py_ and write a story a bit like this one: [role="sourcecode"] .functional_tests.py (ch02l001) ==== [source,python] ---- from selenium import webdriver browser = webdriver.Firefox() # Edith has heard about a cool new online to-do app. # She goes to check out its homepage browser.get("http://localhost:8000") # She notices the page title and header mention to-do lists assert "To-Do" in browser.title # She is invited to enter a to-do item straight away # She types "Buy peacock feathers" into a text box # (Edith's hobby is tying fly-fishing lures) # When she hits enter, the page updates, and now the page lists # "1: Buy peacock feathers" as an item in a to-do list # There is still a text box inviting her to add another item. # She enters "Use peacock feathers to make a fly" (Edith is very methodical) # The page updates again, and now shows both items on her list # Satisfied, she goes back to sleep browser.quit() ---- ==== .We Have a Word for Comments... ******************************************************************************* When I first started at PythonAnywhere, I used to virtuously pepper my code with nice descriptive comments. My colleagues said to me: "Harry, we have a word for comments.((("comments", "usefulness (or lack of)"))) We call them lies." I was shocked! I learned in school that comments are good practice? They were exaggerating for effect. There is definitely a place for comments that add context and intention. But my colleagues were pointing out that comments aren't always as useful as you hope. For starters, it's pointless to write a comment that just repeats what you're doing with the code: [role="skipme"] [source,python] ---- # increment wibble by 1 wibble += 1 ---- Not only is it pointless, but there's a danger that you'll forget to update the comments when you update the code, and they end up being misleading--lies! The ideal is to strive to make your code so readable, to use such good variable names and function names, and to structure it so well that you no longer need any comments to explain _what_ the code is doing. Just a few here and there to explain _why_. There are other places where comments are very useful. We'll see that Django uses them a lot in the files it generates for us to use as a way of suggesting helpful bits of its API. And, of course, we use comments to explain the user story in our functional tests--by forcing us to make a coherent story out of the test, it makes sure we're always testing from the point of view of the user. There is more fun to be had in this area, things like _Behaviour-Driven Development_ (see https://www.obeythetestinggoat.com/book/appendix_bdd.html[Online Appendix: BDD]) and building domain-specific languages (DSLs) for testing, but they're topics for other books.footnote:[Check out this video by the great Dave Farley if you want a taste: https://oreil.ly/bbawE.] For more on comments, I recommend John Ousterhout's _A Philosophy of Software Design_, which you can get a taste of by reading his https://oreil.ly/1cdgY[lecture notes from the chapter on comments]. ******************************************************************************* You'll notice that, apart from writing the test out as comments, I've updated the `assert` to look for "To-Do" instead of Django's "Congratulations". That means we expect the test to fail now. Let's try running it. First, start up the server: [subs="specialcharacters,quotes"] ---- $ *python manage.py runserver* ---- And then, in another terminal, run the tests: [role="pause-first"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] Traceback (most recent call last): File "...goat-book/functional_tests.py", line 10, in assert "To-Do" in browser.title AssertionError ---- ((("expected failures"))) That's what we call an 'expected fail', which is actually good news--not quite as good as a test that passes, but at least it's failing for the right reason; we can have some confidence we've written the test correctly. [role="pagebreak-before less_space"] === The Python Standard Library's unittest Module There are a couple of little annoyances we should probably deal with.((("unittest module", "contents of"))) Firstly, the message "AssertionError" isn't very helpful--it would be nice if the test told us what it actually found as the browser title. Also, it's left a Firefox window hanging around the desktop, so it would be nice if that got cleared up for us automatically. One option would be to use the second parameter of the `assert` keyword, something like: [role="skipme"] [source,python] ---- assert "To-Do" in browser.title, f"Browser title was {browser.title}" ---- And we could also use `try/finally` to clean up the old Firefox window. But these sorts of problems are quite common in testing, and there are some ready-made [keep-together]#solutions# for us in the standard library's `unittest` module. Let's use that! In [keep-together]#_functional_tests.py_#: [role="sourcecode"] .functional_tests.py (ch02l003) ==== [source,python] ---- import unittest from selenium import webdriver class NewVisitorTest(unittest.TestCase): # <1> def setUp(self): # <3> self.browser = webdriver.Firefox() #<4> def tearDown(self): # <3> self.browser.quit() def test_can_start_a_todo_list(self): # <2> # Edith has heard about a cool new online to-do app. # She goes to check out its homepage self.browser.get("http://localhost:8000") # <4> # She notices the page title and header mention to-do lists self.assertIn("To-Do", self.browser.title) # <5> # She is invited to enter a to-do item straight away self.fail("Finish the test!") # <6> [...] # Satisfied, she goes back to sleep if __name__ == "__main__": # <7> unittest.main() # <7> ---- ==== You'll probably notice a few things here: <1> Tests are organised into classes, which ((("unittest.TestCase class")))((("tests", "organized into classes in unittest")))inherit from `unittest.TestCase`. <2> The main body of the test is in a method called pass:[test_can_start_a_todo_list]. Any method whose name starts with `test_` is a test method, and will be run by the test runner. You can have more than one `test_` method per class. Nice descriptive names for our test methods are a good idea too. <3> `setUp` and `tearDown` are special methods that are run before and after each test. I'm using them to start and stop our browser. They're a bit like `try/finally`, in that `tearDown` will run even if there's an error during the test itself.footnote:[The only exception is that if you have an exception inside `setUp`, then `tearDown` doesn't run.] No more Firefox windows left lying around! <4> `browser`, which was previously a global variable, becomes `self.browser`, an attribute of the test class. This lets us pass it between `setUp`, `tearDown`, and the test method itself. <5> We use `self.assertIn` instead of just `assert` to make our test assertions. [.keep-together]#+unittest+# provides lots of helper functions like this to make test assertions, like `assertEqual`, `assertTrue`, `assertFalse`, and so on.((("assertions", "helper functions in unittest for"))) You can find more in the http://docs.python.org/3/library/unittest.html[`unittest` documentation]. <6> `self.fail` just fails no matter what, producing the error message given. I'm using it as a reminder to finish the test. <7> Finally, we have the `if __name__ == "__main__"` clause. (If you've not seen it before, that's how a Python script checks if it's been executed from the command line, rather than just imported by another script.) We call `unittest.main()`, which launches the `unittest` test runner, which will automatically find test classes and methods in the file and run them. [role="pagebreak-before"] NOTE: If you've read the Django testing documentation, you might have seen something called `LiveServerTestCase`, and are wondering whether we should use it now. Full points to you for reading the friendly manual! `LiveServerTestCase` is a bit too complicated for now, but I promise I'll use it in a later chapter. Let's try out our new and improved FT!footnote:[ Are you unable to move on because you're wondering what those 'ch02l00x' things are, next to some of the code listings? They refer to specific https://github.com/hjwp/book-example/commits/chapter_02_unittest[commits] in the book's example repo. It's all to do with my book's own https://github.com/hjwp/Book-TDD-Web-Dev-Python/tree/main/tests[tests]. You know, the tests for the tests in the book about testing. They have tests of their own, naturally.] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] F ====================================================================== FAIL: test_can_start_a_todo_list (__main__.NewVisitorTest.test_can_start_a_todo_list) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/functional_tests.py", line 18, in test_can_start_a_todo_list self.assertIn("To-Do", self.browser.title) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'To-Do' not found in 'The install worked successfully! Congratulations!' --------------------------------------------------------------------- Ran 1 test in 1.747s FAILED (failures=1) ---- That's a bit nicer, isn't it? It tidied up our Firefox window, it gives us a nicely formatted report of how many tests were run and how many failed, and the `assertIn` has given us a helpful error message with useful debugging info. Bonzer! [role="pagebreak-before"] NOTE: If you see some error messages saying `ResourceWarning` about "unclosed files", it's safe to ignore those. They seem to come and go, every few Selenium releases. They don't affect the important things to look for in our tracebacks and test results. .pytest Versus unittest ******************************************************************************* The Python world is increasingly turning from the standard-library provided [.keep-together]#+unittest+# module towards a third-party tool called `pytest`. I'm a big fan too!((("pytest", "versus unittest")))((("unittest module", "pytest versus"))) The Django project has a bunch of helpful tools designed to work with `unittest`. Although it is possible to get them to work with `pytest`, it felt like one thing too many to include in this book. Read Brian Okken's https://pythontest.com/pytest-book[Python Testing with pytest] for an excellent, comprehensive guide to Pytest instead. ******************************************************************************* === Commit ((("Git", "commits"))) This is a good point to do a commit; it's a nicely self-contained change. We've expanded our functional test to include comments that describe the task we're setting ourselves, our minimum viable to-do list. We've also rewritten it to use the Python `unittest` module and its various testing helper functions. Do a **`git status`**—that should assure you that the only file that has changed is 'functional_tests.py'. Then do a **`git diff -w`**, which shows you the difference between the last commit and what's currently on disk, with the `-w` saying "ignore whitespace changes". [role="pagebreak-before"] That should tell you that 'functional_tests.py' has changed quite substantially: [subs="specialcharacters,macros"] ---- $ pass:quotes[*git diff -w*] diff --git a/functional_tests.py b/functional_tests.py index d333591..b0f22dc 100644 --- a/functional_tests.py +++ b/functional_tests.py @@ -1,15 +1,24 @@ +import unittest from selenium import webdriver -browser = webdriver.Firefox() +class NewVisitorTest(unittest.TestCase): + def setUp(self): + self.browser = webdriver.Firefox() + + def tearDown(self): + self.browser.quit() + + def test_can_start_a_todo_list(self): # Edith has heard about a cool new online to-do app. # She goes to check out its homepage -browser.get("http://localhost:8000") + self.browser.get("http://localhost:8000") # She notices the page title and header mention to-do lists -assert "To-Do" in browser.title + self.assertIn("To-Do", self.browser.title) # She is invited to enter a to-do item straight away + self.fail("Finish the test!") [...] ---- Now let's do a: [subs="specialcharacters,quotes"] ---- $ *git commit -a* ---- The `-a` means "automatically add any changes to tracked files" (i.e., any files that we've committed before). It won't add any brand new files (you have to explicitly `git add` them yourself), but often, as in this case, there aren't any new files, so it's a useful shortcut. When the editor pops up, add a descriptive commit message, like "First FT specced out in comments, and now uses unittest". Now that our FT uses a real test framework, and that we've got placeholder comments for what we want it to do, we're in an excellent position to start writing some real code for our lists app. Read on! ((("", startref="FTunittest02"))) ((("", startref="UTMbasic02"))) .Useful TDD Concepts ******************************************************************************* User story:: A description of how the application will work from the point of view of the user; used to structure a functional test ((("Test-Driven Development (TDD)", "concepts", "user stories"))) ((("user stories"))) Expected failure:: When a test fails in the way that we expected it to ((("Test-Driven Development (TDD)", "concepts", "expected failures"))) ((("expected failures"))) ******************************************************************************* ================================================ FILE: chapter_03_unit_test_first_view.asciidoc ================================================ [[chapter_03_unit_test_first_view]] == Testing a Simple Home Page [keep-together]#with Unit Tests# We finished the last chapter with a functional test (FT) failing, telling us that it wanted the home page for our site to have ``To-Do'' in its title. Time to start working on our application. In this chapter, we'll build our first HTML page, find out about URL handling, and create responses to HTTP requests with Django's view functions. .Warning: Things Are About to Get Real ******************************************************************************* The first two chapters were intentionally nice and light. From now on, we get into some more meaty coding. Here's a prediction: at some point, things are going to go wrong. You're going to see different results from what I say you should see. This is a Good Thing, because it will be a genuine character-building Learning Experience(TM). One possibility is that I've given some ambiguous explanations, and you've done something different from what I intended. Step back and have a think about what we're trying to achieve at this point in the book. Which file are we editing, what do we want the user to be able to do, what are we testing and why? It may be that you've edited the wrong file or function, or are running the wrong tests. I reckon you'll learn more about TDD from these "stop and think" moments than you do from all the times when following instructions and copy-pasting goes smoothly. Or it may be a real bug. Be tenacious, read the error message carefully (see <>), and you'll get to the bottom of it. It's probably just a missing comma, or trailing slash, or a missing _s_ in one of the Selenium find methods. But, as Zed Shaw memorably insisted in https://learnpythonthehardway.org[_Learn Python The Hard Way_], debugging is also an absolutely vital part of learning, so do stick it out!((("Test-Driven Development (TDD)", "additional resources")))((("getting help"))) You can always drop me an mailto:obeythetestinggoat@gmail.com[email] if you get really stuck. Happy debugging! ******************************************************************************* === Our First Django App and Our First Unit Test ((("Django framework", "code structure in"))) ((("Django framework", "unit testing in", id="DJFunit03"))) Django encourages you to structure your code into _apps_. The theory is that one project can have many apps; you can use third-party apps developed by other people, and you might even reuse one of your own apps in a different project...although I have to say, I've never actually managed the latter, myself! Still, apps are a good way to keep your code organised. Let's start an app for our to-do lists: [subs="specialcharacters,quotes"] ---- $ *python manage.py startapp lists* ---- That will create a folder called _lists_, next to _manage.py_ and the existing _superlists_ folder, and within it a number of placeholder files for things like models, views, and, of immediate interest to us, tests: ---- . ├── db.sqlite3 ├── functional_tests.py ├── lists │   ├── __init__.py │   ├── admin.py │   ├── apps.py │   ├── migrations │   │   └── __init__.py │   ├── models.py │   ├── tests.py │   └── views.py ├── manage.py └── superlists ├── __init__.py ├── asgi.py ├── settings.py ├── urls.py └── wsgi.py ---- === Unit Tests, and How They Differ from Functional Tests ((("unit tests", "versus functional tests", secondary-sortas="functional"))) ((("functional tests (FTs)", "versus unit tests", secondary-sortas="unit"))) As with so many of the labels we put on things, the line between unit tests and FTs can become a little blurry at times. The basic distinction, though, is that FTs test the application from the outside, from the user's point of view. Unit tests on the other hand test the application from the inside, from the programmer's point of view. [role="pagebreak-before"] The TDD approach I'm demonstrating uses both types of test to drive the development of our application, and ensure its correctness. Our workflow will look a bit like this: 1. We start by writing a _functional test_, describing a typical example of our new functionality from the user's point of view. 2. Once we have an FT that fails, we start to think about how to write code that can get it to pass (or at least to get past its current failure). We now use one or more _unit tests_ to define how we want our code to behave--the idea is that each line of production code we write should be tested by (at least) one of our unit tests. 3. Once we have a failing unit test, we write the smallest amount of _application code_ we can—just enough to get the unit test to pass. We may iterate between steps 2 and 3 a few times, until we think the FT will get a little further. 4. Now we can rerun our FTs and see if they pass, or get a little further. That may prompt us to write some new unit tests, and some new code, and so on. 5. Once we're comfortable that the core functionality works end-to-end, we can extend out to cover more permutations and edge cases, using just unit tests now. You can see that, all the way through, the FTs are driving what development we do from a high level, while the unit tests drive what we do at a low level. The FTs don't aim to cover every single tiny detail of our app's behaviour; they are there to reassure us that everything is wired up correctly. The unit tests are there to exhaustively check all the lower-level details and corner cases. See <>. [[fts_vs_unit_tests_table]] [options="header"] .Functional tests versus unit tests |=== |Functional tests|Unit tests |One test per feature/user story |Many tests per feature |Tests from the user's point of view |Tests the code (i.e., the programmer's point of view) |Can test that the UI "really" works |Tests the internals—individual functions or classes |Provides confidence that everything is wired together correctly and works end-to-end |Can exhaustively check permutations, details, and edge cases |Can warn about problems without telling you exactly what's wrong |Can point at exactly where the problem is |Slow |Fast |=== [role="pagebreak-before"] NOTE: Functional tests should help you build an application that actually works, and guarantee you never accidentally break it. Unit tests should help you to write code that's clean and bug free. Enough theory for now—let's see how it looks in practice. === Unit Testing in Django ((("unit tests", "in Django", "writing basic", secondary-sortas="Django", id="UTdjango03"))) Let's see how to write a unit test for our home page view. Open up the new file at _lists/tests.py_, and you'll see something like this: [role="sourcecode currentcontents"] .lists/tests.py ==== [source,python] ---- from django.test import TestCase # Create your tests here. ---- ==== Django has helpfully suggested we use a special version of `TestCase`, which it provides.((("unittest.TestCase class", "using augmented version of"))) It's an augmented version of the standard `unittest.TestCase`, with some additional Django-specific features, which we'll discover over the next few chapters. You've already seen that the TDD cycle involves starting with a test that fails, then writing code to get it to pass. Well, before we can even get that far, we want to know that the unit test we're writing will definitely be run by our automated test runner, whatever it is. In the case of _functional_tests.py_ we're running it directly, but this file made by Django is a bit more like magic. So, just to make sure, let's make a deliberately silly failing test: [role="sourcecode"] .lists/tests.py (ch03l002) ==== [source,python] ---- from django.test import TestCase class SmokeTest(TestCase): def test_bad_maths(self): self.assertEqual(1 + 1, 3) ---- ==== [role="pagebreak-before"] Now, let's invoke this mysterious Django test runner. As usual, it's a _manage.py_ [keep-together]#command#: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] Creating test database for alias 'default'... Found 1 test(s). System check identified no issues (0 silenced). F ====================================================================== FAIL: test_bad_maths (lists.tests.SmokeTest.test_bad_maths) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/lists/tests.py", line 6, in test_bad_maths self.assertEqual(1 + 1, 3) ~~~~~~~~~~~~~~~~^^^^^^^^^^ AssertionError: 2 != 3 --------------------------------------------------------------------- Ran 1 test in 0.001s FAILED (failures=1) Destroying test database for alias 'default'... ---- Excellent. The machinery seems to be working. This is a good point for a commit: [subs="specialcharacters,quotes"] ---- $ *git status* # should show you lists/ is untracked $ *git add lists* $ *git diff --staged* # will show you the diff that you're about to commit $ *git commit -m "Add app for lists, with deliberately failing unit test"* ---- As you've no doubt guessed, the `-m` flag lets you pass in a commit message at the command line, so you don't need to use an editor. It's up to you to pick the way you like to use the Git command line; I'll just show you the main ones I've seen used. For me, the key rule of VCS hygiene is: _make sure you always review what you're about to commit before you do it_. [[django-mvc]] === Django's MVC, URLs, and View Functions ((("model-view-controller (MVC) pattern")))((("MVC (model-view-controller) pattern"))) Django is structured along a classic _model-view-controller_ (MVC) pattern—well, _broadly_. It definitely does have models, but what Django calls "views" are really controllers, and the view part is actually provided by the templates, but you can see the general idea is there! If you're interested, you can look up the finer points of the discussion https://oreil.ly/fz-ne[in the Django FAQs]. [role="pagebreak-before"] Irrespective of any of that, as with any web server, Django's main job is to decide what to do when a user asks for a particular URL on our site. Django's workflow goes something like this: . An HTTP _request_ comes in for a particular URL. . Django uses some rules to decide which _view_ function should deal with the request (this is referred to as _resolving_ the URL). . The view function processes the request and returns an HTTP _response_. So, we want to test two things: . Can we make this view function return the HTML we need? . Can we tell Django to use this view function when we make a request for the root of the site (``/'')? Let's start with the first. === Unit Testing a View ((("unit tests", "in Django", "unit testing a view", secondary-sortas="Django"))) Open up _lists/tests.py_, and change our silly test to something like this: [role="sourcecode"] .lists/tests.py (ch03l003) ==== [source,python] ---- from django.test import TestCase from django.http import HttpRequest # <1> from lists.views import home_page class HomePageTest(TestCase): def test_home_page_returns_correct_html(self): request = HttpRequest() # <1> response = home_page(request) # <2> html = response.content.decode("utf8") # <3> self.assertIn("To-Do lists", html) # <4> self.assertTrue(html.startswith("")) # <5> self.assertTrue(html.endswith("")) # <5> ---- ==== [role="pagebreak-before"] What's going on in this new test? Well, remember, a view function takes an HTTP request as input, and produces an HTTP response. So, to test that: <1> We import the `HttpRequest` class so that we can then create a request object within our test. This is the kind of object that Django will create when a user's browser asks for a page. <2> We pass the `HttpRequest` object to our `home_page` view, which gives us a response. You won't be surprised to hear that the response is an instance of a class called `HttpResponse`. <3> Then, we extract the `.content` of the response. These are the raw bytes, the ones and zeros that would be sent down the wire to the user's browser. We call `.decode()` to convert them into the string of HTML that's being sent to the user. <4> Now we can make some assertions: we know we want an HTML `` tag somewhere in there, with the words "To-Do lists" in it--because that's what we specified in our FT. <5> And we can do a vague sense-check that it's valid HTML by checking that it starts with an `<html>` tag, which gets closed at the end. So, what do you think will happen when we run the tests? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] Found 1 test(s). System check identified no issues (0 silenced). E ====================================================================== ERROR: lists.tests (unittest.loader._FailedTest.lists.tests) --------------------------------------------------------------------- ImportError: Failed to import test module: lists.tests Traceback (most recent call last): [...] File "...goat-book/lists/tests.py", line 3, in <module> from lists.views import home_page ImportError: cannot import name 'home_page' from 'lists.views' ---- It's a very predictable and uninteresting error: we tried to import something we haven't even written yet. But it's still good news--for the purposes of TDD, an exception that was predicted counts as an expected failure. Because we have both a failing FT and a failing unit test, we have the Testing Goat's full blessing to code away. ==== At Last! We Actually Write Some Application Code! It is exciting, isn't it? Be warned, TDD means that long periods of anticipation are only defused very gradually, and by tiny increments. Especially as we're learning and only just starting out, we only allow ourselves to change (or add) one line of code at a time—and each time, we make just the minimal change required to address the current test failure. I'm being deliberately extreme here, but what's our current test failure? We can't import `home_page` from `lists.views`? OK, let's fix that--and only that. In _lists/views.py_: [role="sourcecode"] .lists/views.py (ch03l004) ==== [source,python] ---- from django.shortcuts import render # Create your views here. home_page = None ---- ==== "You must be joking!" I can hear you say. I can hear you because it's what I used to say (with feeling) when my colleagues first demonstrated TDD to me. Well, bear with me, and we'll talk about whether or not this is all taking it too far in a little while. But for now, let yourself follow along, even if it's with some exasperation, and see if our tests can help us write the correct code, one tiny step at a time. Let's run the tests again: ---- [...] File "...goat-book/lists/tests.py", line 9, in test_home_page_returns_correct_html response = home_page(request) TypeError: 'NoneType' object is not callable ---- We still get an error, but it's moved on a bit. Instead of an import error, our tests are telling us that our `home_page` "function" is not callable. That gives us a justification for changing it from being `None` to being an actual function. At the very smallest level of detail, every single code change can be driven by the tests! Back in _lists/views.py_: [role="sourcecode"] .lists/views.py (ch03l005) ==== [source,python] ---- from django.shortcuts import render def home_page(): pass ---- ==== Again, we're making the smallest, simplest change we can possibly make, that addresses precisely the current test failure. Our tests wanted something callable, so we gave them the simplest possible callable thing: a function that takes no arguments and returns nothing. Let's run the tests again and see what they think: ---- response = home_page(request) TypeError: home_page() takes 0 positional arguments but 1 was given ---- Once more, our error message has changed slightly, and is guiding us towards fixing the next thing that's wrong. The Unit-Test/Code Cycle ^^^^^^^^^^^^^^^^^^^^^^^^ ((("unit tests", "in Django", "unit-test/code cycle", secondary-sortas="Django"))) ((("unit-test/code cycle"))) ((("Test-Driven Development (TDD)", "concepts", "unit-test/code cycle"))) We can start to settle into the TDD _unit-test/code cycle_ now: 1. In the terminal, run the unit tests and see how they fail. 2. In the editor, make a minimal code change to address the current test failure. And repeat! The more nervous we are about getting our code right, the smaller and more minimal we make each code change--the idea is to be absolutely sure that each bit of code is justified by a test. This may seem laborious—and at first, it will be. But once you get into the swing of things, you'll find yourself coding quickly even if you take microscopic steps--this is how we write all of our production code at work. Let's see how fast we can get this cycle going: [role="simplelist"] * Minimal code change: + [role="sourcecode"] .lists/views.py (ch03l006) ==== [source,python] ---- def home_page(request): pass ---- ==== * Tests: + ---- html = response.content.decode("utf8") ^^^^^^^^^^^^^^^^ AttributeError: 'NoneType' object has no attribute 'content' ---- [role="pagebreak-before simplelist"] * Code--we use `django.http.HttpResponse`, as predicted: + [role="sourcecode"] .lists/views.py (ch03l007) ==== [source,python] ---- from django.http import HttpResponse def home_page(request): return HttpResponse() ---- ==== * Tests again: + ---- AssertionError: '<title>To-Do lists' not found in '' ---- * Code again: + [role="sourcecode"] .lists/views.py (ch03l008) ==== [source,python] ---- def home_page(request): return HttpResponse("To-Do lists") ---- ==== * Tests yet again: + ---- self.assertTrue(html.startswith("")) AssertionError: False is not true ---- * Code yet again: + [role="sourcecode"] .lists/views.py (ch03l009) ==== [source,python] ---- def home_page(request): return HttpResponse("To-Do lists") ---- ==== * Tests--almost there? + ---- self.assertTrue(html.endswith("")) AssertionError: False is not true ---- * Come on, one last effort: + [role="sourcecode"] .lists/views.py (ch03l010) ==== [source,python] ---- def home_page(request): return HttpResponse("To-Do lists") ---- ==== [role="pagebreak-before simplelist"] * Surely? + [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] Creating test database for alias 'default'... Found 1 test(s). System check identified no issues (0 silenced). . --------------------------------------------------------------------- Ran 1 test in 0.001s OK Destroying test database for alias 'default'... ---- Hooray! Our first ever unit test pass! That's so momentous that I think it's worthy of a commit: [subs="specialcharacters,quotes"] ---- $ *git diff* # should show changes to tests.py, and views.py $ *git commit -am "First unit test and view function"* ---- That was the last variation on `git commit` I'll show, the `a` and `m` flags together, which adds all changes to tracked files and uses the commit message from the command line.footnote:[ I'm quite casual about my commit messages in this book, but in professional organisations or open source projects, people often want to be a bit more formal. Check out https://cbea.ms/git-commit and https://www.conventionalcommits.org.] WARNING: `git commit -am` is the quickest formulation, but also gives you the least feedback about what's being committed, so make sure you've done a `git status` and a `git diff` beforehand, and are clear on what changes are about to go in. [role="pagebreak-before less_space"] === Our Functional Tests Tell Us We're Not Quite Done Yet We've got our unit test passing, so let's go back to running our FTs to see if we've made progress. Don't forget to spin up the dev server again, if it's not still running. [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] F ====================================================================== FAIL: test_can_start_a_todo_list (__main__.NewVisitorTest.test_can_start_a_todo_list) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/functional_tests.py", line 18, in test_can_start_a_todo_list self.assertIn("To-Do", self.browser.title) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'To-Do' not found in 'The install worked successfully! Congratulations!' --------------------------------------------------------------------- Ran 1 test in 1.609s FAILED (failures=1) ---- Looks like something isn't quite right. This is the reason we have functional tests! Do you remember at the beginning of the chapter, we said we needed to do two things: firstly, create a view function to produce responses for requests, and secondly, tell the server which functions should respond to which URLs? Thanks to our FT, we have been reminded that we still need to do the second thing. ((("Django framework", "Test Client", id="DJFtestclient04"))) ((("Test Client (Django)", id="testclient04"))) How can we write a test for URL resolution? At the moment, we just test the view function directly by importing it and calling it. But we want to test more layers of the Django stack. Django, like most web frameworks, supplies a tool for doing just that, called the https://docs.djangoproject.com/en/5.2/topics/testing/tools/#the-test-client[Django test client]. [role="pagebreak-before"] Let's see how to use it by adding a second, alternative test to our unit tests: [role="sourcecode"] .lists/tests.py (ch03l011) ==== [source,python] ---- class HomePageTest(TestCase): def test_home_page_returns_correct_html(self): <1> request = HttpRequest() response = home_page(request) html = response.content.decode("utf8") self.assertIn("To-Do lists", html) self.assertTrue(html.startswith("")) self.assertTrue(html.endswith("")) def test_home_page_returns_correct_html_2(self): response = self.client.get("/") # <2> self.assertContains(response, "To-Do lists") # <3> ---- ==== <1> This is our existing test. <2> In our new test, we access the test client via `self.client`, which is available on any test that uses `django.test.TestCase`. It provides methods like `.get()`, which simulates a browser making HTTP requests, and takes a URL as its first parameter. We use this instead of manually creating a request object and calling the view function directly. <3> Django also provides some assertion helpers like `assertContains`, which save us from having to manually extract and decode response content, and have some other nice properties besides, as we'll see. [role="pagebreak-before"] Let's see how that works: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] Found 2 test(s). Creating test database for alias 'default'... System check identified no issues (0 silenced). .F ====================================================================== FAIL: test_home_page_returns_correct_html_2 (lists.tests.HomePageTest.test_home_page_returns_correct_html_2) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/lists/tests.py", line 17, in test_home_page_returns_correct_html_2 self.assertContains(response, "To-Do lists") [...] AssertionError: 404 != 200 : Couldn't retrieve content: Response code was 404 (expected 200) --------------------------------------------------------------------- Ran 2 tests in 0.004s FAILED (failures=1) Destroying test database for alias 'default'... ---- Hmm, something about 404s? Let's dig into it. [[reading_tracebacks]] === Reading Tracebacks ((("tracebacks"))) Let's spend a moment talking about how to read tracebacks, as it's something we have to do a lot in TDD. You soon learn to scan through them and pick up relevant clues: ---- ====================================================================== FAIL: test_home_page_returns_correct_html_2 <2> (lists.tests.HomePageTest.test_home_page_returns_correct_html_2) <2> --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/lists/tests.py", line 17, in test_home_page_returns_correct_html_2 self.assertContains(response, "To-Do lists") <3> ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ <4> AssertionError: 404 != 200 : Couldn't retrieve content: Response code was 404 <1> (expected 200) --------------------------------------------------------------------- [...] ---- [role="pagebreak-before"] <1> The first place you look is usually _the error itself_. Sometimes that's all you need to see, and it will let you identify the problem immediately. But sometimes, like in this case, it's not quite self-evident. <2> The next thing to double-check is: _which test is failing?_ Is it definitely the one we expected--that is, the one we just wrote? In this case, the answer is yes. <3> Then we look for the place in _our test code_ that kicked off the failure. We work our way down from the top of the traceback, looking for the filename of the tests file to check which test function, and what line of code, the failure is coming from. In this case, it's the line where we call the `assertContains` method. <4> In Python 3.11 and later, you can also look out for the string of carets, which try to tell you exactly where the exception came from. This is more useful for unexpected exceptions than for assertion failures like we have now. There is ordinarily a fifth step, where we look further down for any of _our own application code_ that was involved with the problem. In this case, it's all Django code, but we'll see plenty of examples of this fifth step later in the book. Pulling it all together, we interpret the traceback as telling us that: * When we tried to do our assertion on the content of the response. * Django's test helpers failed, saying that they could not do that. * Because the response is an HTML 404 Not Found error, instead of a normal 200 OK response. In other words, Django isn't yet configured to respond to requests for the root URL ("/") of our site. Let's make that happen now. [role="pagebreak-before less_space"] === urls.py ((("URL mappings"))) Django uses a file called _urls.py_ to map URLs to view functions. This mapping is also called _routing_.((("routing", seealso="URL mappings"))) There's a main _urls.py_ for the whole site in the _superlists_ folder. Let's go take a look: [role="sourcecode currentcontents"] .superlists/urls.py ==== [source,python] ---- """ URL configuration for superlists project. The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/5.2/topics/http/urls/ Examples: Function views 1. Add an import: from my_app import views 2. Add a URL to urlpatterns: path('', views.home, name='home') Class-based views 1. Add an import: from other_app.views import Home 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') Including another URLconf 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin from django.urls import path urlpatterns = [ path("admin/", admin.site.urls), ] ---- ==== As usual, lots of helpful comments and default suggestions from Django. In fact, that very first example is pretty much exactly what we want! Let's use that, with some minor changes: [role="sourcecode"] .superlists/urls.py (ch03l012) ==== [source,python] ---- from django.urls import path # <1> from lists.views import home_page # <2> urlpatterns = [ path("", home_page, name="home"), # <3> ] ---- ==== <1> No need to import `admin` from `django.contrib`. Django's admin site is amazing, but it's a topic for another book. <2> But we will import our home page view function. <3> And we wire it up here, as a `path()` entry in the `urlpatterns` global. Django strips the leading slash from all URLs, so `"/url/path/to"` becomes `"url/path/to"` and the base URL is just the empty string, `""`. So this config says, the "base URL should point to our home page view". Now we can run our unit tests again, with *`python manage.py test`*: ---- [...] .. --------------------------------------------------------------------- Ran 2 tests in 0.003s OK ---- Hooray! Time for a little tidy-up. We don't need two separate tests, so let's move everything out of our low-level test that calls the view function directly, into the test that uses the Django test client: [role="sourcecode"] .lists/tests.py (ch03l013) ==== [source,python] ---- class HomePageTest(TestCase): def test_home_page_returns_correct_html(self): response = self.client.get("/") self.assertContains(response, "To-Do lists") self.assertContains(response, "") self.assertContains(response, "") ---- ==== .Why Didn't We Just Use the Django Test Client All Along? ******************************************************************************* You may be asking yourself, "Why didn't we just use the Django test client from the very beginning?" In real life, that's what I would do. But I wanted to show you the "manual" way of doing it first, for a couple of reasons. Firstly, because it enabled me to introduce concepts one by one, and keep the learning curve as shallow as possible. Secondly, because you may not always be using Django to build your apps, and testing tools may not always be available--but calling functions directly and examining their responses is always possible! The Django test client does also have disadvantages; later in the book (in <>) we'll discuss the difference between fully isolated unit tests and the types of test that the test client pushes us towards (people often say these are technically "integration tests"). But for now, it's very much the pragmatic choice. ((("", startref="testclient04"))) ((("", startref="DJFtestclient04"))) ******************************************************************************* [role="pagebreak-before"] But now the moment of truth: will our functional tests pass? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] [...] ====================================================================== FAIL: test_can_start_a_todo_list (__main__.NewVisitorTest.test_can_start_a_todo_list) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/functional_tests.py", line 21, in test_can_start_a_todo_list self.fail("Finish the test!") AssertionError: Finish the test! ---- Failed? What? Oh, it's just our little reminder? Yes? Yes! We have a web page! Ahem. Well, _I_ thought it was a thrilling end to the chapter. You may still be a little baffled, perhaps keen to hear a justification for all these tests (and don't worry; all that will come), but I hope you felt just a tinge of excitement near the end there. Just a little commit to calm down, and reflect on what we've covered: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *git diff* # should show our modified test in tests.py, and the new config in urls.py $ *git commit -am "url config, map / to home_page view"* ---- ((("", startref="DJFunit03"))) ((("", startref="UTdjango03"))) That was quite a chapter! Why not try typing `git log`, possibly using the `--oneline` flag, for a reminder of what we got up to: [subs="specialcharacters,quotes"] ---- $ *git log --oneline* a6e6cc9 url config, map / to home_page view 450c0f3 First unit test and view function ea2b037 Add app for lists, with deliberately failing unit test [...] ---- Not bad--we covered the following: * Starting a Django app * The Django unit test runner * The difference between FTs and unit tests * Django view functions, and request and response objects * Django URL resolving and _urls.py_ * The Django test client * Returning basic HTML from a view [role="pagebreak-before less_space"] .Useful Commands and Concepts ******************************************************************************* Running the Django dev server:: *`python manage.py runserver`* ((("Django framework", "commands and concepts", "python manage.py runserver"))) Running the functional tests:: *`python functional_tests.py`* ((("Django framework", "commands and concepts", "python functional_tests.py"))) Running the unit tests:: *`python manage.py test`* ((("Django framework", "commands and concepts", "python manage.py test"))) The unit-test/code cycle:: 1. Run the unit tests in the terminal. 2. Make a minimal code change in the editor. 3. Repeat! ((("Django framework", "commands and concepts", "unit-test/code cycle"))) ((("unit-test/code cycle"))) ******************************************************************************* ================================================ FILE: chapter_04_philosophy_and_refactoring.asciidoc ================================================ [[chapter_04_philosophy_and_refactoring]] == What Are We Doing with All These Tests? (And, Refactoring) ((("Test-Driven Development (TDD)", "need for", id="TDDneed04"))) Now that we've seen the basics of TDD in action, it's time to pause and talk about why we're doing it. I'm imagining several of you, dear readers, have been holding back some seething frustration--perhaps some of you have done a bit of unit testing before, and perhaps some of you are just in a hurry. You've been biting back questions like: * Aren't all these tests a bit excessive? * Surely some of them are redundant? There's duplication between the functional tests and the unit tests. * Those unit tests seemed way too trivial--testing a one-line function that returns a constant! Isn't that just a waste of time? Shouldn't we save our tests for more complex things? * What about all those tiny changes during the unit-test/code cycle? Couldn't we just skip to the end? I mean, `home_page = None`!? Really? * You're not telling me you _actually_ code like this in real life? Ah, young grasshopper. I too was once full of questions like these. But only because they're perfectly good questions. In fact, I still ask myself questions like these—all the time. Does all this stuff really have value? Is this a bit of a cargo cult? === Programming Is Like Pulling a Bucket of Water [.keep-together]#Up from a Well# ((("Test-Driven Development (TDD)", "philosophy of", "bucket of water analogy"))) Ultimately, programming is hard. Often, we are smart, so we succeed. TDD is there to help us out when we're not so smart. Kent Beck (who basically invented TDD) uses the metaphor of lifting a bucket of water out of a well with a rope: when the well isn't too deep, and the bucket isn't very full, it's easy. And even lifting a full bucket is pretty easy at first. But after a while, you're going to get tired. TDD is like having a ratchet that lets you save your progress, so you can take a break, and make sure you never slip backwards. That way, you don't have to be smart _all_ the time (see <>). [[figure4-1]] .Test ALL the things (adapted from https://oreil.ly/n_8R_[Allie Brosh, Hyperbole and a Half]) image::images/tdd3_0401.png["Test ALL the things",float="right"] OK, perhaps _in general_, you're prepared to concede that TDD is a good idea, but maybe you still think I'm overdoing it? Testing the tiniest thing, and taking ridiculously many small steps? TDD is a _discipline_, and that means it's not something that comes naturally. Because many of the payoffs aren't immediate but only come in the longer term, you have to force yourself to do it in the moment. That's what the image of the Testing Goat is supposed to represent--you need to be a bit bloody-minded about it. [role="pagebreak-before less_space"] [[trivial_tests_trivial_functions]] .On the Merits of Trivial Tests for Trivial Functions ********************************************************************** In the short term, it may feel a bit silly to write tests for simple functions and [.keep-together]#constants#. It's perfectly possible to imagine still doing ``mostly'' TDD, but following more relaxed rules where you don't unit test _absolutely_ everything. But in this book my aim is to demonstrate full, rigorous TDD. Like a kata in a martial art, the idea is to learn the motions in a controlled context, when there is no adversity, so that the techniques are part of your muscle memory. It seems trivial now, because we've started with a very simple example. The problem comes when your application gets complex--that's when you really need your tests. And the danger is that complexity tends to sneak up on you, gradually. You may not notice it happening, but soon you're a boiled frog. There are two other things to say in favour of tiny, simple tests for simple functions. Firstly, if they're really trivial tests, then they won't take you that long to write. So stop moaning and just write them already. Secondly, it's always good to have a placeholder. Having a test _there_ for a simple function means it's that much less of a psychological barrier to overcome when the simple function gets a tiny bit more complex--perhaps it grows an `if`. Then a few weeks later, it grows a `for` loop. Before you know it, it's a recursive metaclass-based polymorphic tree parser factory. But because it's had tests from the very beginning, adding a new test each time has felt quite natural, and it's well tested. The alternative involves trying to decide when a function becomes ``complicated enough'', which is highly subjective. And worse, because there's no placeholder, it feels like that much more effort to start, so you're tempted each time to put it off...and pretty soon--frog soup! Instead of trying to figure out some hand-wavy subjective rules for when you should write tests, and when you can get away with not bothering, I suggest following the discipline for now--and as with any discipline, you have to take the time to learn the rules before you can break them. ********************************************************************** Now, let us return to our muttons. ((("", startref="TDDneed04"))) [role="pagebreak-before less_space"] === Using Selenium to Test User Interactions ((("Selenium", "testing user interactions with", id="Suser04"))) ((("user interactions", "testing with Selenium", id="UIselenium04"))) Where were we at the end of the last chapter? Let's rerun the test and find out: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] F ====================================================================== FAIL: test_can_start_a_todo_list (__main__.NewVisitorTest.test_can_start_a_todo_list) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/functional_tests.py", line 21, in test_can_start_a_todo_list self.fail("Finish the test!") ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ AssertionError: Finish the test! --------------------------------------------------------------------- Ran 1 test in 1.609s FAILED (failures=1) ---- Did you try it, and get an error saying "Problem loading page" or "Unable to connect"? So did I. It's because we forgot to spin up the dev server first using `manage.py runserver`. Do that, and you'll get the failure message we're after. NOTE: One of the great things about TDD is that you never have to worry about forgetting what to do next--just rerun your tests and they will tell you what you need to work on. [role="pagebreak-before"] ``Finish the test'', it says, so let's do just that! Open up 'functional_tests.py' and we'll extend our FT: [role="sourcecode small-code"] .functional_tests.py (ch04l001) ==== [source,python] ---- from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys import time import unittest class NewVisitorTest(unittest.TestCase): def setUp(self): self.browser = webdriver.Firefox() def tearDown(self): self.browser.quit() def test_can_start_a_todo_list(self): # Edith has heard about a cool new online to-do app. # She goes to check out its homepage self.browser.get("http://localhost:8000") # She notices the page title and header mention to-do lists self.assertIn("To-Do", self.browser.title) header_text = self.browser.find_element(By.TAG_NAME, "h1").text # <1> self.assertIn("To-Do", header_text) # She is invited to enter a to-do item straight away inputbox = self.browser.find_element(By.ID, "id_new_item") # <1> self.assertEqual(inputbox.get_attribute("placeholder"), "Enter a to-do item") # She types "Buy peacock feathers" into a text box # (Edith's hobby is tying fly-fishing lures) inputbox.send_keys("Buy peacock feathers") # <2> # When she hits enter, the page updates, and now the page lists # "1: Buy peacock feathers" as an item in a to-do list table inputbox.send_keys(Keys.ENTER) # <3> time.sleep(1) # <4> table = self.browser.find_element(By.ID, "id_list_table") rows = table.find_elements(By.TAG_NAME, "tr") # <1> self.assertTrue(any(row.text == "1: Buy peacock feathers" for row in rows)) # There is still a text box inviting her to add another item. # She enters "Use peacock feathers to make a fly" # (Edith is very methodical) self.fail("Finish the test!") # The page updates again, and now shows both items on her list [...] ---- ==== //IDEA: stop using id_new_item, just use name= [role="pagebreak-before"] <1> We're using the two methods that Selenium provides to examine web pages: `find_element` and `find_elements` (notice the extra `s`, which means it will return several elements rather than just one). Each one is parameterised with a `By.SOMETHING`, which lets us search using different HTML properties and attributes. <2> We also use `send_keys`, which is Selenium's way of typing into input elements. <3> The `Keys` class (don't forget to import it) lets us send special keys like Enter.footnote:[ You could also just use the string +"\n"+, but `Keys` also lets you send special keys like Ctrl, so I thought I'd show it.] <4> When we hit Enter, the page will refresh. The `time.sleep` is there to make sure the browser has finished loading before we make any assertions about the new page.((("explicit and implicit waits"))) This is called an "explicit wait" (a very simple one; we'll improve it in <>). TIP: Watch out for the difference between the Selenium `find_element()` and `find_elements()` functions. One returns an element and raises an exception if it can't find it, whereas the other returns a list, which may be empty. Also, just look at that `any()` function. It's a little-known Python built-in. I don't even need to explain it, do I? Python is such a joy.footnote:[ Python _is_ most definitely a joy, but if you think I'm being a bit smug here, I don't blame you! Actually, I wish I'd picked up on this feeling of self-satisfaction and seen it as a warning sign that I was being a little _too_ clever. In the next chapter, you'll see I get my comeuppance.] NOTE: If you're one of my readers who doesn't know Python, what's happening _inside_ the `any()` may need some explaining. The basic syntax is that of a _list comprehension_, and if you haven't learned about them, you should do so immediately! https://oreil.ly/6bX0h[Trey Hunner's explanation is excellent]. In point of fact, because we're omitting the square brackets, we're actually using a _generator expression_ rather than a list comprehension. It's probably less important to understand the difference between those two, but if you're curious, Guido van Rossum, the inventor of Python, has written https://oreil.ly/Om6vK[a blog post explaining the difference].((("generator expressions")))((("list comprehensions"))) [role="pagebreak-before"] Let's see how it gets on: [subs="specialcharacters,quotes"] ---- $ *python functional_tests.py* [...] File "...goat-book/functional_tests.py", line 22, in [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: h1; For documentation on this error, please visit: [...] ---- Decoding that, the test is saying it can't find an `

` element on the page. Let's see what we can do to add that to the HTML of our home page. ((("", startref="Suser04"))) ((("", startref="UIselenium04"))) Big changes to a functional test are usually a good thing to commit on their own. I failed to do so when I was first working out the code for this chapter, and I regretted it later when I changed my mind and had the change mixed up with a bunch of others. The more atomic your commits, the better: [subs="specialcharacters,quotes"] ---- $ *git diff* # should show changes to functional_tests.py $ *git commit -am "Functional test now checks we can input a to-do item"* ---- === The "Don't Test Constants" Rule, and Templates [.keep-together]#to the Rescue# ((("“Don’t Test Constants” rule", primary-sortas="Don’t Test Constants rule"))) ((("unit tests", "“Don’t Test Constants” rule", secondary-sortas="Don’t Test Constants rule")))((("constants", "“Don’t Test Constants” rule", secondary-sortas="Don't"))) Let's take a look at our unit tests, _lists/tests.py_. Currently we're looking for specific HTML strings, but that's not a particularly efficient way of testing HTML. In general, one of the rules of unit testing is "don't test constants", and testing HTML as text is a lot like testing a constant. In other words, if you have some code that says: [role="skipme"] [source,python] ---- wibble = 3 ---- There's not much point in a test that says: [role="skipme"] [source,python] ---- from myprogram import wibble assert wibble == 3 ---- Unit tests are really about testing logic, flow control, and configuration. Making assertions about exactly what sequence of characters we have in our HTML strings isn't doing that. It's not _quite_ that simple, because HTML is code after all, and we do want something to check that we've written code that works—but that's our FT's job, not the unit test's. So maybe "don't test constants" isn't the online guideline at play here, but in any case, mangling raw strings in Python really isn't a great way of dealing with HTML. There's a much better solution, which is to use templates. Quite apart from anything else, if we can keep HTML to one side in a file whose name ends in '.html', we'll get better syntax highlighting! There are lots of Python templating frameworks out there, and Django has its own which works very well. Let's use that. ==== Refactoring to Use a Template ((("unit tests", "refactoring in", id="UTrefactor04"))) ((("refactoring", id="refactor04")))((("templates", "refactoring unit tests to use", id="ix_tmplref"))) What we want to do now is make our view function return exactly the same HTML, but just using a different process. That's a refactor--when we try to improve the code _without changing its functionality_. That last bit is really important. If you try to add new functionality at the same time as refactoring, you're much more likely to run into trouble. Refactoring is actually a whole discipline in itself, and it even has a reference book: Martin Fowler's http://refactoring.com[_Refactoring_]. The first rule is that you can't refactor without tests. Thankfully, we're doing TDD, so we're way ahead of the game. Let's check that our tests pass; they will be what makes sure that our refactoring is behaviour-preserving: [subs="specialcharacters,quotes"] ---- $ *python manage.py test* [...] OK ---- Great! We'll start by taking our HTML string and putting it into its own file. Create a directory called _lists/templates_ to keep templates in, and then open a file at _lists/templates/home.html_, to which we'll transfer our HTML:footnote:[ Some people like to use another subfolder named after the app (i.e., _lists/templates/lists_) and then refer to the template as _lists/home.html_. This is called "template namespacing". I figured it was overcomplicated for this small project, but it may be worth it on larger projects. There's more in the https://docs.djangoproject.com/en/5.2/intro/tutorial03/#write-views-that-actually-do-something[Django tutorial].] [role="sourcecode"] .lists/templates/home.html (ch04l002) ==== [source,html] ---- To-Do lists ---- ==== Mmm, syntax-highlighted...much nicer! Now to change our view function: [role="sourcecode"] .lists/views.py (ch04l003) ==== [source,python] ---- from django.shortcuts import render def home_page(request): return render(request, "home.html") ---- ==== Instead of building our own `HttpResponse`, we now use the Django `render()` function. It takes the request as its first parameter (for reasons we'll go into later) and the name of the template to render. Django will automatically search folders called _templates_ inside any of your apps' directories. Then it builds an `HttpResponse` for you, based on the content of the template. NOTE: Templates are a very powerful feature of Django's, and their main strength consists of substituting Python variables into HTML text. We're not using this feature yet, but we will in future chapters. That's why we use `render()` rather than, say, manually reading the file from disk with the built-in `open()`. Let's see if it works: [role="small-code"] [subs="specialcharacters,macros,callouts"] ---- $ pass:quotes[*python manage.py test*] [...] ====================================================================== ERROR: test_home_page_returns_correct_html (lists.tests.HomePageTest.test_home_page_returns_correct_html) <2> ---------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/lists/tests.py", line 7, in test_home_page_returns_correct_html response = self.client.get("/") <3> ^^^^^^^^^^^^^^^^^^^^ [...] File "...goat-book/lists/views.py", line 4, in home_page return render(request, "home.html") <4> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../django/shortcuts.py", line 24, in render content = loader.render_to_string(template_name, context, request, using=using) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../django/template/loader.py", line 61, in render_to_string template = get_template(template_name, using=using) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File ".../django/template/loader.py", line 19, in get_template raise TemplateDoesNotExist(template_name, chain=chain) django.template.exceptions.TemplateDoesNotExist: home.html <1> ---------------------------------------------------------------------- Ran 1 test in 0.074s ---- Another chance to analyse a traceback: <1> We start with the error: it can't find the template. <2> Then we double-check what test is failing: sure enough, it's our test of the view HTML. <3> Then we find the line in our tests that caused the failure: it's when we request the root URL ("/"). <4> Finally, we look for the part of our own application code that caused the failure: it's when we try to call `render`. So why can't Django find the template? It's right where it's supposed to be, in the _lists/templates_ folder. The thing is that we haven't yet _officially_ registered our lists app with Django. Unfortunately, just running the `startapp` command and having what is obviously an app in your project folder isn't quite enough. You have to tell Django that you _really_ mean it, and add it to 'settings.py' as well—belt and braces. Open it up and look for a variable called `INSTALLED_APPS`, to which we'll add `lists`: [role="sourcecode"] .superlists/settings.py (ch04l004) ==== [source,python] ---- # Application definition INSTALLED_APPS = [ "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", "lists", ] ---- ==== You can see there's lots of apps already in there by default. We just need to add ours to the bottom of the list. Don't forget the trailing comma--it may not be required, but one day you'll be really annoyed when you forget it and Python concatenates two strings on different lines... Now we can try running the tests again: [subs="specialcharacters,quotes"] ---- $ *python manage.py test* [...] OK ---- And we can double-check with the FTs: [subs="specialcharacters,quotes"] ---- $ *python functional_tests.py* [...] File "...goat-book/functional_tests.py", line 22, in [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: h1; For documentation on this error, please visit: [...] ---- Good, they still get to the same place they did before. Our refactor of the code is now complete, and the tests mean we're happy that behaviour is preserved. Now we can change the tests so that they're no longer testing constants; instead, they should just check that we're rendering the right template.((("templates", "refactoring unit tests to use", startref="ix_tmplref"))) ((("", startref="refactor04"))) ((("", startref="UTrefactor04"))) === Revisiting Our Unit Tests Our unit tests are currently essentially checking HTML by hand—certainly that's very close to "testing constants". [role="sourcecode currentcontents"] .lists/tests.py ==== [source,python] ---- def test_home_page_returns_correct_html(self): response = self.client.get("/") self.assertContains(response, "To-Do lists") # <1> self.assertContains(response, "") self.assertContains(response, "") ---- ==== We don't want to be duplicating the full content of our HTML template in our tests, or even last sections of it. What could we do instead? Rather than testing the full template, we could just check that we're using the _right_ template. The Django test client has a method, `assertTemplateUsed`, which will let us do just that.((("templates", "checking that right template is used"))) [role="sourcecode"] .lists/tests.py (ch04l005) ==== [source,python] ---- def test_home_page_returns_correct_html(self): response = self.client.get("/") self.assertContains(response, "To-Do lists") # <1> self.assertContains(response, "") self.assertContains(response, "") self.assertTemplateUsed(response, "home.html") # <2> ---- ==== <1> We'll leave the old tests there for now, just to make sure everything is working the way we think it is. <2> `.assertTemplateUsed` lets us check what template was used to render a response. (NB: It will only work for responses that were retrieved by the test client.) And that test will still pass: ---- Ran 1 tests in 0.016s OK ---- Just because I'm always suspicious of a test I haven't seen fail, let's deliberately break it: [role="sourcecode"] .lists/tests.py (ch04l006) ==== [source,python] ---- self.assertTemplateUsed(response, "wrong.html") ---- ==== [role="pagebreak-before"] That way, we'll also learn what its error messages look like: ---- AssertionError: False is not true : Template 'wrong.html' was not a template used to render the response. Actual template(s) used: home.html ---- That's very helpful! Let's change the assert back to the right thing. [role="sourcecode"] .lists/tests.py (ch04l007) ==== [source,python] ---- from django.test import TestCase class HomePageTest(TestCase): def test_uses_home_template(self): response = self.client.get("/") self.assertTemplateUsed(response, "home.html") ---- ==== Now, instead of testing constants we're testing at a higher level of abstraction. Great! ==== Test Behaviour, Not Implementation As so often in the world of programming though, things are not black and white.((("behaviour", "testing behaviour, not implementation"))) Yes, on the plus side, our tests no longer care about the specific content of our HTML so they are no longer brittle with respect to minor changes of the copy in our template. But on the other hand, they depend on some Django implementation details, so they _are_ brittle with respect to changing the template rendering library, or even just renaming templates. In a way, testing for the template name (and implicitly, even checking that we used a template at all) is a lot like testing implementation. So what is the _behaviour_ that we want? Yes, in a sense, the "behaviour" we want from the view is "render the template". But from the point of view of the user, it's "show me the home page". We're also vulnerable to accidentally breaking the template. Let's try it now, by just deleting all the contents of the template file: [subs="specialcharacters,quotes"] ---- $ *mv lists/templates/home.html lists/templates/home.html.bak* $ *touch lists/templates/home.html* $ *python manage.py test* [...] OK ---- [role="pagebreak-before"] Yes, our FTs will pick up on this, so ultimately we're OK: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] [...] self.assertIn("To-Do", self.browser.title) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'To-Do' not found in '' ---- But it would be nice to have our unit tests pick up on this too: [subs="specialcharacters,quotes"] ---- $ *mv lists/templates/home.html.bak lists/templates/home.html* ---- Deciding exactly what to test with FTs and what to test with unit tests is a fine line, and the objective is not to double-test everything. But in general, the more we can test with unit tests the better. They run faster, and they give more specific feedback.((("smoke tests"))) So, let's bring back a minimal "smoke test"footnote:[ A smoke test is a minimal test that can quickly tell you if something is wrong, without exhaustively testing every aspect that you might care about. Wikipedia has some fun speculation on the https://oreil.ly/1_isl[etymology].] to check that what we're rendering is actually the home page: [role="sourcecode"] .lists/tests.py (ch04l008) ==== [source,python] ---- class HomePageTest(TestCase): def test_uses_home_template(self): response = self.client.get("/") self.assertTemplateUsed(response, "home.html") # <1> def test_renders_homepage_content(self): response = self.client.get("/") self.assertContains(response, "To-Do") # <2> ---- ==== <1> We'll keep this first test, which asserts on whether we're rendering the right "constant". <2> And this gives us a minimal smoke test that we have got the right content in the template. [role="pagebreak-before"] As our home page template gains more functionality over the next couple of chapters, we'll come back to talking about what to test here in the unit tests and what to leave to the FTs. TIP: Unit tests give you faster and more specific feedback than FTs. Bear this in mind when deciding what to test where. We'll visit the trade-offs between different types of tests at several points in the book, and particularly in <>. === On Refactoring ((("unit tests", "refactoring in"))) ((("refactoring"))) That was an absolutely trivial example of refactoring. But, as Kent Beck puts it in _Test-Driven Development: By Example_, "Am I recommending that you actually work this way? No. I'm recommending that you be _able_ to work this way". In fact, as I was writing this my first instinct was to dive in and change the test first--make it use the `assertTemplateUsed()` function straight away; against the expected render; and then go ahead and make the code change. But notice how that actually would have left space for me to break things: I could have defined the template as containing 'any' arbitrary string, instead of the string with the right `` and `` tags. TIP: When refactoring, work on either the code or the tests, but not both at once. // SEBASTIAN: I'd put in other words, perhaps as an additional paragraph. // Change one thing at a time - either code or the tests. // After introducing changes to one of them, run the tests. // If they pass, carry on with changing the other. If they don't - fix the tests first. // *It's virtually impossible (also for experienced software developer) to precisely* // *pinpoint source of error if you ended up with failing tests after changing both* // *the tests and code.* There's always a tendency to skip ahead a couple of steps, to make a few tweaks to the behaviour while you're refactoring. But pretty soon you've got changes to half a dozen different files, you've totally lost track of where you are, and nothing works anymore. If you don't want to end up like https://oreil.ly/F_Hqf[Refactoring Cat] (<<RefactoringCat>>), stick to small steps; keep refactoring and functionality changes entirely separate. [[RefactoringCat]] .Refactoring Cat--be sure to look up the full animated GIF (source: 4GIFs.com) image::images/tdd3_0402.png["An adventurous cat, trying to refactor its way out of a slippery bathtub"] NOTE: We'll come across Refactoring Cat again during this book, as an example of what happens when we get carried away and change too many things at once. Think of it as the little cartoon demon counterpart to the Testing Goat, popping up over your other shoulder and giving you bad advice. It's a good idea to do a commit after any refactoring: [subs="specialcharacters,quotes"] ---- $ *git status* # see tests.py, views.py, settings.py, + new templates folder $ *git add .* # will also add the untracked templates folder $ *git diff --staged* # review the changes we're about to commit $ *git commit -m "Refactor homepage view to use a template"* ---- [role="pagebreak-before less_space"] === A Little More of Our Front Page In the meantime, our FT is still failing.((("functional tests (FTs)", "passing test on home page"))) Let's now make an actual code change to get it passing. Because our HTML is now in a template, we can feel free to make changes to it, without needing to write any extra unit tests. NOTE: This is another distinction between FTs and unit tests; because the FTs use a real web browser, we use them as the primary tool for testing our UI, and the HTML that implements it. So, we wanted an `<h1>`: [role="sourcecode"] .lists/templates/home.html (ch04l009) ==== [source,html] ---- <html> <head> <title>To-Do lists

Your To-Do list

---- ==== Let's see if our FT likes it a little better: ---- selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_new_item"]; For documentation on this error, [...] ---- OK, let's add an input with that ID: [role="sourcecode"] .lists/templates/home.html (ch04l010) ==== [source,html] ---- [...]

Your To-Do list

---- ==== And now what does the FT say? ---- AssertionError: '' != 'Enter a to-do item' ---- We add our placeholder text... [role="sourcecode"] .lists/templates/home.html (ch04l011) ==== [source,html] ---- ---- ==== Which gives: ---- selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_list_table"]; [...] ---- So we can go ahead and put the table onto the page. At this stage it'll just be empty: [role="sourcecode"] .lists/templates/home.html (ch04l012) ==== [source,html] ----
---- ==== What does the FT think? ---- [...] File "...goat-book/functional_tests.py", line 40, in test_can_start_a_todo_list self.assertTrue(any(row.text == "1: Buy peacock feathers" for row in rows)) ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: False is not true ---- Slightly cryptic! We can use the line number to track it down, and it turns out it's that `any()` function I was so smug about earlier--or, more precisely, the `assertTrue`, which doesn't have a very explicit failure message.((("error messages", "passing custom error message to assertX methods in unittest")))((("unittest module", "passing custom error message to assertX methods in"))) We can pass a custom error message as an argument to most `assertX` methods in `unittest`: [role="sourcecode"] .functional_tests.py (ch04l013) ==== [source,python] ---- self.assertTrue( any(row.text == "1: Buy peacock feathers" for row in rows), "New to-do item did not appear in table", ) ---- ==== If you run the FT again, you should see our helpful message: ---- AssertionError: False is not true : New to-do item did not appear in table ---- But now, to get this to pass, we will need to actually process the user's form submission. And that's a topic for the next chapter. For now let's do a commit: [subs="specialcharacters,quotes"] ---- $ *git diff* $ *git commit -am "Front page HTML now generated from a template"* ---- Thanks to a bit of refactoring, we've got our view set up to render a template, we've stopped testing constants, and we're now well placed to start processing user input. === Recap: The TDD Process ((("Test-Driven Development (TDD)", "concepts", "Red/Green/Refactor"))) ((("Red/Green/Refactor"))) ((("unit-test/code cycle"))) ((("Test-Driven Development (TDD)", "overall process of", id="TDDprocess04"))) We've now seen all the main aspects of the TDD process, in practice: * Functional tests * Unit tests * The unit-test/code cycle * Refactoring It's time for a little recap, and perhaps even some flowcharts. (Forgive me, my years misspent as a management consultant have ruined me. On the plus side, said flowcharts will feature recursion!) What does the overall TDD process look like? * We write a test. * We run the test and see it fail. * We write some minimal code to get it a little further. * We rerun the test and repeat until it passes (the unit-test/code cycle) * Then, we look for opportunities to refactor our code, using our tests to make sure we don't break anything. * Then, we look for opportunities to refactor our tests too, while attempting to stick to rules like "test behaviour, not implementation" and "don't test constants". * And start again from the top! See <>. [[simple-tdd-diagram]] .TDD process as a flowchart, including the unit-test/code cycle image::images/tdd3_0403.png["A flowchart with boxes for tests, coding and refactoring, with yes/no labels showing when we move forwards or backwards"] It's very common to talk about this process using the three words: _red/green/refactor_. See <>. [[red-green-refactor]] .Red/green/refactor image::images/tdd3_0404.png["Red, green, and refactor as three nodes in a circle, with arrows flowing around."] * We write a test, and see it fail ("red"). * We cycle between code and tests until the test passes ("green"). * Then, we look for opportunities to refactor. * Repeat as required! ==== Double-loop TDD But how does this apply when we have functional tests _and_ unit tests? Well, you can think of the FT as driving a higher-level version of the same((("double-loop TDD")))((("Red/Green/Refactor", "inner and outer loops in"))) cycle, with an inner red/green/refactor loop being required to get an FT from red to green; see <>. [[double-loop-tdd-diagram]] .Double-loop TDD: Inner and outer loops image::images/tdd3_0702.png["An inner red/green/refactor loop surrounded by an outer red/green of FTs"] When a new feature or business requirement comes along, we write a new (failing) FT to capture a high-level view of the requirement. It may not cover every last edge case, but it should be enough to reassure ourselves that things are working. To get that FT to green, we then enter into the lower-level unit test cycle, where we put together all the moving parts required, and add tests for all the edge cases. Any time we get to green and refactored at the unit test level, we can pop back up to the FT level to guide us towards the next thing we need to work on. Once both levels are green, we can do any extra refactoring or work on edge cases. // SEBASTIAN: It's a great moment to stop and reflect on the things a reader learnt so far. // From my POV, introducing diagrams and explaining them encourages reader to // generalise knowledge they acquired so far. // Therefore, it would be great to recap on how these cycles were in play in this // and previous chapters. // TL;DR: more diagrams pls. We'll explore all of the different parts of this workflow in more detail over the coming chapters. ((("", startref="TDDprocess04"))) .How to "Check" Your Code, or Skip Ahead (If You Must) ******************************************************************************* ((("GitHub"))) ((("code examples, obtaining and using"))) All of the code examples I've used in the book are available in https://github.com/hjwp/book-example/[my repo on GitHub]. So, if you ever want to compare your code against mine, you can take a look at it there. Each chapter has its own branch, which is named after its short name. The https://github.com/hjwp/book-example/tree/chapter_04_philosophy_and_refactoring[one for this chapter] is a snapshot of the code as it should be at the _end_ of the chapter. You can find a full list of them in <>, as well as instructions on how to download them or use Git to compare your code to mine. Obviously I can't possibly condone it, but you can also use my repo to "skip ahead" and check out the code to let you work on a later chapter without having worked through all the earlier chapters yourself. You're only cheating yourself you know! ******************************************************************************* ================================================ FILE: chapter_05_post_and_database.asciidoc ================================================ [[chapter_05_post_and_database]] == Saving User Input: Testing the Database // ((("user interactions", "testing database input", id="UIdatabase05"))) // disabled due to pdf rendering issue So far, we've managed to return a static HTML page with an input box in it. Next, we want to take the text that the user types into that input box and send it to the server, so that we can save it somehow and display it back to them later. The first time I started writing code for this chapter, I immediately wanted to skip to what I thought was the right design: multiple database tables for lists and list items, a bunch of different URLs for adding new lists and items, three new view functions, and about half a dozen new unit tests for all of the above. But I stopped myself. Although I was pretty sure I was smart enough to handle coding all those problems at once, the point of TDD is to enable you to do one thing at a time, when you need to. So I decided to be deliberately short-sighted, and at any given moment _only_ do what was necessary to get the functional tests (FTs) a little further. ((("iterative development style"))) This will be a demonstration of how TDD can support an incremental, iterative style of development--it may not be the quickest route, but you do get there in the end.footnote:[ "Geepaw" Hill, another one of the TDD OGs, has https://oreil.ly/qTCLk[a series of blog posts] advocating for taking "Many More Much Smaller Steps (MMMSS)". In this chapter I'm being unrealistically short-sighted for effect, so don't do that! But Geepaw argues that in the real world, when you slice your work into tiny increments, not only do you get there in the end, but you end up delivering business value _faster_. ] There's a neat side benefit, which is that it enables me to introduce new concepts like models, dealing with POST requests, Django template tags, and so on, _one at a time_ rather than having to dump them on you all at once. None of this says that you _shouldn't_ try to think ahead and be clever. In the next chapter, we'll use a bit more design and up-front thinking, and show how that fits in with TDD. But for now, let's plough on mindlessly and just do what the tests tell us to. === Wiring Up Our Form to Send a POST Request ((("database testing", "HTML POST requests", "creating", id="DBIpostcreate05")))((("form data validation", "wiring up form to send POST request", id="ix_form"))) ((("POST requests", "creating", id="POSTcreate05"))) ((("HTML", "POST requests", "creating"))) At the end of the last chapter, the tests were telling us we weren't able to save the user's input: ---- File "...goat-book/functional_tests.py", line 40, in test_can_start_a_todo_list [...] AssertionError: False is not true : New to-do item did not appear in table ---- To get it to the server, for now we'll use a standard HTML POST request. A little boring, but also nice and easy to deliver--we can use all sorts of sexy HTML5 and JavaScript later in the book. To get our browser to send a POST request, we need to do two things: 1. Give the `` element a `name=` attribute. 2. Wrap it in a `
` tagfootnote:[Did you know that you don't need a button to make a form submit? I can't remember when I learned that, but readers have mentioned that it's unusual so I thought I'd draw your attention to it.] with `method="POST"`. === Testing the Contract Between Frontend and Backend If you remember in the last chapter, we said we wanted to come back and revisit the smoke test of our home page template content.((("frontend, testing contract between backend and")))((("backend, testing contract between frontend and"))) Let's have a quick look at our unit tests: [role="sourcecode currentcontents"] .lists/tests.py ==== [source,python] ---- class HomePageTest(TestCase): def test_uses_home_template(self): response = self.client.get("/") self.assertTemplateUsed(response, "home.html") def test_renders_homepage_content(self): response = self.client.get("/") self.assertContains(response, "To-Do") ---- ==== What's important about our home page content? How can we obey both the "don't test constants" rule and the "test behaviour, not implementation" rule? The specific spelling of the word "To-Do" is not important. As we've just seen, the most important _behaviour_ that our home page is enabling, is the ability to submit a to-do item.((("behaviour", "testing for To-Do page"))) The way we're going to deliver that is by adding a `` tag with `method="POST"`, and inside that, making sure our `` has a `name="item_text"`. Our FTs are telling us that it's not working at a high level, so what unit tests can we write at the lower level?((("unit tests", "writing for form in To-Do list home page"))) Let's start with the form: [role="sourcecode"] .lists/tests.py (ch05l001) ==== [source,python] ---- class HomePageTest(TestCase): def test_uses_home_template(self): [...] def test_renders_input_form(self): # <1> response = self.client.get("/") self.assertContains(response, '') # <2> ---- ==== <1> We change the name of the test. <2> And we assert on the `` tag specifically. That gives us: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] [...] AssertionError: False is not true : Couldn't find '' in the following response b'\n \n To-Do lists\n \n \n

Your To-Do list

\n \n \n
\n \n\n' ---- Let's adjust our template at 'lists/templates/home.html': [role="sourcecode"] .lists/templates/home.html (ch05l002) ==== [source,html] ----

Your To-Do list

---- ==== That gives us passing unit tests: ---- OK ---- And next, let's add a test for the `name=` attribute on the `` tag: [role="sourcecode"] .lists/tests.py (ch05l003) ==== [source,python] ---- def test_renders_input_form(self): response = self.client.get("/") self.assertContains(response, '
') self.assertContains(response, '\n \n To-Do lists\n \n \n

Your To-Do list

\n \n \n
\n \n
\n \n\n' ---- And we fix it like this: [role="sourcecode small-code"] .lists/templates/home.html (ch05l004) ==== [source,html] ----

Your To-Do list

---- ==== That gives us passing unit tests: ---- OK ---- The lesson here is that we've tried to identify the "contract" between the frontend and the backend of our site. For our HTML form to work, it needs the form with the right `method`, and the input with the right `name`. Everything else is cosmetic. So that's what we test for in our unit tests.((("form data validation", "wiring up form to send POST request", startref="ix_form"))) === Debugging Functional Tests Time to go back to our FT.((("functional tests (FTs)", "debugging for To-Do list home page form", id="ix_FTdbg"))) It gives us a slightly cryptic, unexpected error: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] [...] Traceback (most recent call last): File "...goat-book/functional_tests.py", line 38, in test_can_start_a_todo_list table = self.browser.find_element(By.ID, "id_list_table") [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_list_table"]; [...] ---- Oh dear, we're now failing two lines _earlier_, after we submit the form, but before we are able to do the assert. Selenium seems to be unable to find our list table. Why on earth would that happen? Let's take another look at our code: [role="sourcecode currentcontents"] .functional_tests.py ==== [source,python] ---- # When she hits enter, the page updates, and now the page lists # "1: Buy peacock feathers" as an item in a to-do list table inputbox.send_keys(Keys.ENTER) time.sleep(1) table = self.browser.find_element(By.ID, "id_list_table") # <1> rows = table.find_elements(By.TAG_NAME, "tr") self.assertTrue( any(row.text == "1: Buy peacock feathers" for row in rows), "New to-do item did not appear in table", ) ---- ==== <1> Our test unexpectedly fails on this line. How do we figure out what's going on? ((("functional tests (FTs)", "debugging techniques"))) ((("time.sleeps"))) ((("error messages", seealso="troubleshooting"))) ((("print", "debugging with"))) ((("debugging", "of functional tests"))) When a functional test fails with an unexpected failure, there are several things we can do to debug it: * Add `print` statements to show, for example, what the current page text is. * Improve the _error message_ to show more info about the current state. * Manually visit the site yourself. * Use `time.sleep` to pause the test during execution so you can inspect what was happening.footnote:[ Another common technique for debugging tests is to use `breakpoint()` to drop into a debugger like `pdb`. This is more useful for _unit_ tests rather than FTs though, because in an FT you usually can't step into actual application code. Personally, I only find debuggers useful for really fiddly algorithms, which we won't see in this book.] We'll look at all of these over the course of this book, but the `time.sleep` option is the one that leaps to mind with this kind of error in an FT. Let's try it now.((("sleep", see="time.sleeps"))) ==== Debugging with time.sleep Conveniently, we've already got a ((("time.sleeps", "debugging with")))sleep just before the error occurs; let's just extend it a little: [role="sourcecode"] .functional_tests.py (ch05l005) ==== [source,python] ---- # When she hits enter, the page updates, and now the page lists # "1: Buy peacock feathers" as an item in a to-do list table inputbox.send_keys(Keys.ENTER) time.sleep(10) table = self.browser.find_element(By.ID, "id_list_table") ---- ==== ((("debugging", "Django debug page"))) Depending on how fast Selenium runs on your PC, you may have caught a glimpse of this already, but when we run the FTs again, we've got time to see what's going on: you should see a page that looks like <>, with lots of Django debug information. [[csrf_error_screenshot]] .Django debug page showing CSRF error image::images/tdd3_0501.png["Django debug page showing CSRF error"] .Security: Surprisingly Fun! ******************************************************************************* ((("cross-site request forgery (CSRF)"))) ((("security issues and settings", "cross-site request forgery"))) If you've never heard of a _cross-site request forgery_ (CSRF) exploit, why not look it up now? Like all security exploits, it's entertaining to read about, being an ingenious use of a system in unexpected ways. When I went to university to get my computer science degree, I signed up for the "security" module out of a sense of duty: _Oh well, it'll probably be very dry and boring, but I suppose I'd better take it. Eat your vegetables, and so forth_. It turned out to be one of the most fascinating modules of the whole course! Absolutely full of the joy of hacking, of the particular mindset it takes to think about how systems can be used in unintended ways. I want to recommend the textbook from that course, Ross Anderson's https://oreil.ly/TKmYQ[_Security Engineering_]. It's quite light on pure crypto, but it's absolutely full of interesting discussions of unexpected topics like lock picking, forging bank notes, inkjet printer cartridge [keep-together]#economics#, and spoofing South African Air Force jets with replay attacks. It's a huge tome, about three inches thick, and I promise you it's an absolute page-turner. ******************************************************************************* ((("templates", "tags", "{% csrf_token %}"))) ((("{% csrf_token %}"))) Django's CSRF protection involves placing a little autogenerated unique token into each generated form, to be able to verify that POST requests have definitely come from the form generated by the server. So far, our template has been pure HTML, and in this step we make the first use of Django's template magic. To add the CSRF token, we use a 'template tag', which has the curly-bracket/percent syntax, `{% ... %}`—famous for being the world's most annoying two-key touch-typing combination: // IDEA: unit test this? can use Client(enforce_csrf_checks=True) [role="sourcecode"] .lists/templates/home.html (ch05l006) ==== [source,html] ----
{% csrf_token %} ---- ==== Django will substitute the template tag during rendering with an `` containing the CSRF token. Rerunning the functional test will now bring us back to our previous (expected) failure: ---- File "...goat-book/functional_tests.py", line 40, in test_can_start_a_todo_list [...] AssertionError: False is not true : New to-do item did not appear in table ---- Because our long `time.sleep` is still there, the test will pause on the final screen, showing us that the new item text disappears after the form is submitted, and the page refreshes to show an empty form again. That's because we haven't wired up our server to deal with the POST request yet--it just ignores it and displays the normal home page. ((("", startref="DBIpostcreate05"))) ((("", startref="POSTcreate05"))) We can put our normal short `time.sleep` back now though: [role="sourcecode"] .functional_tests.py (ch05l007) ==== [source,python] ---- # "1: Buy peacock feathers" as an item in a to-do list table inputbox.send_keys(Keys.ENTER) time.sleep(1) table = self.browser.find_element(By.ID, "id_list_table") ---- ==== === Processing a POST Request on the Server ((("functional tests (FTs)", "debugging for To-Do list home page form", startref="ix_FTdbg")))((("database testing", "HTML POST requests", "processing"))) ((("POST requests", "processing"))) ((("HTML", "POST requests", "processing"))) Because we haven't specified an `action=` attribute in the form, it is submitting back to the same URL it was rendered from by default (i.e., `/`), which is dealt with by our `home_page` function. That's fine for now; let's adapt the view to be able to deal with a POST request. That means a new unit test for the `home_page` view. Open up 'lists/tests.py', and add a new method to `HomePageTest`: [role="sourcecode small-code"] .lists/tests.py (ch05l008) ==== [source,python] ---- class HomePageTest(TestCase): def test_uses_home_template(self): [...] def test_renders_input_form(self): response = self.client.get("/") self.assertContains(response, '
') self.assertContains(response, ' def test_can_save_a_POST_request(self): response = self.client.post("/", data={"item_text": "A new list item"}) # <1><2> self.assertContains(response, "A new list item") # <3> ---- ==== <1> To do a POST, we call `self.client.post` and, as you can see, it takes a `data` argument that contains the form data we want to send. <2> Notice the echo of the `item_text` name from earlier.footnote:[ You could even define a constant for this, to make the link more explicit.] <3> Then we check that the text from our POST request ends up in the rendered HTML. That gives us our expected fail: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] [...] AssertionError: False is not true : Couldn't find 'A new list item' in the following response b'\n \n To-Do lists\n \n \n

Your To-Do list

\n \n \n \n
\n \n\n' ---- In (slightly exaggerated) TDD style, we can single-mindedly do "the simplest thing that could possibly work" to address this test failure, which is to add an `if` and a new code path for POST requests, with a deliberately silly return value: [role="sourcecode"] .lists/views.py (ch05l009) ==== [source,python] ---- from django.http import HttpResponse from django.shortcuts import render def home_page(request): if request.method == "POST": # <1> return HttpResponse("You submitted: " + request.POST["item_text"]) # <2> return render(request, "home.html") ---- ==== <1> `request.method` lets us check whether we got a POST or a GET request. <2> `request.POST` is a dictionary-like object containing the form data (in this case, the `item_text` value we expect from the form `input` tag). Fine, that gets our unit tests passing: ---- OK ---- ...but it's not really what we want.footnote:[ But we _did_ learn about `request.method` and `request.POST`, right? I know it might seem that I'm overdoing it, but doing things in tiny little steps really does have a lot of advantages, and one of them is that you can really think about (or in this case, learn) one thing at a time.] And even if we were genuinely hoping this was the right solution, our FTs are here to remind us that this isn't how things are supposed to work: ---- selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_list_table"]; [...] ---- The list table disappears after the form submission. If you didn't see it in the FT run, try it manually with `runserver`; you'll see something like <>. [[table_gone_screenshot]] .I see my item text but no table... image::images/tdd3_0502.png["A screenshot of the page after submission, which just has the raw text You submitted: Buy asparagus"] What we really want to do is add the POST submission to the to-do items table in the home page template. We need some sort of way to pass data from our view, to be shown in the template. === Passing Python Variables to Be Rendered in the Template We've already had a hint of it, and now it's time to start to get to know the real power of the Django template syntax, which is to pass variables from our Python view code into HTML templates.((("database testing", "template syntax", id="DTtemplate05")))((("templates", "syntax")))((("templates", "passing variables to"))) Let's start by seeing how the template syntax lets us include a Python object in our template. The notation is `{{ ... }}`, which displays the object as a string: [role="sourcecode small-code"] .lists/templates/home.html (ch05l010) ==== [source,html] ----

Your To-Do list

{% csrf_token %} <1>
{{ new_item_text }}
---- ==== <1> Here's our template variable. `new_item_text` will be the variable name for the user input we display in the template. Let's adjust our unit test so that it checks whether we are still using the template: [role="sourcecode"] .lists/tests.py (ch05l011) ==== [source,python] ---- def test_can_save_a_POST_request(self): response = self.client.post("/", data={"item_text": "A new list item"}) self.assertContains(response, "A new list item") self.assertTemplateUsed(response, "home.html") ---- ==== And that will fail as expected: ---- AssertionError: No templates used to render the response ---- Good; our deliberately silly return value is now no longer fooling our tests, so we are allowed to rewrite our view, and tell it to pass the POST parameter to the template. The `render` function takes, as its third argument, a dictionary, which maps template variable names to their values. In theory, we can use it for the POST case as well as the default GET case, so let's remove the `if request.method == "POST"` and simplify our view right down to: [role="sourcecode"] .lists/views.py (ch05l012) ==== [source,python] ---- def home_page(request): return render( request, "home.html", {"new_item_text": request.POST["item_text"]}, ) ---- ==== What do the tests think? ---- ERROR: test_uses_home_template (lists.tests.HomePageTest.test_uses_home_template) [...] {"new_item_text": request.POST["item_text"]}, ~~~~~~~~~~~~^^^^^^^^^^^^^ [...] django.utils.datastructures.MultiValueDictKeyError: 'item_text' ---- ==== An Unexpected Failure ((("unexpected failures"))) ((("Test-Driven Development (TDD)", "concepts", "unexpected failures"))) Oops, an _unexpected failure_. If you remember the rules for reading tracebacks, you'll spot that it's actually a failure in a _different_ test. We got the actual test we were working on to pass, but the unit tests have picked up an unexpected consequence, a regression: we broke the code path where there is no POST request. This is the whole point of having tests. Yes, perhaps we could have predicted this would happen, but imagine if we'd been having a bad day or weren't paying attention: our tests have just saved us from accidentally breaking our application and, because we're using TDD, we found out immediately. We didn't have to wait for a QA team, or switch to a web browser and click through our site manually, so we can get on with fixing it straight away. Here's how: [role="sourcecode"] .lists/views.py (ch05l013) ==== [source,python] ---- def home_page(request): return render( request, "home.html", {"new_item_text": request.POST.get("item_text", "")}, ) ---- ==== We use http://docs.python.org/3/library/stdtypes.html#dict.get[`dict.get`] to supply a default value, for the case where we are doing a normal GET request, when the POST dictionary is empty. [role="pagebreak-before"] The unit tests should now pass. Let's see what the FTs say: ---- AssertionError: False is not true : New to-do item did not appear in table ---- TIP: If your functional tests show you a different error at this point, or at any point in this chapter, complaining about a +S⁠t⁠a⁠l⁠e⁠E⁠l⁠e⁠m⁠e⁠n⁠t​R⁠e⁠f⁠e⁠r⁠e⁠n⁠c⁠e⁠Exception+, you may need to increase the `time.sleep` explicit wait--try two or three seconds instead of one; then read on to the next chapter for a more robust solution. ==== Improving Error Messages in Tests ((("debugging", "improving error messages")))((("error messages", "improving in tests"))) Hmm, not a wonderfully helpful error. Let's use another of our FT debugging techniques: improving the error message. This is probably the most constructive technique, because those improved error messages stay around to help debug any future errors: [role="sourcecode"] .functional_tests.py (ch05l014) ==== [source,python] ---- self.assertTrue( any(row.text == "1: Buy peacock feathers" for row in rows), f"New to-do item did not appear in table. Contents were:\n{table.text}", ) ---- ==== That gives us a more helpful message: ---- AssertionError: False is not true : New to-do item did not appear in table. Contents were: Buy peacock feathers ---- Actually, you know what would be even better? Making that assertion a bit less clever! As you may remember from <>, I was very pleased with myself for using the `any()` function, but one of my early release readers (thanks, Jason!) suggested a much simpler implementation. We can replace all four lines of the `assertTrue` with a single `assertIn`: [role="sourcecode"] .functional_tests.py (ch05l015) ==== [source,python] ---- self.assertIn("1: Buy peacock feathers", [row.text for row in rows]) ---- ==== Much better. You should always be very worried whenever you think you're being clever, because what you're probably being is _overcomplicated_. Now we get the error message for free: ---- self.assertIn("1: Buy peacock feathers", [row.text for row in rows]) AssertionError: '1: Buy peacock feathers' not found in ['Buy peacock feathers'] ---- Consider me suitably chastened. TIP: If, instead, your FT seems to be saying the table is empty ("not found in ['']"), check your `` tag--does it have the correct `name="item_text"` attribute? And does it have `method="POST"`? Without them, the user's input won't be in the right place in `request.POST`. The point is that the FT wants us to enumerate list items with a "1:" at the beginning of the first list item. The fastest way to get that to pass is with another quick "cheating" change to the template: [role="sourcecode"] .lists/templates/home.html (ch05l016) ==== [source,html] ---- 1: {{ new_item_text }} ---- ==== .When Should You Stop Cheating? DRY Versus Triangulation ******************************************************************************* People often ask about when it's OK to "stop cheating", and change from an implementation we know to be wrong, to one we're happy with.((("Test-Driven Development (TDD)", "concepts", "triangulation"))) ((("triangulation"))) ((("Don't Repeat Yourself (DRY)"))) ((("Test-Driven Development (TDD)", "concepts", "DRY"))) ((("duplication, eliminating"))) One justification is _eliminate duplication_—aka DRY (don’t repeat yourself)—which (with some caveats) is a good guideline for any kind of code. If your test uses a magic constant (like the "1:" in front of our list item), and your application code also uses it, some people say _that_ counts as duplication, so it justifies refactoring. Removing the magic constant from the application code usually means you have to stop cheating. It's a judgement call, but I feel that this is stretching the definition of "repetition" a little, so I often like to use a second technique, which is called _triangulation_: if your tests let you get away with writing "cheating" code that you're not happy with (like returning a magic constant), then _write another test_ that forces you to write some better code. That's what we're doing when we extend the FT to check that we get a "2:" when inputting a second list item. See also <> for a further note of caution on applying DRY too quickly. ******************************************************************************* [role="pagebreak-before"] Now we get to the `self.fail('Finish the test!')`. If we get rid of that and finish writing our FT, to add the check for adding a second item to the table (copy and paste is our friend), we begin to see that our first cut solution really isn't going to, um, [.keep-together]#cut it#: [role="sourcecode"] .functional_tests.py (ch05l017) ==== [source,python] ---- # There is still a text box inviting her to add another item. # She enters "Use peacock feathers to make a fly" # (Edith is very methodical) inputbox = self.browser.find_element(By.ID, "id_new_item") inputbox.send_keys("Use peacock feathers to make a fly") inputbox.send_keys(Keys.ENTER) time.sleep(1) # The page updates again, and now shows both items on her list table = self.browser.find_element(By.ID, "id_list_table") rows = table.find_elements(By.TAG_NAME, "tr") self.assertIn( "2: Use peacock feathers to make a fly", [row.text for row in rows], ) self.assertIn( "1: Buy peacock feathers", [row.text for row in rows], ) # Satisfied, she goes back to sleep ---- ==== ((("", startref="DTtemplate05"))) Sure enough, the FTs return an error: ---- AssertionError: '2: Use peacock feathers to make a fly' not found in ['1: Use peacock feathers to make a fly'] ---- [role="pagebreak-before less_space"] [[three_strikes_and_refactor]] === Three Strikes and Refactor ((("code smell"))) ((("three strikes and refactor rule", id="threestrikes05"))) ((("refactoring", id="refactor05"))) But before we go further--we've got a bad __code smell__footnote:[ If you've not come across the concept, a "code smell" is something about a piece of code that makes you want to rewrite it. Jeff Atwood has https://oreil.ly/GFrNp[a compilation on his blog, _Coding Horror_]. The more experience you gain as a programmer, the more fine-tuned your nose becomes to code smells...] in this FT. We have three almost identical code blocks checking for new items in the list table. ((("Don’t Repeat Yourself (DRY)"))) When we want to apply the DRY principle, I like to follow the motto _three strikes and refactor_. You can copy and paste code once, and it may be premature to try to remove the duplication it causes, but once you get three occurrences, it's time to tidy up. Let's start by committing what we have so far. Even though we know our site has a major flaw--it can only handle one list item--it's still further ahead than it was. We may have to rewrite it all, and we may not, but the rule is that before you do any refactoring, always do a commit: [subs="specialcharacters,quotes"] ---- $ *git diff* # should show changes to functional_tests.py, home.html, # tests.py and views.py $ *git commit -a* ---- TIP: Always do a commit before embarking on a refactor. // TODO: also, make sure the tests are passing? Onto our functional test refactor. Let's use a helper method--remember, only methods that begin with `test_` will be run as tests, so you can use other methods for your own purposes: [role="sourcecode"] .functional_tests.py (ch05l018) ==== [source,python] ---- def tearDown(self): self.browser.quit() def check_for_row_in_list_table(self, row_text): table = self.browser.find_element(By.ID, "id_list_table") rows = table.find_elements(By.TAG_NAME, "tr") self.assertIn(row_text, [row.text for row in rows]) def test_can_start_a_todo_list(self): [...] ---- ==== [role="pagebreak-before"] I like to put helper methods near the top of the class, between the `tearDown` and the first test. Let's use it in the FT: [role="sourcecode"] .functional_tests.py (ch05l019) ==== [source,python] ---- # When she hits enter, the page updates, and now the page lists # "1: Buy peacock feathers" as an item in a to-do list table inputbox.send_keys(Keys.ENTER) time.sleep(1) self.check_for_row_in_list_table("1: Buy peacock feathers") # There is still a text box inviting her to add another item. # She enters "Use peacock feathers to make a fly" # (Edith is very methodical) inputbox = self.browser.find_element(By.ID, "id_new_item") inputbox.send_keys("Use peacock feathers to make a fly") inputbox.send_keys(Keys.ENTER) time.sleep(1) # The page updates again, and now shows both items on her list self.check_for_row_in_list_table("2: Use peacock feathers to make a fly") self.check_for_row_in_list_table("1: Buy peacock feathers") # Satisfied, she goes back to sleep ---- ==== We run the FT again to check that it still behaves in the same way: ---- AssertionError: '2: Use peacock feathers to make a fly' not found in ['1: Use peacock feathers to make a fly'] ---- Good. Now we can commit the FT refactor as its own small, atomic change: [subs="specialcharacters,quotes"] ---- $ *git diff* # check the changes to functional_tests.py $ *git commit -a* ---- There are a couple more bits of duplication in the FTs, like the repetition of finding the `inputbox`, but they're not as egregious yet, so we'll deal with them later. // SEBASTIAN: One could mention there's still an option to cheat and keep items in a list in memory. // I think there's no need to demonstrate it, though. Instead, back to work. If we're ever going to handle more than one list item, we're going to need some kind of persistence, and databases are a stalwart solution in this area. ((("", startref="threestrikes05"))) ((("", startref="refactor05"))) [role="pagebreak-before less_space"] [[django_ORM_first_model]] === The Django ORM and Our First Model ((("object-relational mapper (ORM)", id="orm05"))) ((("Django framework", "object-relational mapper (ORM)", id="DJForm05"))) ((("database testing", "object-relational mapper (ORM)", id="DBTorm05"))) An object-relational mapper (ORM) is a layer of abstraction for data stored in a database with tables, rows, and columns. It lets us work with databases using familiar object-oriented metaphors that work well with code. Classes map to database tables, attributes map to columns, and an individual instance of the class represents a row of data in the database. Django comes with an excellent ORM, and writing a unit test that uses it is actually an excellent way of learning it, because it exercises code by specifying how we want it to work. // SEBASTIAN: This reminds me of (https://github.com/gregmalcolm/python_koans)[Python Koans]. // Perhaps one could link it here as an example of learning with tests Let's create a new class in _lists/tests.py_: [role="sourcecode"] .lists/tests.py (ch05l020) ==== [source,python] ---- from django.test import TestCase from lists.models import Item class HomePageTest(TestCase): [...] class ItemModelTest(TestCase): def test_saving_and_retrieving_items(self): first_item = Item() first_item.text = "The first (ever) list item" first_item.save() second_item = Item() second_item.text = "Item the second" second_item.save() saved_items = Item.objects.all() self.assertEqual(saved_items.count(), 2) first_saved_item = saved_items[0] second_saved_item = saved_items[1] self.assertEqual(first_saved_item.text, "The first (ever) list item") self.assertEqual(second_saved_item.text, "Item the second") ---- ==== You can see that creating a new record in the database is a relatively simple matter of creating an object, assigning some attributes, and calling a `.save()` function. Django also gives us an API for querying the database via a class attribute, `.objects`, and we use the simplest possible query, `.all()`, which retrieves all the records for that table. The results are returned as a list-like object called a `QuerySet`, from which we can extract individual objects, and also call further functions, like `.count()`. We then check the objects as saved to the database, to check whether the right information was saved. ((("Django framework", "tutorials"))) Django's ORM has many other helpful and intuitive features; this might be a good time to skim through the https://docs.djangoproject.com/en/5.2/intro/tutorial01[Django tutorial], which has an excellent intro to them. NOTE: I've written this unit test in a very verbose style, as a way of introducing the Django ORM. I wouldn't recommend writing your model tests like this "in real life", because it's testing the framework, rather than testing our own code. We'll actually rewrite this test to be much more concise in <> (specifically, at <>). .Unit Tests Versus Integration Tests, and the Database ******************************************************************************* ((("unit tests", "versus integration tests", secondary-sortas="integration"))) ((("integration tests", "versus unit tests", secondary-sortas="unit"))) Some people will tell you that a "real" unit test should never touch the database, and that the test I've just written should be more properly called an "integration" test, because it doesn't _only_ test our code, but also relies on an external system--that is, a database. It's OK to ignore this distinction for now--we have two types of test: the high-level FTs, which test the application from the user's point of view, and these lower-level tests, which test it from the programmer's point of view. We'll come back to this topic and talk about the differences between unit tests, integration tests, and more in <>, at the end of the book. ******************************************************************************* Let's try running the unit test. Here comes another unit-test/code cycle: [subs="specialcharacters,macros"] ---- ImportError: cannot import name 'Item' from 'lists.models' ---- Very well, let's give it something to import from 'lists/models.py'. We're feeling confident so we'll skip the `Item = None` step, and go straight to creating a class: [[first-django-model]] [role="sourcecode"] .lists/models.py (ch05l021) ==== [source,python] ---- from django.db import models # Create your models here. class Item: pass ---- ==== [role="pagebreak-before"] That gets our test as far as: ---- [...] File "...goat-book/lists/tests.py", line 25, in test_saving_and_retrieving_items first_item.save() ^^^^^^^^^^^^^^^ AttributeError: 'Item' object has no attribute 'save' ---- To give our `Item` class a `save` method, and to make it into a real Django model, we make it inherit from the `Model` class: [role="sourcecode"] .lists/models.py (ch05l022) ==== [source,python] ---- from django.db import models class Item(models.Model): pass ---- ==== ==== Our First Database Migration ((("database migrations"))) The next thing that happens is a huuuuge traceback, the long and short of which is that there's a problem with the database: ---- django.db.utils.OperationalError: no such table: lists_item ---- In Django, the ORM's job is to model and read and write from database tables, but there's a second system that's in charge of actually _creating_ the tables in the database called "migrations". Its job is to let you add, remove, and modify tables and columns, based on changes you make to your _models.py_ files. One way to think of it is as a version control system (VCS) for your database. As we'll see later, it proves particularly useful when we need to upgrade a database that's deployed on a live server. For now all we need to know is how to build our first database migration, which we do using the `makemigrations` command:footnote:[ If you've done a bit of Django before, you may be wondering about when we're going to run "migrate" as well as "makemigrations"? Read on; that's coming up later in the chapter.] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py makemigrations*] Migrations for 'lists': lists/migrations/0001_initial.py + Create model Item $ pass:quotes[*ls lists/migrations*] 0001_initial.py __init__.py __pycache__ ---- If you're curious, you can go and take a look in the migrations file, and you'll see it's a representation of our additions to 'models.py'. In the meantime, we should find that our tests get a little further. ==== The Test Gets Surprisingly Far The test actually gets surprisingly far: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] [...] self.assertEqual(first_saved_item.text, "The first (ever) list item") ^^^^^^^^^^^^^^^^^^^^^ AttributeError: 'Item' object has no attribute 'text' ---- That's a full eight lines later than the last failure--we've been all the way through saving the two ++Item++s, and we've checked that they're saved in the database, but Django just doesn't seem to have "remembered" the `.text` attribute. If you're new to Python, you might have been surprised that we were allowed to assign the `.text` attribute at all. In a language like Java, you would probably get a compilation error. Python is more relaxed. Classes that inherit from `models.Model` will map to tables in the database. By default, they get an autogenerated `id` attribute, which will be a primary key columnfootnote:[ Database tables usually have a special column called a "primary key", which is the unique identifier for each row in the table.((("primary key"))) It's worth brushing up on a _tiny_ bit of relational database theory, if you're not familiar with the concept or why it's useful.((("relational database theory"))) The top three articles I found when searching for "introduction to databases" all seemed pretty good, at the time of writing.] in the database, but you have to define any other columns and attributes you want explicitly. Here's how we set up a text column: [role="sourcecode"] .lists/models.py (ch05l024) ==== [source,python] ---- class Item(models.Model): text = models.TextField() ---- ==== Django has many other field types, like `IntegerField`, `CharField`, `DateField`, and so on. I've chosen `TextField` rather than `CharField` because the latter requires a length restriction, which seems arbitrary at this point. You can read more on field types in the Django https://docs.djangoproject.com/en/5.2/intro/tutorial02/#creating-models[tutorial] and in the https://docs.djangoproject.com/en/5.2/ref/models/fields[documentation]. [role="pagebreak-before less_space"] ==== A New Field Means a New Migration Running the tests gives us another database error: ---- django.db.utils.OperationalError: table lists_item has no column named text ---- It's because we've added another new field to our database, which means we need to create another migration.((("database migrations", "new field requiring new migration"))) Nice of our tests to let us know! Let's try it: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py makemigrations*] It is impossible to add a non-nullable field 'text' to item without specifying a default. This is because the database needs something to populate existing rows. Please select a fix: 1) Provide a one-off default now (will be set on all existing rows with a null value for this column) 2) Quit and manually define a default value in models.py. Select an option:pass:quotes[*2*] ---- Ah. It won't let us add the column without a default value. Let's pick option 2 and set a default in 'models.py'. I think you'll find the syntax reasonably self-explanatory: [role="sourcecode"] .lists/models.py (ch05l025) ==== [source,python] ---- class Item(models.Model): text = models.TextField(default="") ---- ==== //IDEA: default could get another unit test, which could actually replace the // overly verbose one. And now the migration should complete: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py makemigrations*] Migrations for 'lists': lists/migrations/0002_item_text.py + Add field text to item ---- So, two new lines in 'models.py', two database migrations, and as a result, the `.text` attribute on our model objects is now recognised as a special attribute, so it does get saved to the database, and the tests pass: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] [...] Ran 4 tests in 0.010s OK ---- ((("", startref="orm05"))) ((("", startref="DBTorm05"))) ((("", startref="DJForm05"))) So let's do a commit for our first ever model! [subs="specialcharacters,quotes"] ---- $ *git status* # see tests.py, models.py, and 2 untracked migrations $ *git diff* # review changes to tests.py and models.py $ *git add lists* $ *git commit -m "Model for list Items and associated migration"* ---- === Saving the POST to the Database So, we have a model; now we need to use it! ((("database testing", "HTML POST requests", "saving", id="DTpostsave05"))) ((("HTML", "POST requests", "saving", id="HTMLpostsave05"))) ((("POST requests", "saving", id="POSTsave05"))) Let's adjust the test for our home page POST request, and say we want the view to save a new item to the database instead of just passing it through to its response. We can do that by adding three new lines to the existing test called +test_can_save_a_POST_request+: [role="sourcecode"] .lists/tests.py (ch05l027) ==== [source,python] ---- def test_can_save_a_POST_request(self): response = self.client.post("/", data={"item_text": "A new list item"}) self.assertEqual(Item.objects.count(), 1) # <1> new_item = Item.objects.first() # <2> self.assertEqual(new_item.text, "A new list item") # <3> self.assertContains(response, "A new list item") self.assertTemplateUsed(response, "home.html") ---- ==== <1> We check that one new `Item` has been saved to the database. `objects.count()` is a shorthand for `objects.all().count()`. <2> `objects.first()` is the same as doing `objects.all()[0]`, except it will return `None` if there are no objects.footnote:[ You can also use `objects.get()`, which will immediately raise an exception if there are no objects in the database, or if there are more than one. On the plus side you get a more immediate failure, and you get warned if there are too many objects. The downside is that I find it slightly less readable. As so often, it's a trade-off.] <3> We check that the item's text is correct. [role="pagebreak-before"] ((("unit tests", "length of"))) This test is getting a little long-winded. It seems to be testing lots of different things. That's another _code smell_—a long unit test either needs to be broken into two, or it may be an indication that the thing you're testing is too complicated. Let's add that to a little to-do list of our own, perhaps on a piece of scrap paper: [role="scratchpad"] ***** * 'Code smell: POST test is too long?' ***** .An Alternative Testing Strategy: Staying at the HTTP Level ******************************************************************************* It's a very common pattern in Django to test POST views by asserting on the side effects, as seen in the database. Sandi Metz, a TDD legend from the Ruby world, puts it like this: "test commands via public side effects".footnote:[ This advice is in her talk https://oreil.ly/Gqxgg[The Magic Tricks of Testing], which I highly recommend watching.] But is the database really a public API? That's arguable. Certainly it's at a different level of abstraction, or a different conceptual "layer" in the application, to the HTTP requests we're working with in our current unit tests. If you wanted to write our tests in a way that stays at the HTTP level—that treats the application as more of an "opaque box"—you can prove to yourself that to-do items are persisted, by sending more than one: [role="sourcecode skipme"] .lists/tests/tests.py ==== [source,python] ---- def test_can_save_multiple_items(self): self.client.post("/", data={"item_text": "first item"}) response = self.client.post("/", data={"item_text": "second item"}) self.assertContains(response, "first item") self.assertContains(response, "second item") ---- ==== If you feel like going off road, why not give it a try? //// HARRY NOTES 2023-07-07 had a quick go at a new flow for the chapter based on this idea. https://github.com/hjwp/book-example/tree/chapter_05_post_and_database_possible_alternative def test_post_saves_items(self): self.client.post("/", data={"item_text": "onions"}) response1 = self.client.get("/") self.assertContains(response1, "onions") def test_multiple_posts_save_all_items(self): self.client.post("/", data={"item_text": "onions"}) self.client.post("/", data={"item_text": "carrots"}) response = self.client.get("/") self.assertContains(response, "onions") self.assertContains(response, "carrots") def test_no_items_by_default(self): response = self.client.get("/") empty_table = '
', self.assertContains(response, empty_table, html=True) notes * you can start with just the first test * you can cheat to get this to pass by hardcoding 'onions' in the template obvs * then maybe we add the last test, no items by default * separate calls to get and post eliminates the weird return-things-from-a-post dance, may or may not be a good thing - totally possible to imagine keeping that dance mind you. * then can move on to multiple items * if you want to cheat, you can just use a global variable, but that will never pass the 'no items by default' test * it does end up being a less obvious segue into why use a database tho. because global vars are weirdly less persistent than a db, because the test runner resets the database between each test? that's a lot to explain. overally, definitely intrigued but haven't quite figured out the perfect way to rewrite this chapter. //// ******************************************************************************* [role="pagebreak-before"] Writing things down on a scratchpad like this reassures us that we won't forget them, so we are comfortable getting back to what we were working on. We rerun the tests and see an expected failure: ---- self.assertEqual(Item.objects.count(), 1) AssertionError: 0 != 1 ---- Let's adjust our view: [role="sourcecode"] .lists/views.py (ch05l028) ==== [source,python] ---- from django.shortcuts import render from lists.models import Item def home_page(request): item = Item() item.text = request.POST.get("item_text", "") item.save() return render( request, "home.html", {"new_item_text": request.POST.get("item_text", "")}, ) ---- ==== I've coded a very naive solution and you can probably spot a very obvious problem, which is that we're going to be saving empty items with every request to the home page. Let's add that to our list of things to fix later. You know, along with the painfully obvious fact that we currently have no way at all of having different lists for different people. That we'll keep ignoring for now. Remember, I'm not saying you should always ignore glaring problems like this in "real life". Whenever we spot problems in advance, there's a judgement call to make over whether to stop what you're doing and start again, or leave them until later. Sometimes finishing off what you're doing is still worth it, and sometimes the problem may be so major as to warrant a stop and rethink. Let's see how the unit tests get on... ---- Ran 4 tests in 0.010s OK ---- [role="pagebreak-before"] They pass! Good. Let's have a little look at our scratchpad. I've added a couple of the other things that are on our mind: [role="scratchpad"] ***** * 'Don't save blank items for every request.' * 'Code smell: POST test is too long?' * 'Display multiple items in the table.' * 'Support more than one list!' ***** Let's start with the first scratchpad item: "Don't save blank items for every request". We could tack on an assertion to an existing test, but it's best to keep unit tests to testing one thing at a time, so let's add a new one: [role="sourcecode"] .lists/tests.py (ch05l029) ==== [source,python] ---- class HomePageTest(TestCase): def test_uses_home_template(self): [...] def test_can_save_a_POST_request(self): [...] def test_only_saves_items_when_necessary(self): self.client.get("/") self.assertEqual(Item.objects.count(), 0) ---- ==== // TODO: consider Item.objects.all() == [] instead // and explain why it gives you a nicer error message That gives us a `1 != 0` failure. Let's fix it by bringing the `if request.method` check back and putting the `Item` creation in there: [role="sourcecode"] .lists/views.py (ch05l030) ==== [source,python] ---- def home_page(request): if request.method == "POST": # <1> item = Item() item.text = request.POST["item_text"] # <2> item.save() return render( request, "home.html", {"new_item_text": request.POST.get("item_text", "")}, ) ---- ==== <1> We bring back the `request.method` check. <2> And we can switch from using `request.POST.get()` to `request.POST[]` with square brackets, because we know for sure that the `item_text` key should be in there, and it's better to fail hard if it isn't. ((("", startref="DTpostsave05"))) ((("", startref="HTMLpostsave05"))) ((("", startref="POSTsave05"))) And that gets the test passing: ---- Ran 5 tests in 0.010s OK ---- === Redirect After a POST ((("database testing", "HTML POST requests", "redirect following", id="DThtmlredirect05"))) ((("HTML", "POST requests", "redirect following", id="HTMLpostredirect05"))) ((("POST requests", "redirect following", id="POSTredirect05"))) But, yuck—those duplicated `request.POST` accesses are making me pretty unhappy. Thankfully we are about to have the opportunity to fix it. A view function has two jobs: processing user input and returning an appropriate response. We've taken care of the first part, which is saving the user's input to the database, so now let's work on the second part. https://oreil.ly/yGSl0[Always redirect after a POST], they say, so let's do that. Once again we change our unit test for saving a POST request: instead of expecting a response with the item in it, we want it to expect a redirect back to the home page. [role="sourcecode"] .lists/tests.py (ch05l031) ==== [source,python] ---- def test_can_save_a_POST_request(self): response = self.client.post("/", data={"item_text": "A new list item"}) self.assertEqual(Item.objects.count(), 1) new_item = Item.objects.first() self.assertEqual(new_item.text, "A new list item") self.assertRedirects(response, "/") # <1> def test_only_saves_items_when_necessary(self): [...] ---- ==== <1> We no longer expect a response with HTML content rendered by a template, so we lose the `assertContains` calls that looked at that. Instead, we use Django's `assertRedirects` helper, which checks that we return an HTTP 302 redirect, back to the home URL. [role="pagebreak-before"] That gives us this expected failure: ---- AssertionError: 200 != 302 : Response didn't redirect as expected: Response code was 200 (expected 302) ---- We can now tidy up our view substantially: [role="sourcecode"] .lists/views.py (ch05l032) ==== [source,python] ---- from django.shortcuts import redirect, render from lists.models import Item def home_page(request): if request.method == "POST": item = Item() item.text = request.POST["item_text"] item.save() return redirect("/") return render( request, "home.html", {"new_item_text": request.POST.get("item_text", "")}, ) ---- ==== And the tests should now pass: ---- Ran 5 tests in 0.010s OK ---- We're at green; time for a little refactor! Let's have a look at _views.py_ and see what opportunities for improvement there might be: [role="sourcecode currentcontents"] .lists/views.py ==== [source,python] ---- def home_page(request): if request.method == "POST": item = Item() # <1> item.text = request.POST["item_text"] # <1> item.save() # <1> return redirect("/") return render( request, "home.html", {"new_item_text": request.POST.get("item_text", "")}, # <2> ) ---- ==== <1> There's a quicker way to do these three lines with `.objects.create()`. <2> This line doesn't seem quite right now; in fact, it won't work at all. Let's make a note on our scratchpad to sort out passing list items to the template. It's actually closely related to "Display multiple items", so we'll put it just before that one: [role="scratchpad"] ***** * '[strikethrough line-through]#Don't save blank items for every request.#' * 'Code smell: POST test is too long?' * 'Pass existing list items to the template somehow.' * 'Display multiple items in the table.' * 'Support more than one list!' ***** And here's the refactored version of _views.py_ using the `.objects.create()` helper method that Django provides, for one-line creation of objects: [role="sourcecode"] .lists/views.py (ch05l033) ==== [source,python] ---- def home_page(request): if request.method == "POST": Item.objects.create(text=request.POST["item_text"]) return redirect("/") return render( request, "home.html", {"new_item_text": request.POST.get("item_text", "")}, ) ---- ==== [role="pagebreak-before less_space"] === Better Unit Testing Practice: Each Test Should Test [.keep-together]#One Thing# ((("unit tests", "testing only one thing"))) ((("testing best practices")))((("POST requests", "POST test is too long code smell, addressing"))) Let's address the "POST test is too long" code smell. Good unit testing practice says that each test should only test one thing. The reason is that it makes it easier to track down bugs. Having multiple assertions in a test means that, if the test fails on an early assertion, you don't know what the statuses of the later assertions are. As we'll see in the next chapter, if we ever break this view accidentally, we want to know whether it's the saving of objects that's broken, or the type of response. You may not always write perfect unit tests with single assertions on your first go, but now feels like a good time to separate out our concerns: [role="sourcecode"] .lists/tests.py (ch05l034) ==== [source,python] ---- def test_can_save_a_POST_request(self): self.client.post("/", data={"item_text": "A new list item"}) self.assertEqual(Item.objects.count(), 1) new_item = Item.objects.first() self.assertEqual(new_item.text, "A new list item") def test_redirects_after_POST(self): response = self.client.post("/", data={"item_text": "A new list item"}) self.assertRedirects(response, "/") ---- ==== ((("", startref="HTMLpostredirect05"))) ((("", startref="DThtmlredirect05"))) ((("", startref="POSTredirect05"))) And we should now see six tests pass instead of five: ---- Ran 6 tests in 0.010s OK ---- [role="pagebreak-before less_space"] === Rendering Items in the Template ((("database testing", "rendering items in the template", id="DTrender05"))) Much better! Back to our to-do list: [role="scratchpad"] ***** * '[strikethrough line-through]#Don't save blank items for every request.#' * '[strikethrough line-through]#Code smell: POST test is too long?#' * 'Pass existing list items to the template somehow.' * 'Display multiple items in the table.' * 'Support more than one list!' ***** Crossing things off the list is almost as satisfying as seeing tests pass! The third and fourth items are the last of the "easy" ones. Our view now does the right thing for POST requests; it saves new list items to the database. Now we want GET requests to load all currently existing list items, and pass them to the template for rendering. Let's have a new unit test for that: [role="sourcecode"] .lists/tests.py (ch05l035) ==== [source,python] ---- class HomePageTest(TestCase): def test_uses_home_template(self): [...] def test_renders_input_form(self): [...] def test_displays_all_list_items(self): Item.objects.create(text="itemey 1") Item.objects.create(text="itemey 2") response = self.client.get("/") self.assertContains(response, "itemey 1") self.assertContains(response, "itemey 2") def test_can_save_a_POST_request(self): [...] ---- ==== [role="pagebreak-before less_space"] .Arrange-Act-Assert or Given-When-Then ******************************************************************************* Did you notice the use of whitespace in this test? I'm visually separating out the code into three blocks: ((("Arrange, Act, Assert"))) ((("Given / When / Then"))) [role="sourcecode currentcontentss"] .lists/tests.py ==== [source,python] ---- def test_displays_all_list_items(self): Item.objects.create(text="itemey 1") # <1> Item.objects.create(text="itemey 2") # <1> response = self.client.get("/") # <2> self.assertContains(response, "itemey 1") # <3> self.assertContains(response, "itemey 2") # <3> ---- ==== <1> Arrange: where we set up the data we need for the test. <2> Act: where we call the code under test <3> Assert: where we check on the results This isn't obligatory, but it's a common convention, and it does help see the structure of the test. Another popular way to talk about this structure is _given-when-then_: * _Given_ the database contains our list with two items, * _When_ I make a GET request for our list, * _Then_ I see the both items in our list. This latter phrasing comes from the world of behaviour-driven development (BDD), and I actually prefer it somewhat.((("BDD (behaviour-driven development)"))) You can see that it encourages phrasing things in a more natural way, and we're gently nudged to think of things in terms of behaviour and the perspective of the user. ******************************************************************************* That fails as expected: ---- AssertionError: False is not true : Couldn't find 'itemey 1' in the following response b'\n \n To-Do lists\n \n \n [...] ---- [role="pagebreak-before"] ((("templates", "tags", "{% for ... endfor %}"))) ((("{% for ... endfor %}"))) The Django template syntax has a tag for iterating through lists, `{% for .. in .. %}`; we can use it like this: [role="sourcecode"] .lists/templates/home.html (ch05l036) ==== [source,html] ---- {% for item in items %} {% endfor %}
1: {{ item.text }}
---- ==== This is one of the major strengths of the templating system. Now the template will render with multiple `` rows, one for each item in the variable `items`. Pretty neat! I'll introduce a few more bits of Django template magic as we go, but at some point you'll want to go and read up on the rest of them in the https://docs.djangoproject.com/en/5.2/topics/templates[Django docs]. Just changing the template doesn't get our tests to green; we need to actually pass the items to it from our home page view: [role="sourcecode"] .lists/views.py (ch05l037) ==== [source,python] ---- def home_page(request): if request.method == "POST": Item.objects.create(text=request.POST["item_text"]) return redirect("/") items = Item.objects.all() return render(request, "home.html", {"items": items}) ---- ==== That does get the unit tests to pass. Moment of truth...will the functional test pass? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] [...] AssertionError: 'To-Do' not found in 'OperationalError at /' ---- [role="pagebreak-before"] ((("", startref="DTrender05"))) ((("debugging", "using manual visits to the site"))) Oops, apparently not. Let's use another FT debugging technique, and it's one of the most straightforward: manually visiting the site! Open up pass:[http://localhost:8000] in your web browser, and you'll see a Django debug page saying "no such table: lists_item", as in <>. [[operationalerror]] [role="width-75"] .Another helpful debug message image::images/tdd3_0503.png["Screenshot of Django debug page, saying OperationalError at / no such table: lists_item"] [role="pagebreak-before less_space"] === Creating Our Production Database with migrate ((("database testing", "production database creation", id="DTproduction05"))) ((("database migrations"))) So, we've got another helpful error message from Django, which is basically complaining that we haven't set up the database properly. How come everything worked fine in the unit tests, I hear you ask? Because Django creates a special 'test database' for unit tests; it's one of the magical things that Django's `TestCase` does. To set up our "real" database, we need to explicitly create it. SQLite databases are just a file on disk, and you'll see in 'settings.py' that Django, by default, will just put it in a file called 'db.sqlite3' in the base project directory: [role="sourcecode currentcontents"] .superlists/settings.py ==== [source,python] ---- [...] # Database # https://docs.djangoproject.com/en/5.2/ref/settings/#databases DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", "NAME": BASE_DIR / "db.sqlite3", } } ---- ==== We've told Django everything it needs to create the database, first via 'models.py' and then when we created the migrations file. To actually apply it to creating a real database, we use another Django Swiss Army knife 'manage.py' command, `migrate`: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py migrate*] Operations to perform: Apply all migrations: admin, auth, contenttypes, lists, sessions Running migrations: Applying contenttypes.0001_initial... OK Applying auth.0001_initial... OK Applying admin.0001_initial... OK Applying admin.0002_logentry_remove_auto_add... OK Applying admin.0003_logentry_add_action_flag_choices... OK Applying contenttypes.0002_remove_content_type_name... OK Applying auth.0002_alter_permission_name_max_length... OK Applying auth.0003_alter_user_email_max_length... OK Applying auth.0004_alter_user_username_opts... OK Applying auth.0005_alter_user_last_login_null... OK Applying auth.0006_require_contenttypes_0002... OK Applying auth.0007_alter_validators_add_error_messages... OK Applying auth.0008_alter_user_username_max_length... OK Applying auth.0009_alter_user_last_name_max_length... OK Applying auth.0010_alter_group_name_max_length... OK Applying auth.0011_update_proxy_permissions... OK Applying auth.0012_alter_user_first_name_max_length... OK Applying lists.0001_initial... OK Applying lists.0002_item_text... OK Applying sessions.0001_initial... OK ---- It seems to be doing quite a lot of work! That's because it's the first ever migration, and Django is creating tables for all its built-in "batteries included" apps, like the admin site and the built-in auth modules. We don't need to pay attention to them for now. But you can see our `lists.0001_initial` and `lists.0002_item_text` in there! At this point, you can refresh the page on _localhost_ and see that the error is gone. Let's try running the functional tests again:footnote:[ If you get a different error at this point, try restarting your dev server--it may have gotten confused by the changes to the database happening under its feet.] // DAVID: FWIW I'm not sure how this might happen - interested to know // if you have a real example of someone running into this problem. ---- AssertionError: '2: Use peacock feathers to make a fly' not found in ['1: Buy peacock feathers', '1: Use peacock feathers to make a fly'] ---- So close!((("templates", "tags", "{{ forloop.counter }}"))) We just need to get our list numbering right. Another awesome Django template tag, `forloop.counter`, will help here: [role="sourcecode"] .lists/templates/home.html (ch05l038) ==== [source,html] ---- {% for item in items %} {{ forloop.counter }}: {{ item.text }} {% endfor %} ---- ==== [role="pagebreak-before"] If you try it again, you should now see the FT gets to the end: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python functional_tests.py*] . --------------------------------------------------------------------- Ran 1 test in 5.036s OK ---- Hooray! But, as it's running, you may notice something is amiss, like in <>. [[items_left_over_from_previous_run]] .There are list items left over from the last run of the test image::images/tdd3_0504.png["There are list items left over from the last run of the test"] Oh dear. It looks like previous runs of the test are leaving stuff lying around in our database. In fact, if you run the tests again, you'll see it gets worse: [role="skipme"] ---- 1: Buy peacock feathers 2: Use peacock feathers to make a fly 3: Buy peacock feathers 4: Use peacock feathers to make a fly 5: Buy peacock feathers 6: Use peacock feathers to make a fly ---- Grrr. We're so close! We're going to need some kind of automated way of tidying up after ourselves. For now, if you feel like it, you can do it manually by deleting the database and re-creating it fresh with `migrate` (you'll need to shut down your Django server first): [subs="specialcharacters,quotes"] ---- $ *rm db.sqlite3* $ *python manage.py migrate --noinput* ---- And then (after restarting your server!) reassure yourself that the FT still passes. Apart from that little bug in our functional testing, we've got some code that's more or less working. Let's do a commit. ((("", startref="DTproduction05"))) Start by doing a `git status` and a `git diff`, and you should see changes to 'home.html', 'tests.py', and 'views.py'. Let's add them: [subs="specialcharacters,quotes"] ---- $ *git add lists* $ *git commit -m "Redirect after POST, and show all items in template"* ---- TIP: You might find it useful to add markers for the end of each chapter, like *`git tag end-of-chapter-05`*. === Recap Where are we? How is progress on our app, and what have we learned? * We've got a form set up to add new items to the list using POST. * We've set up a simple model in the database to save list items. * We've learned about creating database migrations, both for the test database (where they're applied automatically) and for the real database (where we have to apply them manually). * We've used our first couple of Django template tags: `{% csrf_token %}` and the `{% for ... endfor %}` loop. * And we've used two different FT debugging techniques: ++time.sleep++s, and improving the error messages. But we've got a couple of items on our own to-do list, namely getting the FT to clean up after itself, and perhaps more critically, adding support for more than one list: [role="scratchpad"] ***** * '[strikethrough line-through]#Don't save blank items for every request.#' * '[strikethrough line-through]#Code smell: POST test is too long?#' * '[strikethrough line-through]#Pass existing list items to the template somehow.#' * '[strikethrough line-through]#Display multiple items in the table.#' * 'Clean up after FT runs.' * 'Support more than one list!' ***** I mean, we _could_ ship the site as it is, but people might find it strange that the entire human population has to share a single to-do list. I suppose it might get people to stop and think about how connected we all are to one another, how we all share a common destiny here on Spaceship Earth, and how we must all work together to solve the global problems that we face. But in practical terms, the site wouldn't be very useful. Ah well. // (("", startref="UIdatabase05")) .Useful TDD Concepts ******************************************************************************* Regression:: When a change unexpectedly breaks some aspect of the application that used to work. ((("Test-Driven Development (TDD)", "concepts", "regression"))) ((("regression"))) Unexpected failure:: When a test fails in a way we weren't expecting. This either means that we've made a mistake in our tests, or that the tests have helped us find a regression, and we need to fix something in our code. ((("Test-Driven Development (TDD)", "concepts", "unexpected failures"))) ((("unexpected failures"))) Triangulation:: Adding a test case with a new specific example for some existing code, to justify generalising the implementation (which may be a "cheat" until that point). ((("Test-Driven Development (TDD)", "concepts", "triangulation"))) ((("triangulation"))) Three strikes and refactor:: A rule of thumb for when to remove duplication from code. When two pieces of code look very similar, it often pays to wait until you see a third use case, so that you're more sure about what part of the code really is the common, reusable part to refactor out. ((("Test-Driven Development (TDD)", "concepts", "three strikes and refactor"))) ((("three strikes and refactor rule"))) The scratchpad to-do list:: A place to write down things that occur to us as we're coding, so that we can finish up what we're doing and come back to them later. Love a good old-fashioned piece of paper now and again! ((("Test-Driven Development (TDD)", "concepts", "scratchpad to-do list"))) ((("scratchpad to-do list"))) // SEBASTIAN: (idea) alternative to maintaining a scratchpad could be to write empty unit tests without implementation. // Such "tests prototypes" could be skipped initially until we work on them. ******************************************************************************* ================================================ FILE: chapter_06_explicit_waits_1.asciidoc ================================================ [[chapter_06_explicit_waits_1]] == Improving Functional Tests: Ensuring Isolation and Removing Magic Sleeps Before we dive in and fix our single-global-list problem, let's take care of a couple of housekeeping items. At the end of the last chapter, we made a note that different test runs were interfering with each other, so we'll fix that. I'm also not happy with all these ++time.sleep++s peppered through the code; they seem a bit unscientific, so we'll replace them with something more reliable: [role="scratchpad"] ***** * _Clean up after FT runs._ * _Remove time.sleeps._ ***** Both of these changes will be moving us towards testing "best practices", making our tests more deterministic and more reliable. [role="pagebreak-before less_space"] === Ensuring Test Isolation in Functional Tests ((("functional tests (FTs)", "ensuring isolation", id="FTisolation06"))) ((("isolation of tests", "ensuring in functional tests", id="ix_isoFTs"))) We ended the last chapter with a classic testing problem: how to ensure _isolation_ between tests. Each run of our functional tests (FTs) left list items lying around in the database, and that interfered with the test results when next running the tests. ((("unit tests", "in Django", "test databases", secondary-sortas="Django"))) When we run _unit_ tests, the Django test runner automatically creates a brand new test database (separate from the real one), which it can safely reset before each individual test is run, and then thrown away at the end. But our FTs currently run against the "real" database, _db.sqlite3_. One way to tackle this would be to "roll our own" solution, and add some code to _functional_tests.py_, which would do the cleaning up. The `setUp` and `tearDown` methods are perfect for this sort of thing. ((("LiveServerTestCase"))) But as this is a common problem, Django supplies a test class called `LiveServerTestCase` that addresses this issue. It will automatically create a test database (just like in a unit test run) and start up a development server for the FTs to run against.((("database testing", "creating test database automatically"))) Although as a tool it has some limitations, which we'll need to work around later, it's dead useful at this stage, so let's check it out. `LiveServerTestCase` expects to be run by the Django test runner using _manage.py_, which will run tests from any files whose name begins with _test__. To keep things neat and tidy, let's make a folder for our FTs, so that it looks a bit like an app. All Django needs is for it to be a valid Python package directory (i.e., one with a +++___init___.py+++ [.keep-together]#in it#): [subs=""] ---- $ mkdir functional_tests $ touch functional_tests/__init__.py ---- ((("Git", "moving files"))) Now we want to 'move' our functional tests, from being a standalone file called 'functional_tests.py', to being the 'tests.py' of the `functional_tests` app. We use `git mv` so that Git keeps track of the fact that this is the same file and should have a single history. [subs=""] ---- $ git mv functional_tests.py functional_tests/tests.py $ git status # shows the rename to functional_tests/tests.py and __init__.py ---- [role="pagebreak-before"] At this point, your directory tree should look like this: ---- . ├── db.sqlite3 ├── functional_tests │   ├── __init__.py │   └── tests.py ├── lists │   ├── __init__.py │   ├── admin.py │   ├── apps.py │   ├── migrations │   │   ├── 0001_initial.py │   │   ├── 0002_item_text.py │   │   └── __init__.py │   ├── models.py │   ├── templates │   │   └── home.html │   ├── tests.py │   └── views.py ├── manage.py └── superlists ├── __init__.py ├── asgi.py ├── settings.py ├── urls.py └── wsgi.py ---- 'functional_tests.py' is gone, and has turned into 'functional_tests/tests.py'. Now, whenever we want to run our FTs, instead of running `python functional_tests.py`, we will use `python manage.py test functional_tests`. NOTE: You could mix your functional tests into the tests for the `lists` app. I tend to prefer keeping them separate, because FTs usually have cross-cutting concerns that run across different apps. FTs are meant to see things from the point of view of your users, and your users don't care about how you've split work between different apps! [role="pagebreak-before"] Now, let's edit 'functional_tests/tests.py' and change our `NewVisitorTest` class to make it use `LiveServerTestCase`: [role="sourcecode"] .functional_tests/tests.py (ch06l001) ==== [source,python] ---- from django.test import LiveServerTestCase from selenium import webdriver [...] class NewVisitorTest(LiveServerTestCase): def setUp(self): [...] ---- ==== Next, instead of hardcoding the visit to localhost port `8000`, `LiveServerTestCase` gives us an attribute called `live_server_url`: [role="dofirst-ch06l003 sourcecode"] .functional_tests/tests.py (ch06l002) ==== [source,python] ---- def test_can_start_a_todo_list(self): # Edith has heard about a cool new online to-do app. # She goes to check out its homepage self.browser.get(self.live_server_url) ---- ==== We can also remove the `if __name__ == '__main__'` from the end if we want, as we'll be using the Django test runner to launch the FT. Now we are able to run our functional tests using the Django test runner, by telling it to run just the tests for our new `functional_tests` app: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests*] Creating test database for alias 'default'... Found 1 test(s). System check identified no issues (0 silenced). . --------------------------------------------------------------------- Ran 1 test in 10.519s OK Destroying test database for alias 'default'... ---- NOTE: When I ran this test today, I ran into the ((("Firefox", "upgrading")))Firefox upgrade pop-up. Just a little reminder, in case you happen to see it too, we talked about it in <> in a little <>. [role="pagebreak-before"] The FT still passes, reassuring us that our refactor didn't break anything. You'll also notice that if you run the tests a second time, there aren't any old list items lying around from the previous test--it has cleaned up after itself. Success! We should commit it as an atomic change: [subs=""] ---- $ git status # functional_tests.py renamed + modified, new __init__.py $ git add functional_tests $ git diff --staged $ git commit # msg eg "make functional_tests an app, use LiveServerTestCase" ---- ==== Running Just the Unit Tests ((("Django framework", "running functional and/or unit tests"))) Now if we run `manage.py test`, Django will run both the functional and the unit tests: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] Creating test database for alias 'default'... Found 8 test(s). System check identified no issues (0 silenced). ........ --------------------------------------------------------------------- Ran 8 tests in 10.859s OK Destroying test database for alias 'default'... ---- ((("", startref="FTisolation06"))) ((("isolation of tests", "ensuring in functional tests", startref="ix_isoFTs"))) To run just the unit tests, we can specify that we want to only run the tests for the `lists` app: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] Creating test database for alias 'default'... Found 7 test(s). System check identified no issues (0 silenced). ....... --------------------------------------------------------------------- Ran 7 tests in 0.009s OK Destroying test database for alias 'default'... ---- [role="pagebreak-before less_space"] .Useful Commands Updated ******************************************************************************* ((("Django framework", "commands and concepts", "python manage.py test functional_tests")))To run the functional tests:: *`python manage.py test functional_tests`* ((("Django framework", "commands and concepts", "python manage.py test lists")))To run the unit tests:: *`python manage.py test lists`* What to do if I say "run the tests", and you're not sure which ones I mean? Have another look at the flowchart at the end of <>, and try to figure out where we are. As a rule of thumb, we usually only run the FTs once all the unit tests are passing, so if in doubt, try both! ******************************************************************************* === On Implicit and Explicit Waits, and Magic time.sleeps ((("functional tests (FTs)", "implicit/explicit waits and time.sleeps", id="FTimplicit06")))((("waits", "explicit and implicit and time.sleeps", id="ix_wait"))) ((("implicit and explicit waits", id="implicit06"))) ((("explicit and implicit waits", id="explicit06"))) ((("time.sleeps", "removing magic sleeps", id="ix_tmslprmv"))) Let's talk about the `time.sleep` in our FT: [role="sourcecode currentcontents"] .functional_tests/tests.py ==== [source,python] ---- # When she hits enter, the page updates, and now the page lists # "1: Buy peacock feathers" as an item in a to-do list table inputbox.send_keys(Keys.ENTER) time.sleep(1) self.check_for_row_in_list_table("1: Buy peacock feathers") ---- ==== This is what's called an "explicit wait". That's in contrast with "implicit waits": in certain cases, Selenium tries to wait "automatically" for you when it thinks the page is loading. It even provides a method called `implicitly_wait` that lets you control how long it will wait if you ask it for an element that doesn't seem to be on the page yet. In fact, in the first edition of this book, I was able to rely entirely on implicit waits. The problem is that implicit waits are always a little flakey, and with the release of Selenium 4, implicit waits were disabled by default. At the same time, the general opinion from the Selenium team is that implicit waits are just a bad idea, and https://www.selenium.dev/documentation/webdriver/waits[should be avoided]. So this edition has explicit waits from the very beginning. But the problem is that those ++time.sleep++s have their own issues. Currently we're waiting for one second, but who's to say that's the right amount of time? For most tests we run against our own machine, one second is way too long, and it's going to really slow down our FT runs. 0.1s would be fine. But the problem is that if you set it that low, every so often you're going to get a spurious failure because, for whatever reason, the laptop was being a bit slow just then. And even at one second, there’s still a chance of random failures that don’t indicate a real problem—and false positives in tests are a real annoyance.footnote:[There's lots more on this in https://oreil.ly/YdRx-[an article by Martin Fowler].] [TIP] ==== Unexpected `NoSuchElementException` and `StaleElementException` errors are often a sign that you need an explicit wait.((("NoSuchElementException")))((("StaleElementException"))) ==== So let's replace our sleeps with a tool that will wait for just as long as is needed, up to a nice long timeout to catch any glitches. We'll rename `check_for_row_in_list_table` to `wait_for_row_in_list_table`, and add some polling/retry logic to it: [role="sourcecode"] .functional_tests/tests.py (ch06l004) ==== [source,python] ---- [...] from selenium.common.exceptions import WebDriverException import time MAX_WAIT = 5 # <1> class NewVisitorTest(LiveServerTestCase): def setUp(self): [...] def tearDown(self): [...] def wait_for_row_in_list_table(self, row_text): start_time = time.time() while True: # <2> try: table = self.browser.find_element(By.ID, "id_list_table") # <3> rows = table.find_elements(By.TAG_NAME, "tr") self.assertIn(row_text, [row.text for row in rows]) return # <4> except (AssertionError, WebDriverException): # <5> if time.time() - start_time > MAX_WAIT: # <6> raise # <6> time.sleep(0.5) # <5> ---- ==== [role="pagebreak-before"] <1> We'll use a constant called `MAX_WAIT` to set the maximum amount of time we're prepared to wait. Five seconds should be enough to catch any glitches or random slowness. <2> Here's the loop, which will keep going forever, unless we get to one of two possible exit routes. <3> Here are our three lines of assertions from the old version of the method. <4> If we get through them, and our assertion passes, we return from the function and escape the loop. <5> But if we catch an exception, we wait a short amount of time and loop around to retry. There are two types of exceptions we want to catch: `WebDriverException` for when the page hasn't loaded and Selenium can't find the table element on the page; and `AssertionError` for when the table is there, but it's perhaps a table from before the page reloads, so it doesn't have our row in yet.((("WebDriverException")))((("AssertionError"))) <6> Here's our second escape route. If we get to this point, that means our code kept raising exceptions every time we tried it until we exceeded our timeout. So this time, we reraise the exception and let it bubble up to our test, and most likely end up in our traceback, telling us why the test failed. Are you thinking this code is a little ugly, and makes it a bit harder to see exactly what we're doing? I agree. Later on (<>), we'll refactor out a general `wait_for` helper, to separate the timing and reraising logic from the test assertions. But we'll wait until we need it in multiple places. NOTE: If you've used Selenium before, you may know that it has a few https://www.selenium.dev/documentation/webdriver/waits/#explicit-waits[helper functions to conduct waits]. I'm not a big fan of them, though not for any objective reason really. Over the course of the book, we'll build a couple of wait helper tools, which I think will make for nice and readable code.((("Selenium", "helper functions to conduct waits"))) But of course you should check out the homegrown Selenium waits in your own time, and see if you prefer them. [role="pagebreak-before"] Now we can rename our method calls, and remove the magic ++time.sleep++s: [role="sourcecode"] .functional_tests/tests.py (ch06l005) ==== [source,python] ---- [...] # When she hits enter, the page updates, and now the page lists # "1: Buy peacock feathers" as an item in a to-do list table inputbox.send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Buy peacock feathers") # There is still a text box inviting her to add another item. # She enters "Use peacock feathers to make a fly" # (Edith is very methodical) inputbox = self.browser.find_element(By.ID, "id_new_item") inputbox.send_keys("Use peacock feathers to make a fly") inputbox.send_keys(Keys.ENTER) # The page updates again, and now shows both items on her list self.wait_for_row_in_list_table("2: Use peacock feathers to make a fly") self.wait_for_row_in_list_table("1: Buy peacock feathers") [...] ---- ==== And rerun the tests: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] Creating test database for alias 'default'... Found 8 test(s). System check identified no issues (0 silenced). ........ --------------------------------------------------------------------- Ran 8 tests in 4.552s OK Destroying test database for alias 'default'... ---- Hooray we're back to passing, and notice we've shaved a few of seconds off the execution time too. That might not seem like a lot right now, but it all adds up. Just to check we've done the right thing, let's deliberately break the test in a couple of ways and see some errors. First, let’s try searching for some text that we know isn’t there, and check that we get the expected error: [role="sourcecode"] .functional_tests/tests.py (ch06l006) ==== [source,python] ---- def wait_for_row_in_list_table(self, row_text): [...] rows = table.find_elements(By.TAG_NAME, "tr") self.assertIn("foo", [row.text for row in rows]) return ---- ==== [role="pagebreak-before"] We see we still get a nice self-explanatory test failure message: [subs="specialcharacters,macros"] ---- self.assertIn("foo", [row.text for row in rows]) AssertionError: 'foo' not found in ['1: Buy peacock feathers'] ---- NOTE: Did you get a bit bored waiting five seconds for the test to fail? That's one of the downsides of explicit waits. There's a tricky trade-off between waiting long enough that little glitches don't throw you, versus waiting so long that expected failures are painfully slow to watch. Making `MAX_WAIT` configurable so that it's fast in local dev, but more conservative on continuous integration (CI) servers can be a good idea. See <> for an introduction to CI. Let's put that back the way it was and break something else: [role="sourcecode"] .functional_tests/tests.py (ch06l007) ==== [source,python] ---- try: table = self.browser.find_element(By.ID, "id_nothing") rows = table.find_elements(By.TAG_NAME, "tr") self.assertIn(row_text, [row.text for row in rows]) return [...] ---- ==== Sure enough, we get the errors for when the page doesn't contain the element we're looking for too: ---- selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_nothing"]; For documentation on this error, [...] ---- Everything seems to be in order. Let's put our code back to the way it should be, and do one final test run: [role="dofirst-ch06l008"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test*] [...] OK ---- Great. With that little interlude over, let's crack on with getting our application actually working for multiple lists. Don't forget to commit first! ((("waits", "explicit and implicit and time.sleeps", startref="ix_wait")))((("", startref="FTimplicit06"))) ((("", startref="implicit06"))) ((("", startref="explicit06"))) ((("time.sleeps", "removing magic sleeps", startref="ix_tmslprmv"))) [role="pagebreak-before less_space"] .Testing "Best Practices" Applied in this Chapter ******************************************************************************* Ensuring test isolation and managing global state:: Different tests shouldn't affect one another. This means we need to reset any permanent state at the end of each test. Django's test runner helps us do this by creating a test database, which it wipes clean in between each test. ((("testing best practices"))) Avoid "magic" sleeps:: Whenever we need to wait for something to load, it's always tempting to throw in a quick-and-dirty `time.sleep`. But the problem is that the length of time we wait is always a bit of a shot in the dark, either too short and too vulnerable to spurious failures, or too long and it'll slow down our test runs. Prefer a retry loop that polls our app and moves on as soon as possible. Don't rely on Selenium's implicit waits:: Selenium does theoretically do some "implicit" waits, but the implementation varies between browsers, and is not always reliable.((("Selenium", "implicit waits, avoiding"))) "Explicit is better than implicit", as the Zen of Python says,footnote:[`python -c "import this"`] so prefer explicit waits. ******************************************************************************* ================================================ FILE: chapter_07_working_incrementally.asciidoc ================================================ [[chapter_07_working_incrementally]] == Working Incrementally ((("Test-Driven Development (TDD)", "adapting existing code incrementally", id="TDDadapt07"))) ((("Testing Goat", "working state to working state"))) Now let's address our real problem, which is that our design only allows for one global list. In this chapter I'll demonstrate a critical TDD technique: how to adapt existing code using an incremental, step-by-step process that takes you from working state to working state. Testing Goat, not Refactoring Cat! === Small Design When Necessary ((("small vs. big design", id="small07"))) ((("multiple lists testing", "small vs. big design", id="MLTsmall07"))) Let's have a think about how we want support for multiple lists to work. At the moment, the only URL for our site is the home page, and that's why there's only one global list. The most obvious way to support multiple lists is to say that each list gets its own URL, so that people can start multiple lists, or so that different people can have different lists. How might that work? ==== Not Big Design Up Front ((("agile movement"))) ((("Big Design Up Front"))) ((("minimum viable applications"))) TDD is closely associated with the Agile movement in software development, which includes a reaction against “_big design up front_”—the traditional software engineering practice whereby, after a lengthy requirements-gathering exercise, there is an equally lengthy design stage where the software is planned out on paper. The Agile philosophy is that you learn more from solving problems in practice than in theory, especially when you confront your application with real users as soon as possible. Instead of a long up-front design phase, we try to put a _minimum viable product_ out there early, and let the design evolve gradually based on feedback from real-world usage. [role="pagebreak-before"] But that doesn't mean that thinking about design is outright banned! In <>, we saw how just blundering ahead without thinking can _eventually_ get us to the right answer, but often a little thinking about design can help us get there faster. So, let's think about our minimum viable lists app, and what kind of design we'll need to deliver it: * We want each user to be able to store their own list--at least one, for now. * A list is made up of several items, whose primary attribute is a bit of descriptive text. * We need to save lists from one visit to the next. For now, we can give each user a unique URL for their list. Later on, we may want some way of automatically recognising users and showing them their lists. To deliver the "for now" items, we're going to have to store lists and their items in a database. Each list will have a unique URL, and each list item will be a bit of descriptive text, associated with a particular list—something like <>. [[multiple-lists-users-and-urls]] .Multiple users with multiple lists at multiple URLs image::images/tdd3_0701.png["An illustration showing two users looking at two different lists with different items, at different URLs."] ==== YAGNI! ((("Test-Driven Development (TDD)", "philosophy of", "YAGNI"))) ((("YAGNI (You ain't gonna need it!)"))) Once you start thinking about design, it can be hard to stop. All sorts of other thoughts are occurring to us--we might want to give each list a name or title, we might want to recognise users using usernames and passwords, we might want to add a longer notes field as well as short descriptions to our list, we might want to store some kind of ordering, and so on. But we should obey another tenet of the Agile gospel: YAGNI (pronounced yag-knee), which stands for "You ain't gonna need it!" As software developers, we have fun creating, and sometimes it's hard to resist the urge to build things just because an idea occurred to us and we _might_ need it. The trouble is that more often than not, no matter how cool the idea was, you _won't_ end up using it. Instead you just end up with a load of unused code, adding to the complexity of your application. YAGNI is the motto we use to resist our overenthusiastic creative urges. We avoid writing any code that's not strictly required. TIP: Don't write any code unless you absolutely have to.footnote:[ This is a much more widely applicable rule for programming in business, actually. If you can solve a problem without any coding at all, that's a big win.] ==== REST-ish ((("Representational State Transfer (REST)", "inspiration gained from"))) ((("model-view-controller (MVC) pattern")))((("MVC (model-view-controller) pattern"))) We have an idea of the data structure we want--the "model" part of model-view-controller (we talked about MVC in <>). What about the "view" and "controller" parts? How should the user interact with ++List++s and their ++Item++s using a web browser? Representational state transfer (REST) is an approach to web design that's usually used to guide the design of web-based APIs. When designing a user-facing site, it's not possible to stick _strictly_ to the REST rules, but they still provide some useful inspiration (take a look at https://www.obeythetestinggoat.com/book/appendix_rest_api.html[Online Appendix: Building a REST API] if you want to see a real REST API). REST suggests that we have a URL structure that matches our data structure—in this case, lists and list items. Each list can have its own URL: [role="skipme"] ---- /lists// ---- To view a list, we use a GET request (a normal browser visit to the page). To create a brand new list, we'll have a special URL that accepts POST requests: [role="skipme"] ---- /lists/new ---- // DAVID: for consistency, personally I would add trailing slashes to all the URLs. // SEBASTIAN: Why not just POST /lists/ ? // Unless it's a URL for a view with a form! To add a new item to an existing list, we'll have a separate URL, to which we can send POST requests: [role="skipme"] ---- /lists//add_item ---- // DAVID: I would use kebab case for URLs -> /add-item/ // SEBASTIAN: Why not just POST /lists//item ? // Unless it's a URL for a view with a form! (Again, we're not trying to perfectly follow the rules of REST, which would use a PUT request here--we're just using REST for inspiration. Apart from anything else, you can't use PUT in a standard HTML form.) [role="pagebreak-before"] ((("", startref="small07"))) ((("", startref="MLTsmall07"))) In summary, our scratchpad for this chapter looks something like this: [role="scratchpad"] ***** * _Adjust model so that items are associated with different lists._ * _Add unique URLs for each list._ * _Add a URL for creating a new list via POST._ * _Add URLs for adding a new item to an existing list via POST._ ***** === Implementing the New Design Incrementally Using TDD ((("Test-Driven Development (TDD)", "overall process of"))) ((("multiple lists testing", "incremental design implementation"))) How do we use TDD to implement the new design? Let's take another look at the flowchart for the TDD process, duplicated in <> for your convenience. At the top level, we're going to use a combination of adding new functionality (by adding a new FT and writing new application code) and refactoring our application--that is, rewriting some of the existing implementation so that it delivers the same functionality to the user but using aspects of our new design. We'll be able to use the existing FT to verify that we don't break what already works, and the new FT to drive the new features. At the unit test level, we'll be adding new tests or modifying existing ones to test for the changes we want, and we'll be able to similarly use the unit tests we _don't_ touch to help make sure we don't break anything in the process. [[double-loop-tdd-diagram-2]] .The TDD process with both functional and unit tests image::images/tdd3_1708.png["An inner red/green/refactor loop surrounded by an outer red/green of FTs"] === Ensuring We Have a Regression Test ((("regression", id="regression07"))) ((("multiple lists testing", "regression test", id="MLTregression07"))) Our existing FT, `test_can_start_a_todo_list()`, is going to act as our regression test. Let's translate our scratchpad into a new FT method, which introduces a second user and checks that their to-do list is separate from Edith's. We'll start out very similarly to the first. Edith adds a first item to create a to-do list, but we introduce our first new assertion—Edith's list should live at its own, unique URL: [role="sourcecode"] .functional_tests/tests.py (ch07l005) ==== [source,python] ---- def test_can_start_a_todo_list(self): # Edith has heard about a cool new online to-do app. [...] # Satisfied, she goes back to sleep def test_multiple_users_can_start_lists_at_different_urls(self): # Edith starts a new to-do list self.browser.get(self.live_server_url) inputbox = self.browser.find_element(By.ID, "id_new_item") inputbox.send_keys("Buy peacock feathers") inputbox.send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Buy peacock feathers") # She notices that her list has a unique URL edith_list_url = self.browser.current_url self.assertRegex(edith_list_url, "/lists/.+") # <1> ---- ==== [role="pagebreak-before"] <1> `assertRegex` is a helper function from `unittest` that checks whether a string matches a regular expression. We use it to check that our new REST-ish design has been implemented. Find out more in the https://docs.python.org/3/library/unittest.html[`unittest` documentation]. ((("assertRegex"))) ((("unittest module", "documentation"))) Next, we imagine a new user coming along. We want to check that they don't see any of Edith's items when they visit the home page, and that they get their own unique URL for their list: [role="sourcecode"] .functional_tests/tests.py (ch07l006) ==== [source,python] ---- [...] self.assertRegex(edith_list_url, "/lists/.+") # Now a new user, Francis, comes along to the site. ## We delete all the browser's cookies ## as a way of simulating a brand new user session # <1> self.browser.delete_all_cookies() # Francis visits the home page. There is no sign of Edith's # list self.browser.get(self.live_server_url) page_text = self.browser.find_element(By.TAG_NAME, "body").text self.assertNotIn("Buy peacock feathers", page_text) # Francis starts a new list by entering a new item. He # is less interesting than Edith... inputbox = self.browser.find_element(By.ID, "id_new_item") inputbox.send_keys("Buy milk") inputbox.send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Buy milk") # Francis gets his own unique URL francis_list_url = self.browser.current_url self.assertRegex(francis_list_url, "/lists/.+") self.assertNotEqual(francis_list_url, edith_list_url) # Again, there is no trace of Edith's list page_text = self.browser.find_element(By.TAG_NAME, "body").text self.assertNotIn("Buy peacock feathers", page_text) self.assertIn("Buy milk", page_text) # Satisfied, they both go back to sleep ---- ==== <1> I'm using the convention of double-hashes (`##`) to indicate "meta-comments"—comments about _how_ the test is working and why--so that we can distinguish them from regular comments in FTs, which explain the user story. They're a message to our future selves, which might otherwise be wondering why we're faffing about deleting cookies... ((("double-hashes (##)"))) ((("## (double-hashes)"))) ((("meta-comments"))) Other than that, the new test is fairly self-explanatory. Let's see how we do when we run our FTs: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests*] [...] .F ====================================================================== FAIL: test_multiple_users_can_start_lists_at_different_urls (functional_tests.t ests.NewVisitorTest.test_multiple_users_can_start_lists_at_different_urls) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/functional_tests/tests.py", line 77, in test_multiple_users_can_start_lists_at_different_urls self.assertRegex(edith_list_url, "/lists/.+") AssertionError: Regex didn't match: '/lists/.+' not found in 'http://localhost:8081/' --------------------------------------------------------------------- Ran 2 tests in 5.786s FAILED (failures=1) ---- ((("", startref="regression07"))) ((("", startref="MLTregression07"))) Good, our first test still passes, and the second one fails where we might expect. Let's do a commit, and then go and build some new models and views: [subs="specialcharacters,quotes"] ---- $ *git commit -a* ---- === Iterating Towards the New Design ((("multiple lists testing", "iterative development style"))) ((("iterative development style"))) Being all excited about our new design, I had an overwhelming urge to dive in at this point and start changing _models.py_, which would have broken half the unit tests, and then pile in and change almost every single line of code, all in one go. That's a natural urge, and TDD, as a discipline, is a constant fight against it. Obey the Testing Goat, not Refactoring Cat! We don't need to implement our new, shiny design in a single big bang. Let's make small changes that take us from a working state to a working state, with our design guiding us gently at each stage. There are four items on our to-do list. The FT, with its `Regex didn't match` error, is suggesting to us that the second item--giving lists their own URL and identifier--is the one we should work on next. Let's have a go at fixing that, and only that. [role="pagebreak-before"] The URL comes from the redirect after POST. In _lists/tests.py_, let's find `test_redirects_after_POST` and change the expected redirect location: [role="sourcecode"] .lists/tests.py (ch07l007) ==== [source,python] ---- def test_redirects_after_POST(self): response = self.client.post("/", data={"item_text": "A new list item"}) self.assertRedirects(response, "/lists/the-only-list-in-the-world/") ---- ==== Does that seem slightly strange? Clearly, _/lists/the-only-list-in-the-world_ isn't a URL that's going to feature in the final design of our application. But we're committed to changing one thing at a time. While our application only supports one list, this is the only URL that makes sense. We're still moving forwards, in that we'll have a different URL for our list and our home page, which is a step along the way to a more REST-ful design. Later, when we have multiple lists, it will be easy to change. // SEBASTIAN: Yet another mantra that also fits TDD and here is "fake it till you make it" // Perhaps worth mentioning here to explain in advance how this helps making small steps // to eventually detect the trick using simply more / other tests NOTE: Another way of thinking about it is as a problem-solving [keep-together]#technique#: our new URL design is currently not implemented, so it works for zero items. Ultimately, we want to solve for _n_ items, but solving for one item is a good step along the way. Running the unit tests gives us an expected fail: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] AssertionError: '/' != '/lists/the-only-list-in-the-world/' - / + /lists/the-only-list-in-the-world/ : Response redirected to '/', expected '/lists/the-only-list-in-the-world/': Expected '/' to equal '/lists/the-only-list-in-the-world/'. ---- We can go adjust our `home_page` view in 'lists/views.py': [role="sourcecode"] .lists/views.py (ch07l008) ==== [source,python] ---- def home_page(request): if request.method == "POST": Item.objects.create(text=request.POST["item_text"]) return redirect("/lists/the-only-list-in-the-world/") items = Item.objects.all() return render(request, "home.html", {"items": items}) ---- ==== Django's unit test runner picks up on the fact that this is not a real URL yet: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] AssertionError: 404 != 200 : Couldn't retrieve redirection page '/lists/the-only-list-in-the-world/': response code was 404 (expected 200) ---- === Taking a First, Self-Contained Step: One New URL ((("URL mappings", id="url07"))) Our singleton list URL doesn't exist yet. We fix that in _superlists/urls.py_: [role="sourcecode small-code"] .superlists/urls.py (ch07l009) ==== [source,python] ---- from django.urls import path from lists import views urlpatterns = [ path("", views.home_page, name="home"), path("lists/the-only-list-in-the-world/", views.home_page, name="view_list"), # <1> ] ---- ==== <1> We'll just point our new URL at the existing home page view. This is the minimal change. TIP: Watch out for trailing slashes in URLs, both here in _urls.py_ and in the tests. They're a common source of confusion: Django will return a 301 redirect rather than a 404 if you try to access a URL that's missing its trailing slash.footnote:[ The setting that controls this is called https://docs.djangoproject.com/en/5.2/ref/settings/#append-slash[`APPEND_SLASH`]. ] ((("troubleshooting", "URL mappings"))) That gets our unit tests passing: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] OK ---- What do the FTs think? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests*] [...] AssertionError: 'Buy peacock feathers' unexpectedly found in 'Your To-Do list\n1: Buy peacock feathers' ---- Good, they get a little further along. We now confirm that we have a new URL, but the actual page content is still the same; it shows the old list. ==== Separating Out Our Home Page and List View Functionality We now have two URLs, but they're actually doing the exact same thing. ((("home page and list view functionality, separating"))) ((("list view functionality, separating from home page"))) Under the hood, they're just pointing at the same function. Continuing to work incrementally, we can start to break apart the responsibilities for these two different URLs: * The home page only needs to display a static form, and support creating a brand new list based on its first item. * The list view page needs to be able to display existing list items and add new items to the list. Let's split out some tests for our new URL. Open up 'lists/tests.py', and add a new test class called `ListViewTest`. Then: 1. Copy across the `test_renders_input_form()` test from `HomePageTest` into our new class. 2. Move the method called `test_displays_all_list_items()`. 3. In both, change just the URL that is invoked by `self.client.get()`. 4. We _won't_ copy across the `test_uses_home_template()` yet, as we're not quite sure what template we want to use. We'll stick to the tests that check behaviour, rather than implementation. [role="sourcecode"] .lists/tests.py (ch07l010) ==== [source,python] ---- class HomePageTest(TestCase): def test_uses_home_template(self): [...] def test_renders_input_form(self): [...] def test_can_save_a_POST_request(self): [...] def test_redirects_after_POST(self): [...] class ListViewTest(TestCase): def test_renders_input_form(self): response = self.client.get("/lists/the-only-list-in-the-world/") self.assertContains(response, '
') self.assertContains(response, ' ---- ==== By default, the browser sends the POST data back to the same URL it's currently on. When we're on the home page that works fine, but when we're on our _/only-list-in-the-world/_ page, it doesn't. ==== Getting Back to a Working State as Quickly as Possible Now, we could dive in and add POST request handling to our new view, but that would involve writing a bunch more tests and code, and at this point we'd like to get back to a working state as quickly as possible. Actually the _quickest_ thing we can do to get things fixed is to just use the existing home page view, which already works, for all POST requests. In other words, we've identified a new important part of the behaviour we want from our two views and their templates, which is the URL that the form points to. Let's add a check for that URL explicitly, in our two tests for each view (I'll use a diff to show the changes, hopefully that makes it nice and clear): [role="sourcecode"] .lists/tests.py (ch07l013-1) ==== [source,diff] ---- @@ -10,7 +10,7 @@ class HomePageTest(TestCase): def test_renders_input_form(self): response = self.client.get("/") - self.assertContains(response, '') + self.assertContains(response, '') self.assertContains(response, '') + self.assertContains(response, '') self.assertContains(response, '') ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: False is not true : Couldn't find '' in the following response b'\n \n To-Do lists\n \n \n

Your To-Do list

\n \n \n \n \n \n
\n \n\n' ====================================================================== FAIL: test_renders_input_form (lists.tests.ListViewTest.test_renders_input_form) [...] AssertionError: False is not true : Couldn't find '' in the following response b'\n \n To-Do lists\n \n \n [...] ---- And so we can fix it like this—the input form, for now, will always point at the home URL: [role="sourcecode"] .lists/templates/home.html (ch07l013-2) ==== [source,html] ---- ---- ==== Unit test pass: ---- OK ---- And we should see our FTs get back to a happier place: [subs="specialcharacters,macros"] ---- FAIL: test_multiple_users_can_start_lists_at_different_urls (functional_tests.t ests.NewVisitorTest.test_multiple_users_can_start_lists_at_different_urls) [...] AssertionError: 'Buy peacock feathers' unexpectedly found in 'Your To-Do list\n1: Buy peacock feathers' Ran 2 tests in 8.541s FAILED (failures=1) ---- Our old FT (the one we're using as a regression test) passes once again, so we know we're back to a working state. The new functionality may not be working yet, but at least the old stuff works as well as it used to. ((("", startref="url07"))) ==== Green? Refactor ((("multiple lists testing", "refactoring"))) ((("refactoring"))) ((("Red/Green/Refactor"))) Time for a little tidying up. In the red/green/refactor dance, our unit tests pass and all our old FTs pass, so we've arrived at green. That means it's time to see if anything needs a refactor. We now have two views: one for the home page, and one for an individual list. Both are currently using the same template, and passing it all the list items currently in the database. Post requests are only handled by the home page though. It feels like the responsibilities of our two views are a little tangled up. Let's try and disentangle them. [role="pagebreak-before less_space"] === Another Small Step: A Separate Template [.keep-together]#for Viewing Lists# ((("multiple lists testing", "separate list viewing templates", id="MLTseparate07"))) ((("templates", "separate list viewing templates", id="TMPseparate07"))) As the home page and the list view are now quite distinct pages, they should be using different HTML templates; _home.html_ can have the single input box, whereas a new template, _list.html_, can take care of showing the table of existing items. We held off on copying across `test_uses_home_template()` until now, because we weren't quite sure what we wanted. Now let's add an explicit test to say that this view uses a different template: [role="sourcecode"] .lists/tests.py (ch07l014) ==== [source,python] ---- class ListViewTest(TestCase): def test_uses_list_template(self): response = self.client.get("/lists/the-only-list-in-the-world/") self.assertTemplateUsed(response, "list.html") def test_renders_input_form(self): [...] def test_displays_all_list_items(self): [...] ---- ==== // DAVID: FWIW I don't think this test adds value, it's an internal detail. We're refactoring anyway // so we would expect not to have to change tests - because we don't want tests to be overly coupled // to the way our code is factored anyway. Let's see what it says: ---- AssertionError: False is not true : Template 'list.html' was not a template used to render the response. Actual template(s) used: home.html ---- Looks about right, let's change the view: [role="sourcecode"] .lists/views.py (ch07l015) ==== [source,python] ---- def view_list(request): items = Item.objects.all() return render(request, "list.html", {"items": items}) ---- ==== But, obviously, that template doesn't exist yet. If we run the unit tests, we get: ---- django.template.exceptions.TemplateDoesNotExist: list.html [...] FAILED (errors=4) ---- Let's create a new file at 'lists/templates/list.html': //16 [subs="specialcharacters,quotes"] ---- $ *touch lists/templates/list.html* ---- A blank template, which gives us two errors--good to know the tests are there to make sure we fill it in: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] ====================================================================== FAIL: test_displays_all_list_items (lists.tests.ListViewTest.test_displays_all_list_items) --------------------------------------------------------------------- [...] AssertionError: False is not true : Couldn't find 'itemey 1' in the following response b'' ====================================================================== FAIL: test_renders_input_form (lists.tests.ListViewTest.test_renders_input_form) ---------------------------------------------------------------------- [...] AssertionError: False is not true : Couldn't find '' in the following response [...] ---- The template for an individual list will reuse quite a lot of the stuff we currently have in _home.html_, so we can start by just copying that: [subs="specialcharacters,quotes"] ---- $ *cp lists/templates/home.html lists/templates/list.html* ---- //17 That gets the tests back to passing (green). [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] OK ---- Now let's do a little more tidying up (refactoring). We said the home page doesn't need to list items; it only needs the new list input field. So we can remove some lines from _lists/templates/home.html_, and maybe slightly tweak the `h1` to say "Start a new To-Do list". I'll present the code change as a diff again, as I think that shows nice and clearly what we need to modify: [role="sourcecode small-code"] .lists/templates/home.html (ch07l018) ==== [source,diff] ---- -

Your To-Do list

+

Start a new To-Do list

{% csrf_token %}
- - {% for item in items %} - - {% endfor %} -
{{ forloop.counter }}: {{ item.text }}
---- ==== We rerun the unit tests to check that hasn't broken anything... ---- OK ---- Good. Now there's actually no need to pass all the items to the _home.html_ template in our `home_page` view, so we can simplify that and delete a few lines: [role="sourcecode"] .lists/views.py (ch07l019) ==== [source,diff] ---- if request.method == "POST": Item.objects.create(text=request.POST["item_text"]) return redirect("/lists/the-only-list-in-the-world/") - - items = Item.objects.all() - return render(request, "home.html", {"items": items}) + return render(request, "home.html") ---- ==== Rerun the unit tests once more; they still pass: ---- OK ---- Time to run the FTs: ---- File "...goat-book/functional_tests/tests.py", line 96, in test_multiple_users_can_start_lists_at_different_urls self.wait_for_row_in_list_table("1: Buy milk") ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^ [...] AssertionError: '1: Buy milk' not found in ['1: Buy peacock feathers', '2: Buy milk'] ---------------------------------------------------------------------- Ran 2 tests in 10.606s FAILED (failures=1) ---- Great! Only one failure, so we know our regression test (the first FT) is passing. Let's see where we're getting to with the new FT. [role="pagebreak-before"] Let's take a look at it again: [role="sourcecode currentcontents"] .functional_tests/tests.py ==== [source,python] ---- def test_multiple_users_can_start_lists_at_different_urls(self): # Edith starts a new to-do list self.browser.get(self.live_server_url) inputbox = self.browser.find_element(By.ID, "id_new_item") inputbox.send_keys("Buy peacock feathers") inputbox.send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Buy peacock feathers") # <1> [...] # Now a new user, Francis, comes along to the site. [...] # Francis visits the home page. There is no sign of Edith's # list self.browser.get(self.live_server_url) page_text = self.browser.find_element(By.TAG_NAME, "body").text self.assertNotIn("Buy peacock feathers", page_text) # <2> # Francis starts a new list by entering a new item. He # is less interesting than Edith... inputbox = self.browser.find_element(By.ID, "id_new_item") inputbox.send_keys("Buy milk") inputbox.send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Buy milk") # <3> [...] ---- ==== <1> Edith's list says "Buy peacock feathers". <2> When Francis loads the home page, there's no sign of Edith's list. <3> (This is the line where our test fails.) When Francis adds a new item, he sees Edith's item as number 1, and his appears as number 2. Still, that's progress! The new FT _is_ getting a little further along. ((("", startref="MLTseparate07"))) ((("", startref="TMPseparate07"))) It may feel like we haven't made much headway because, functionally, the site still behaves almost exactly like it did when we started the chapter. But this really _is_ progress. We've started on the road to our new design, and we've implemented a number of stepping stones _without making anything worse than it was before_. [role="pagebreak-before"] Let's commit our work so far: [subs="specialcharacters,quotes"] ---- $ *git status* # should show 4 changed files and 1 new file, list.html $ *git add lists/templates/list.html* $ *git diff* # should show we've simplified home.html, # moved one test to a new class in lists/tests.py, # changed the redirect in homepageTest & the home_page() view # added a new view view_list() in views.py, # and and added a line to urls.py. $ *git commit -a* # add a message summarising the above, maybe something like # "new URL, view and template to display lists" ---- NOTE: If this is all feeling a little abstract, now might be a good time to load up the site with `manage.py runserver` and try adding a couple of different lists yourself, and get a feel for how the site is currently behaving. === A Third Small Step: A New URL for Adding List Items ((("multiple lists testing", "list item URLs", id="MLTlist07"))) ((("URL mappings", id="urlmap07a"))) Where are we with our own to-do list? [role="scratchpad"] ***** * 'Adjust model so that items are associated with different lists.' * 'Add unique URLs for each list.' * 'Add a URL for creating a new list via POST.' * 'Add URLs for adding a new item to an existing list via POST.' ***** We've _sort of_ made progress on the second item, even if there's still only one list in the world. The first item is a bit scary. Can we do something about items 3 or 4? Let's have a new URL for adding new list items at _/lists/new_: If nothing else, it'll simplify the home page view. [role="pagebreak-before less_space"] ==== A Test Class for New List Creation Open up _lists/tests.py_, and _move_ the `test_can_save_a_POST_request()` and `test_redirects_after_POST()` methods into a new class called `NewListTest`. ((("lists", "creating, test class for")))Then, change the URL they POST to: // TODO: handwave about test_only_saves_items_when_necessary() [role="sourcecode small-code"] .lists/tests.py (ch07l020) ==== [source,python] ---- class HomePageTest(TestCase): def test_uses_home_template(self): [...] def test_renders_input_form(self): [...] def test_only_saves_items_when_necessary(self): [...] class NewListTest(TestCase): def test_can_save_a_POST_request(self): self.client.post("/lists/new", data={"item_text": "A new list item"}) self.assertEqual(Item.objects.count(), 1) new_item = Item.objects.get() self.assertEqual(new_item.text, "A new list item") def test_redirects_after_POST(self): response = self.client.post("/lists/new", data={"item_text": "A new list item"}) self.assertRedirects(response, "/lists/the-only-list-in-the-world/") class ListViewTest(TestCase): def test_uses_list_template(self): [...] ---- ==== // TODO: sneaky change from .first() to .get() here, // should grandfather in to chap 5. TIP: This is another place to pay attention to trailing slashes, incidentally. It's `/lists/new`, with no trailing slash. The convention I'm using is that URLs without a trailing slash are "action" URLs, which modify the database.footnote:[ I don't think this is a very common convention anymore these days, but I quite like it. By all means, cast around for a URL naming scheme that makes sense to you in your own projects!] Try running that: ---- self.assertEqual(Item.objects.count(), 1) AssertionError: 0 != 1 [...] self.assertRedirects(response, "/lists/the-only-list-in-the-world/") [...] AssertionError: 404 != 302 : Response didn't redirect as expected: Response code was 404 (expected 302) ---- The first failure tells us we're not saving a new item to the database, and the second says that, instead of returning a 302 redirect, our view is returning a 404. That's because we haven't built a URL for _/lists/new_, so the `client.post` is just getting a "not found" response. NOTE: Do you remember how we split this out into two tests earlier? If we only had one test that checked both the saving and the redirect, it would have failed on the `0 != 1` failure, which would have been much harder to debug. Ask me how I know this. ==== A URL and View for New List Creation Let's build our new ((("URL mappings", "URL for new list creation")))((("lists", "URL and view for new list creation")))URL now: [role="sourcecode"] .superlists/urls.py (ch07l021) ==== [source,python] ---- urlpatterns = [ path("", views.home_page, name="home"), path("lists/new", views.new_list, name="new_list"), path("lists/the-only-list-in-the-world/", views.view_list, name="view_list"), ] ---- ==== Next we get a `no attribute 'new_list'`, so let's fix that, in 'lists/views.py': [role="sourcecode"] .lists/views.py (ch07l022) ==== [source,python] ---- def new_list(request): pass ---- ==== Then we get "The view lists.views.new_list didn't return an HttpResponse object". (This is getting rather familiar!) We could return a raw `HttpResponse`, but because we know we'll need a redirect, let's borrow a line from `home_page`: [role="sourcecode"] .lists/views.py (ch07l023) ==== [source,python] ---- def new_list(request): return redirect("/lists/the-only-list-in-the-world/") ---- ==== That gives: ---- self.assertEqual(Item.objects.count(), 1) AssertionError: 0 != 1 ---- [role="pagebreak-before"] Seems reasonably straightforward. We borrow another line from `home_page`: [role="sourcecode"] .lists/views.py (ch07l024) ==== [source,python] ---- def new_list(request): Item.objects.create(text=request.POST["item_text"]) return redirect("/lists/the-only-list-in-the-world/") ---- ==== And everything now passes: ---- Ran 9 tests in 0.030s OK ---- And we can run the FTs to check that we're still in the same place: ---- [...] AssertionError: '1: Buy milk' not found in ['1: Buy peacock feathers', '2: Buy milk'] Ran 2 tests in 8.972s FAILED (failures=1) ---- Our regression test passes, and the new FT gets to the same point. ==== Removing Now-Redundant Code and Tests We're looking good. As our new views are now doing most of the work that `home_page` used to do, we should be able to massively simplify it. Can we remove the whole `if request.method == 'POST'` section, for example? [role="sourcecode"] .lists/views.py (ch07l025) ==== [source,python] ---- def home_page(request): return render(request, "home.html") ---- ==== //24 Yep! The unit tests pass: ---- OK ---- And while we're at it, we can remove the now-redundant pass:[test_only_saves_​items_when_necessary] test too! Doesn't that feel good? The view functions are looking much simpler. We rerun the tests to make sure... [role="dofirst-ch07l025-2"] ---- Ran 8 tests in 0.016s OK ---- And the FTs? [role="pagebreak-before less_space"] ==== A Regression! Pointing Our Forms at the New URL Oops. When we run the FTs: ---- ====================================================================== ERROR: test_can_start_a_todo_list (functional_tests.tests.NewVisitorTest.test_can_start_a_todo_list) --------------------------------------------------------------------- [...] File "...goat-book/functional_tests/tests.py", line 52, in test_can_start_a_todo_list [...] self.wait_for_row_in_list_table("1: Buy peacock feathers") ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^ [...] table = self.browser.find_element(By.ID, "id_list_table") [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_list_table"]; For documentation [...] ====================================================================== ERROR: test_multiple_users_can_start_lists_at_different_urls (functional_tests. tests.NewVisitorTest.test_multiple_users_can_start_lists_at_different_urls) --------------------------------------------------------------------- [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_list_table"]; For documentation [...] [...] Ran 2 tests in 11.592s FAILED (errors=2) ---- Once again, the FTs pick up a tricky little bug, something that our unit tests alone would find it hard to catch. [role="pagebreak-before less_space"] ==== Debugging in DevTools This is another good time to spin up the dev server, and have a look around with a browser.((("debugging", "in DevTools", secondary-sortas="DevTools")))((("DevTools (developer tools)", "debugging in"))) Let's also open up https://firefox-source-docs.mozilla.org/devtools-user[DevTools],footnote:[ If you've not seen it before, DevTools is short for "developer tools". They're tools that Firefox (and other browsers) give you to be able to look "under the hood" and see what's going on with web pages, including the source code, what network requests are being made, and what JavaScript is doing. You can open up DevTools with Ctrl+Shift+I or Cmd-Opt-I.] and click around to see what's going on: * First I tried submitting a new list item, and saw we get sent back to the home page. * Then I did the same with the browser DevTools open, and in the "network" tab I saw a POST request to "/". See <>. * Finally, I had a look at the HTML source of the home page, and saw that the main form is still pointing at "/". [[post-in-dev-tools]] .DevTools shows a POST request to / image::images/tdd3_0703.png["screenshot of browser dev tools with a POST request in the network tab, with the File column showing /"] Actually, _both_ our forms are still pointing to the old URL. We have tests for this! Let's amend them: [role="sourcecode small-code"] .lists/tests.py (ch07l026) ==== [source,diff] ---- @@ -10,7 +10,7 @@ class HomePageTest(TestCase): def test_renders_input_form(self): response = self.client.get("/") - self.assertContains(response, '
') + self.assertContains(response, '') self.assertContains(response, '') + self.assertContains(response, '') self.assertContains(response, '' in the following response [...] AssertionError: False is not true : Couldn't find '' in the following response [...] ---- In _both_ _home.html_ and _list.html_, let's change them: [role="sourcecode"] .lists/templates/home.html (ch07l027) ==== [source,html] ---- ---- ==== And: [role="sourcecode"] .lists/templates/list.html (ch07l028) ==== [source,html] ---- ---- ==== And that should get us back to working again: ---- Ran 8 tests in 0.006s OK ---- [role="pagebreak-before"] And our FTs are still in the familiar "Francis sees Edith's items" place: ---- AssertionError: '1: Buy milk' not found in ['1: Buy peacock feathers', '2: Buy milk'] [...] FAILED (failures=1) ---- Perhaps this all seems quite pernickety, but that's another nicely self-contained commit, in that we've made a bunch of changes to our URLs, our _views.py_ is looking much neater and tidier with three very short view functions, and we're sure the application is still working as well as it was before. We're getting good at this working-state-to-working-state malarkey! [subs="specialcharacters,quotes"] ---- $ *git status* # 5 changed files $ *git diff* # URLs for forms x2, new test + view with code moves, and new URL $ *git commit -a* ---- ((("", startref="MLTlist07"))) ((("", startref="urlmap07a"))) And we can cross out an item on the to-do list: [role="scratchpad"] ***** * 'Adjust model so that items are associated with different lists.' * 'Add unique URLs for each list.' * '[strikethrough line-through]#Add a URL for creating a new list via POST.#' * 'Add URLs for adding a new item to an existing list via POST.' ***** [role="pagebreak-before less_space"] === Biting the Bullet: Adjusting Our Models Enough housekeeping with our URLs. It's time to bite the bullet and change our models. Let's adjust the model unit test. [role="sourcecode"] .lists/tests.py (ch07l029) ==== [source,diff] ---- @@ -1,5 +1,5 @@ from django.test import TestCase -from lists.models import Item +from lists.models import Item, List class HomePageTest(TestCase): @@ -35,20 +35,30 @@ class ListViewTest(TestCase): self.assertContains(response, "itemey 2") -class ItemModelTest(TestCase): +class ListAndItemModelsTest(TestCase): def test_saving_and_retrieving_items(self): + mylist = List() + mylist.save() + first_item = Item() first_item.text = "The first (ever) list item" + first_item.list = mylist first_item.save() second_item = Item() second_item.text = "Item the second" + second_item.list = mylist second_item.save() + saved_list = List.objects.get() + self.assertEqual(saved_list, mylist) + saved_items = Item.objects.all() self.assertEqual(saved_items.count(), 2) first_saved_item = saved_items[0] second_saved_item = saved_items[1] self.assertEqual(first_saved_item.text, "The first (ever) list item") + self.assertEqual(first_saved_item.list, mylist) self.assertEqual(second_saved_item.text, "Item the second") + self.assertEqual(second_saved_item.list, mylist) ---- ==== Once again, this is a very verbose test, because I'm using it more as a demonstration of how the ORM works. We'll shorten it later,footnote:[In <>, if you're curious.] but for now, let's work through and see how things work. We create a new `List` object and then we assign each item to it by setting it as its `.list` property. We check that the list is properly saved, and we check that the two items have also saved their relationship to the list. You'll also notice that we can compare list objects with each other directly (`saved_list` and `mylist`)—behind the scenes, these will compare themselves by checking that their primary key (the `.id` attribute) is the same. Time for another unit-test/code cycle. For the first few iterations, rather than explicitly showing you what code to enter in between every test run, I'm only going to show you the expected error messages from running the tests. I'll let you figure out what each minimal code change should be, on your own. TIP: Need a hint? Go back and take a look at the steps we took to introduce the `Item` model in <>. Your first error should be: [subs="specialcharacters,macros"] ---- ImportError: cannot import name 'List' from 'lists.models' ---- Fix that, and then you should see: [role="dofirst-ch07l030"] ---- AttributeError: 'List' object has no attribute 'save' ---- Next you should see: [role="dofirst-ch07l031"] ---- django.db.utils.OperationalError: no such table: lists_list ---- So, we run a `makemigrations`: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py makemigrations*] Migrations for 'lists': lists/migrations/0003_list.py + Create model List ---- And then you should see: ---- self.assertEqual(first_saved_item.list, mylist) AttributeError: 'Item' object has no attribute 'list' ---- [role="pagebreak-before less_space"] ==== A Foreign Key Relationship How do we give our `Item` a list attribute?((("foreign keys"))) Let's just try naively making it like the `text` attribute (and here's your chance to see whether your solution so far looks like mine, by the way): [role="sourcecode"] .lists/models.py (ch07l033) ==== [source,python] ---- from django.db import models class List(models.Model): pass class Item(models.Model): text = models.TextField(default="") list = models.TextField(default="") ---- ==== As usual, the tests tell us we need a migration: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] django.db.utils.OperationalError: no such column: lists_item.list $ pass:quotes[*python manage.py makemigrations*] Migrations for 'lists': lists/migrations/0004_item_list.py + Add field list to item ---- Let's see what that gives us: ---- AssertionError: 'List object (1)' != ---- We're not quite there. Look closely at each side of the `!=`. Do you see the quotes (`'`)? Django has only saved the string representation of the `List` object. To save the relationship to the object itself, we tell Django about the relationship between the two classes using a `ForeignKey`: [role="sourcecode"] .lists/models.py (ch07l035) ==== [source,python] ---- class Item(models.Model): text = models.TextField(default="") list = models.ForeignKey(List, default=None, on_delete=models.CASCADE) ---- ==== // DAVID: this provides None as a default, but the field is non-nullable. Consider adding // null=True too? Or else (and I would actually prefer this), don't provide a default // and get them to delete their database and remigrate. We don't really want Items // in the database that have no list. [role="pagebreak-before"] That'll need a migration too. As the last one was a red herring, let's delete it and replace it with a new one: [subs="specialcharacters,macros"] ---- $ pass:quotes[*rm lists/migrations/0004_item_list.py*] $ pass:quotes[*python manage.py makemigrations*] Migrations for 'lists': lists/migrations/0004_item_list.py + Add field list to item ---- //31 WARNING: Deleting migrations is dangerous. Now and again it's nice to do it to keep things tidy, because we don't always get our models' code right on the first go! But if you delete a migration that's already been applied to a database somewhere, Django will be confused about what state it's in, and won't be able to apply future migrations. You should only do it when you're sure the migration hasn't been used. A good rule of thumb is that you should never delete or modify a migration that's already been committed to Git. ==== Adjusting the Rest of the World to Our New Models Back in our tests, now what happens? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] ERROR: test_displays_all_list_items django.db.utils.IntegrityError: NOT NULL constraint failed: lists_item.list_id [...] ERROR: test_redirects_after_POST django.db.utils.IntegrityError: NOT NULL constraint failed: lists_item.list_id [...] ERROR: test_can_save_a_POST_request django.db.utils.IntegrityError: NOT NULL constraint failed: lists_item.list_id Ran 8 tests in 0.021s FAILED (errors=3) ---- Oh dear! There is some good news. Although it's hard to see, our model tests are passing. But three of our view tests are failing nastily. The cause is the new relationship we've introduced between ++Item++s and ++List++s, which requires each item to have a parent list, and which our old tests and code aren't prepared for. [role="pagebreak-before"] Still, this is exactly why we have tests! Let's get them working again. The easiest is the `ListViewTest`; we just create a parent list for our two test items: [role="sourcecode"] .lists/tests.py (ch07l038) ==== [source,python] ---- class ListViewTest(TestCase): [...] def test_displays_all_list_items(self): mylist = List.objects.create() Item.objects.create(text="itemey 1", list=mylist) Item.objects.create(text="itemey 2", list=mylist) ---- ==== That gets us down to two failing tests, both on tests that try to POST to our `new_list` view. Decode the tracebacks using our usual technique, working back from error to line of test code to—buried in there somewhere—the line of our own code that caused the failure: [subs="specialcharacters,macros"] ---- File "...goat-book/lists/tests.py", line 25, in test_redirects_after_POST response = self.client.post("/lists/new", data={"item_text": "A new list item"}) [...] File "...goat-book/lists/views.py", line 11, in new_list Item.objects.create(text=request.POST["item_text"]) ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [...] django.db.utils.IntegrityError: NOT NULL constraint failed: lists_item.list_id ---- It's when we try to create an item without a parent list. So we make a similar change in the view: [role="sourcecode"] .lists/views.py (ch07l039) ==== [source,python] ---- from lists.models import Item, List [...] def new_list(request): nulist = List.objects.create() Item.objects.create(text=request.POST["item_text"], list=nulist) return redirect("/lists/the-only-list-in-the-world/") ---- ==== And that gets our tests passing again:footnote:[Are you wondering about the strange spelling of the "nulist" variable? Other options are "list", which would shadow the built-in `list()` function, and `new_list`, which would shadow the name of the function that contains it. Or `list_` with the trailing underscore, which I find a bit ugly, or `list1` or `listey` or `mylist`, but none are particularly satisfactory.] ---- Ran 8 tests in 0.030s OK ---- ((("Test-Driven Development (TDD)", "philosophy of", "working state to working state"))) ((("working state to working state"))) ((("Testing Goat", "working state to working state"))) Are you cringing internally at this point? _Arg! This feels so wrong; we create a new list for every single new item submission, and we're still just displaying all items as if they belong to the same list!_ I know; I feel the same. The step-by-step approach, in which you go from working code to working code, is counterintuitive. I always feel like just diving in and trying to fix everything all in one go, instead of going from one weird half-finished state to another. But remember the Testing Goat! When you're up a mountain, you want to think very carefully about where you put each foot, and take one step at a time, checking at each stage that the place you've put it hasn't caused you to fall off a cliff. So, just to reassure ourselves that things have worked, we rerun the FT: ---- AssertionError: '1: Buy milk' not found in ['1: Buy peacock feathers', '2: Buy milk'] [...] ---- Sure enough, it gets all the way through to where we were before. We haven't broken anything, and we've made a big change to the database. That's something to be pleased with! Let's commit: [subs="specialcharacters,quotes"] ---- $ *git status* # 3 changed files, plus 2 migrations $ *git add lists* $ *git diff --staged* $ *git commit* ---- And we can cross out another item on the to-do list: [role="scratchpad"] ***** * '[strikethrough line-through]#Adjust model so that items are associated with different lists.#' * 'Add unique URLs for each list.' * '[strikethrough line-through]#Add a URL for creating a new list via POST.#' * 'Add URLs for adding a new item to an existing list via POST.' ***** [role="pagebreak-before less_space"] === Each List Should Have Its Own URL We can get rid of the silly `the-only-list-in-the-world` URL, but what shall we use as the unique identifier for our lists? Probably the simplest thing, for now, is just to use the autogenerated `id` field from the database. Let's change `ListViewTest` so that the two tests point at new URLs. We'll also change the old `test_displays_all_list_items` test and call it `test_displays_only_items_for_that_list` instead, making it check that only the items for a specific list are displayed: [role="sourcecode"] .lists/tests.py (ch07l040) ==== [source,python] ---- class ListViewTest(TestCase): def test_uses_list_template(self): mylist = List.objects.create() response = self.client.get(f"/lists/{mylist.id}/") # <1> self.assertTemplateUsed(response, "list.html") def test_renders_input_form(self): mylist = List.objects.create() response = self.client.get(f"/lists/{mylist.id}/") # <1> self.assertContains(response, '') self.assertContains(response, ' Item.objects.create(text="itemey 1", list=correct_list) Item.objects.create(text="itemey 2", list=correct_list) other_list = List.objects.create() # <2> Item.objects.create(text="other list item", list=other_list) response = self.client.get(f"/lists/{correct_list.id}/") # <3> self.assertContains(response, "itemey 1") self.assertContains(response, "itemey 2") self.assertNotContains(response, "other list item") # <4> ---- ==== <1> Here's where we incorporate the ID of our new list into the GET URL. <2> In the "Given" phase of the test, we now set up two lists: the one we're interested in and an extraneous one. <3> We change this URL too, to point at the 'correct' list. <4> And now, our "Then" section can check that the irrelevant list's items are definitely not present. [role="pagebreak-before"] Running the unit tests gives the expected 404s and another related error: ---- FAIL: test_displays_only_items_for_that_list AssertionError: 404 != 200 : Couldn't retrieve content: Response code was 404 (expected 200) [...] FAIL: test_renders_input_form AssertionError: 404 != 200 : Couldn't retrieve content: Response code was 404 (expected 200) [...] FAIL: test_uses_list_template AssertionError: No templates used to render the response ---- ==== Capturing Parameters from URLs It's time to learn ((("views", "passing URL parameters to")))((("URLs", "parameters from, passing to views")))how we can pass parameters from URLs to views: [role="sourcecode"] .superlists/urls.py (ch07l041-0) ==== [source,python] ---- urlpatterns = [ path("", views.home_page, name="home"), path("lists/new", views.new_list, name="new_list"), path("lists//", views.view_list, name="view_list"), ] ---- ==== We adjust the path string for our URL to include a 'capture group', ``, which will match any numerical characters, up to the following `/`. The captured `id` will be passed to the view as an argument. In other words, if we go to the URL '/lists/1/', `view_list` will get a second argument after the normal `request` argument, namely the integer `1`. But our view doesn't expect an argument yet! Sure enough, this causes problems: ---- ERROR: test_displays_only_items_for_that_list [...] TypeError: view_list() got an unexpected keyword argument 'list_id' [...] ERROR: test_renders_input_form [...] TypeError: view_list() got an unexpected keyword argument 'list_id' [...] ERROR: test_uses_list_template [...] TypeError: view_list() got an unexpected keyword argument 'list_id' [...] FAIL: test_redirects_after_POST [...] AssertionError: 404 != 200 : Couldn't retrieve redirection page '/lists/the-only-list-in-the-world/': response code was 404 (expected 200) [...] FAILED (failures=1, errors=3) ---- We can fix that easily with an unused parameter in _views.py_: [role="sourcecode"] .lists/views.py (ch07l041) ==== [source,python] ---- def view_list(request, list_id): [...] ---- ==== That takes us down to our expected failure, plus something to do with an _/only-list-in-the-world/_ that's still hanging around somewhere, which I'm sure we can fix later. ---- FAIL: test_displays_only_items_for_that_list [...] AssertionError: 1 != 0 : 'other list item' unexpectedly found in the following response [...] FAIL: test_redirects_after_POST AssertionError: 404 != 200 : Couldn't retrieve redirection page '/lists/the-only-list-in-the-world/': response code was 404 (expected 200) ---- Let's make our list view discriminate over which items it sends to the template: [role="sourcecode"] .lists/views.py (ch07l042) ==== [source,python] ---- def view_list(request, list_id): our_list = List.objects.get(id=list_id) items = Item.objects.filter(list=our_list) return render(request, "list.html", {"items": items}) ---- ==== ==== Adjusting new_list to the New World It's time to address the _/only-list-in-the-world/_ failure: ---- FAIL: test_redirects_after_POST [...] AssertionError: 404 != 200 : Couldn't retrieve redirection page '/lists/the-only-list-in-the-world/': response code was 404 (expected 200) ---- Let's have a little look and find the test that's moaning: [role="sourcecode currentcontents small-code"] .lists/tests.py ==== [source,python] ---- class NewListTest(TestCase): [...] def test_redirects_after_POST(self): response = self.client.post("/lists/new", data={"item_text": "A new list item"}) self.assertRedirects(response, "/lists/the-only-list-in-the-world/") ---- ==== It looks like it hasn't been adjusted to the new world of ++List++s and ++Item++s. The test should be saying that this view redirects to the URL of the specific new list it just created. [role="sourcecode small-code"] .lists/tests.py (ch07l043) ==== [source,python] ---- def test_redirects_after_POST(self): response = self.client.post("/lists/new", data={"item_text": "A new list item"}) new_list = List.objects.get() self.assertRedirects(response, f"/lists/{new_list.id}/") ---- ==== The test still fails, but we can now take a look at the view itself, and change it so it redirects to the right place: [role="sourcecode"] .lists/views.py (ch07l044) ==== [source,python] ---- def new_list(request): nulist = List.objects.create() Item.objects.create(text=request.POST["item_text"], list=nulist) return redirect(f"/lists/{nulist.id}/") ---- ==== That gets us back to passing unit tests, phew! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test lists*] [...] ........ --------------------------------------------------------------------- Ran 8 tests in 0.033s OK ---- What about the FTs? === The Functional Tests Detect Another Regression It feels like we're done with migrating to the new URL structure; we must be almost there? Well, almost. When we run the FTs, we get: [subs="specialcharacters,macros"] ---- F. ====================================================================== FAIL: test_can_start_a_todo_list (functional_tests.tests.NewVisitorTest.test_can_start_a_todo_list) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/functional_tests/tests.py", line 62, in test_can_start_a_todo_list self.wait_for_row_in_list_table("2: Use peacock feathers to make a fly") [...] AssertionError: '2: Use peacock feathers to make a fly' not found in ['1: Use peacock feathers to make a fly'] --------------------------------------------------------------------- Ran 2 tests in 8.617s FAILED (failures=1) ---- Our _new_ FT is actually passing: different users can get different lists. But the old test is warning us of a regression. It looks like you can't add a second item to a list any more. It's because of our quick-and-dirty hack where we create a new list for every single POST submission. This is exactly what we have FTs for! And it correlates nicely with the last item on our to-do list: [role="scratchpad"] ***** * '[strikethrough line-through]#Adjust model so that items are associated with different lists.#' * '[strikethrough line-through]#Add unique URLs for each list.#' * '[strikethrough line-through]#Add a URL for creating a new list via POST.#' * 'Add URLs for adding a new item to an existing list via POST.' ***** === One More URL to Handle Adding Items to an Existing List We need a URL and view to handle adding a new item to an existing list (_/lists//add_item_).((("URLs", "URL to handle adding items to existing list"))) We're starting to get used to these now, so we know we'll need: 1. A new test for the new URL 2. A new entry in _urls.py_ 3. A new view function [role="pagebreak-before"] So, let's see if we can knock all that together quickly: [role="sourcecode"] .lists/tests.py (ch07l045) ==== [source,python] ---- class NewItemTest(TestCase): def test_can_save_a_POST_request_to_an_existing_list(self): other_list = List.objects.create() correct_list = List.objects.create() self.client.post( f"/lists/{correct_list.id}/add_item", data={"item_text": "A new item for an existing list"}, ) self.assertEqual(Item.objects.count(), 1) new_item = Item.objects.get() self.assertEqual(new_item.text, "A new item for an existing list") self.assertEqual(new_item.list, correct_list) def test_redirects_to_list_view(self): other_list = List.objects.create() correct_list = List.objects.create() response = self.client.post( f"/lists/{correct_list.id}/add_item", data={"item_text": "A new item for an existing list"}, ) self.assertRedirects(response, f"/lists/{correct_list.id}/") ---- ==== NOTE: Are you wondering about `other_list`? A bit like in the tests for viewing a specific list, it's important that we add items to a specific list. Adding this second object to the database prevents me from using a hack like `List.objects.first()` in the view. Yes, that would be a silly thing to do, and you can go too far down the road of testing for all the silly things you must not do (there are an infinite number of those, after all). It's a judgement call, but this one feels worth it. There's some more discussion of this in <>. Oh, and yes it's an unused variable, and your IDE might nag you about it, but I find it helps me to remember what it's for. So that fails as expected, the list item is not saved, and the new URL currently returns a 404: ---- AssertionError: 0 != 1 [...] AssertionError: 404 != 302 : Response didn't redirect as expected: Response code was 404 (expected 302) ---- ==== The Last New urls.py Entry Now we've got our expected 404, let's add a new URL for adding new items to existing lists: [role="sourcecode"] .superlists/urls.py (ch07l046) ==== [source,python] ---- urlpatterns = [ path("", views.home_page, name="home"), path("lists/new", views.new_list, name="new_list"), path("lists//", views.view_list, name="view_list"), path("lists//add_item", views.add_item, name="add_item"), ] ---- ==== We've got three very similar-looking URLs there. Let's make a note on our to-do list; they look like good candidates for a refactoring: [role="scratchpad"] ***** * '[strikethrough line-through]#Adjust model so that items are associated with different lists.#' * '[strikethrough line-through]#Add unique URLs for each list.#' * '[strikethrough line-through]#Add a URL for creating a new list via POST.#' * 'Add URLs for adding a new item to an existing list via POST.' * 'Refactor away some duplication in urls.py.' ***** ==== The Last New View Back to the tests, we get the usual missing module view objects: ---- AttributeError: module 'lists.views' has no attribute 'add_item' ---- Let's try: [role="sourcecode"] .lists/views.py (ch07l047) ==== [source,python] ---- def add_item(request): pass ---- ==== [role="pagebreak-before"] Aha: ---- TypeError: add_item() got an unexpected keyword argument 'list_id' ---- [role="sourcecode"] .lists/views.py (ch07l048) ==== [source,python] ---- def add_item(request, list_id): pass ---- ==== And then: ---- ValueError: The view lists.views.add_item didn't return an HttpResponse object. It returned None instead. ---- We can copy the `redirect()` from `new_list` and the `List.objects.get()` from `view_list`: [role="sourcecode"] .lists/views.py (ch07l049) ==== [source,python] ---- def add_item(request, list_id): our_list = List.objects.get(id=list_id) return redirect(f"/lists/{our_list.id}/") ---- ==== That takes us to: ---- self.assertEqual(Item.objects.count(), 1) AssertionError: 0 != 1 ---- Finally, we make it save our new list item: [role="sourcecode"] .lists/views.py (ch07l050) ==== [source,python] ---- def add_item(request, list_id): our_list = List.objects.get(id=list_id) Item.objects.create(text=request.POST["item_text"], list=our_list) return redirect(f"/lists/{our_list.id}/") ---- ==== And we're back to passing tests: ---- Ran 10 tests in 0.050s OK ---- Hooray! Did that feel like quite a nice, fluid, unit-test/code cycle? [role="pagebreak-before less_space"] ==== Testing Template Context Directly ((("templates", "testing template context directly"))) We've got our new view and URL for adding items to existing lists; now we just need to actually use it in our _list.html_ template. We have a unit test for the form's action; let's amend it: [role="sourcecode"] .lists/tests.py (ch07l051) ==== [source,python] ---- class ListViewTest(TestCase): def test_uses_list_template(self): [...] def test_renders_input_form(self): mylist = List.objects.create() response = self.client.get(f"/lists/{mylist.id}/") self.assertContains( response, f'', ) self.assertContains(response, '' in the following response [...] ---- So, we open it up to adjust the form tag... [role="sourcecode skipme"] .lists/templates/list.html ==== [source,html] ---- ---- ==== ...oh. To get the URL to add to the current list, the template needs to know what list it's rendering, as well as what the items are. [role="pagebreak-before"] ((("programming by wishful thinking"))) Well, "programming by wishful thinking",footnote:[ TDD is a bit like programming by wishful thinking, in that, when we write the tests before the implementation, we express a wish: we wish we had some code that worked! The phrase "programming by wishful thinking" actually has a wider meaning, of writing your code in a top-down kind of way. We'll come back and talk about it more in <>.] let's just pretend we had access to everything we need, like a `list` variable in the template: [role="sourcecode"] .lists/templates/list.html (ch07l052) ==== [source,html] ---- ---- ==== That changes our error slightly: [role="small-code"] [subs="specialcharacters,macros,callouts"] ---- AssertionError: False is not true : Couldn't find '' in the following response b'\n \n To-Do lists\n \n \n

Your To-Do list

\n \n <1> ---- <1> Do you see it says `/lists//add_item`? It's because Django templates will just silently ignore any undefined variables, and substitute empty strings for them. Let's see if we can make our wish come true and pass our list to the template then: [role="sourcecode"] .lists/views.py (ch07l053) ==== [source,python] ---- def view_list(request, list_id): our_list = List.objects.get(id=list_id) items = Item.objects.filter(list=our_list) return render(request, "list.html", {"items": items, "list": our_list}) ---- ==== That gets us to passing tests: ---- OK ---- And we now have an opportunity to refactor, as passing both the list and its items together is redundant. Here's the change in the template: [role="sourcecode"] .lists/templates/list.html (ch07l054) ==== [source,html] ---- {% for item in list.item_set.all %} <1> {{ forloop.counter }}: {{ item.text }} {% endfor %} ---- ==== <1> `.item_set` is called a https://docs.djangoproject.com/en/5.2/topics/db/queries/#following-relationships-backward[reverse lookup]. It's one of Django's incredibly useful bits of ORM that lets you look up an object's related items from a different table. ((("reverse lookups"))) // DAVID: instead of using item_set, might want to consider defining a related name 'items' when we first // define the foreign key. It's more explicit and I think people new to Django might understand it better. The tests still pass... ---- OK ---- And we can now simplify the view down a little: [role="sourcecode"] .lists/views.py (ch07l055) ==== [source,python] ---- def view_list(request, list_id): our_list = List.objects.get(id=list_id) return render(request, "list.html", {"list": our_list}) ---- ==== And our unit tests still pass: ---- Ran 10 tests in 0.040s OK ---- How about the FTs? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests*] [...] .. --------------------------------------------------------------------- Ran 2 tests in 9.771s OK ---- HOORAY! Oh, and a quick check on our to-do list: [role="scratchpad"] ***** * '[strikethrough line-through]#Adjust model so that items are associated with different lists.#' * '[strikethrough line-through]#Add unique URLs for each list.#' * '[strikethrough line-through]#Add a URL for creating a new list via POST.#' * '[strikethrough line-through]#Add URLs for adding a new item to an existing list via POST.#' * 'Refactor away some duplication in urls.py.' ***** Irritatingly, the Testing Goat is a stickler for tying up loose ends too, so we've got to do one final thing. Before we start, we'll do a commit--always make sure you've got a commit of a working state before embarking on a refactor: [subs="specialcharacters,quotes"] ---- $ *git diff* $ *git commit -am "new URL + view for adding to existing lists. FT passes :-)"* ---- [role="pagebreak-before less_space"] === A Final Refactor Using URL includes _superlists/urls.py_ is really meant for URLs that apply to your entire site. For URLs that only apply to the `lists` app, Django encourages us to use a separate _lists/urls.py_, to make the app more self-contained.((("URLs", "final list view refactor using URL includes")))((("includes, URL, final refactor using"))) The simplest way to make one is to use a copy of the existing _urls.py_: [subs="specialcharacters,quotes"] ---- $ *cp superlists/urls.py lists/* ---- //56 Then we replace the three list-specific lines in _superlists/urls.py_ with an `include()`: [role="sourcecode"] .superlists/urls.py (ch07l057) ==== [source,python] ---- from django.urls import include, path from lists import views as list_views # <1> urlpatterns = [ path("", list_views.home_page, name="home"), path("lists/", include("lists.urls")), # <2> ] ---- ==== <1> While we're at it, we use the `import x as y` syntax to alias `views`. This is good practice in your top-level _urls.py_, because it will let us import views from multiple apps if we want--and indeed we will need to later on in the book.((("views", "import syntax aliasing"))) <2> Here's the `include`. Notice that it can take a part of a URL as a prefix, which will be applied to all the included URLs (this is the bit where we reduce duplication, as well as giving our code a better structure). Back in _lists/urls.py_, we can trim down to only include the latter part of our three URLs, and none of the other stuff from the parent _urls.py_: [role="sourcecode"] .lists/urls.py (ch07l058) ==== [source,python] ---- from django.urls import path from lists import views urlpatterns = [ path("new", views.new_list, name="new_list"), path("/", views.view_list, name="view_list"), path("/add_item", views.add_item, name="add_item"), ] ---- ==== Rerun the unit tests to check that everything worked. ---- Ran 10 tests in 0.040s OK ---- [role="pagebreak-before less_space"] ==== Can You Believe It? When I saw this test pass, I couldn't quite believe I did it correctly on the first go. It always pays to be skeptical of your own abilities, so I deliberately changed one of the URLs slightly, just to check if it broke a test. It did. We're covered. Feel free to try it yourself! Remember to change it back, check that the tests all pass again (including the FTs), and then do a final commit: [subs="specialcharacters,quotes"] ---- $ *git status* $ *git add lists/urls.py* $ *git add superlists/urls.py* $ *git diff --staged* $ *git commit* ---- Phew. This was a marathon chapter. But we covered a number of important topics, starting with some thinking about design. We covered rules of thumb like "YAGNI" and "three strikes and refactor". But, most importantly, we saw how to adapt an existing codebase step by step, going from working state to working state, to iterate towards a new design. // CSANAD: "three strikes and refactor", even though it is what we were doing, // was actually not mentioned in this chapter explicitly until this // paragraph, but only earlier, in chapter Post and Database. I'd say we're pretty close to being able to ship this site, as the very first beta of the superlists website that's going to take over the world. Maybe it needs a little prettification first...let's look at what we need to do to deploy it in the next couple of chapters. ((("", startref="TDDadapt07"))) .Some More TDD Philosophy ******************************************************************************* Working state to working state (aka the Testing Goat versus Refactoring Cat):: Our natural urge is often to dive in and fix everything at once...but if we're not careful, we'll end up like Refactoring Cat, in a situation with loads of changes to our code and nothing working. The Testing Goat encourages us to take one step at a time, and go from working state to working state. ((("Test-Driven Development (TDD)", "philosophy of", "working state to working state"))) ((("working state to working state"))) Split work out into small, achievable tasks:: Sometimes this means starting with "boring" work rather than diving straight in with the fun stuff, but you'll have to trust that YOLO-you in the parallel universe is probably having a bad time, having broken everything and struggling to get the app working again. ((("Test-Driven Development (TDD)", "philosophy of", "split work into smaller tasks"))) ((("small vs. big design"))) YAGNI:: You ain't gonna need it! Avoid the temptation to write code that you think 'might' be useful, just because it suggests itself at the time. Chances are, you won't use it, or you won't have anticipated your future requirements correctly. See <> for one methodology that helps us avoid this trap. ((("Test-Driven Development (TDD)", "philosophy of", "YAGNI"))) ((("YAGNI (You ain’t gonna need it!)"))) ******************************************************************************* ================================================ FILE: chapter_08_prettification.asciidoc ================================================ [[chapter_08_prettification]] == Prettification: Layout and Styling, [.keep-together]#and What to Test About It# ((("layout", see="CSS; design and layout testing"))) ((("style", see="CSS; design and layout testing"))) We're starting to think about releasing the first version of our site, but we're a bit embarrassed by how unfinished it looks at the moment. In this chapter, we'll cover some of the basics of styling, including integrating an HTML/CSS framework called Bootstrap. We'll learn how static files work in Django, and what we need to do about testing them. === Testing Layout and Style ((("design and layout testing", "selecting test targets", id="DLTtargets08"))) Our site is undeniably a bit unattractive at the moment (<>). NOTE: If you spin up your dev server with `manage.py runserver`, you may run into a database error, something like this: "OperationalError: no such table: lists_list". You need to update your local database to reflect the changes we made in 'models.py'.((("manage.py file", "migrate")))((("databases", "local dev database out of sync with migrations")))((("IntegrityErrors"))) Use `manage.py migrate`. If it gives you any grief about `IntegrityErrors`, just delete the database file.footnote:[ What? Delete the database? Have you taken leave of your senses? Not completely. The local dev database often gets out of sync with its migrations as we go back and forth in our development, and it doesn't have any important data in it, so it's OK to blow it away now and again. We'll be much more careful once we have a "production" database on the server.] [role="pagebreak-before"] We can't be going back to https://oreil.ly/ruIZz[Python's historical reputation for being ugly], so let's do a tiny bit of polishing. Here are a few things we might want: * A large input field for adding to new and existing lists * A large, attention-grabbing, centered box to put it in ((("aesthetics, testing", seealso="design and layout testing"))) How do we apply TDD to these things?((("Test-Driven Development (TDD)", "testing behaviour of aesthetics"))) Most people will tell you that you shouldn't test aesthetics, and they're right. It's a bit like testing a constant, in that tests usually wouldn't add any value. [[homepage-looking-ugly]] .Our home page, looking a little ugly... image::images/tdd3_0801.png["Our home page, looking a little ugly."] ((("static files", "challenges of"))) ((("CSS (Cascading Style Sheets)", "challenges of static files"))) But we can test the essential _behaviour_ of our aesthetics (i.e., that we have any at all). All we want to do is reassure ourselves that things are working. For example, we're going to use Cascading Style Sheets (CSS) for our styling, and they are loaded as static files. Static files can be a bit tricky to configure (especially, as we'll see later, when you move off your own computer and onto a server), so we'll want some kind of simple "smoke test" that the CSS has loaded. We don't have to test fonts and colours and every single pixel, but we can do a quick check that the main input box is aligned the way we want it on each page, and that will give us confidence that the rest of the styling for that page is probably loaded too. [role="pagebreak-before"] Let's add a new test method inside our functional test (FT): [role="sourcecode"] .functional_tests/tests.py (ch08l001) ==== [source,python] ---- class NewVisitorTest(LiveServerTestCase): [...] def test_layout_and_styling(self): # Edith goes to the home page, self.browser.get(self.live_server_url) # Her browser window is set to a very specific size self.browser.set_window_size(1024, 768) # She notices the input box is nicely centered inputbox = self.browser.find_element(By.ID, "id_new_item") self.assertAlmostEqual( inputbox.location["x"] + inputbox.size["width"] / 2, 512, delta=10, ) ---- ==== A few new things here. We start by setting the window size to a fixed size. We then find the input element, look at its size and location, and do a little maths to check whether it seems to be positioned in the middle of the page. `assertAlmostEqual` helps us to deal with rounding errors and the occasional weirdness due to scrollbars and the like, by letting us specify that we want our arithmetic to work to within 10 pixels, plus or minus. If we run the FTs, we get: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests*] [...] .F. ====================================================================== FAIL: test_layout_and_styling (functional_tests.tests.NewVisitorTest.test_layout_and_styling) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/functional_tests/tests.py", line 119, in test_layout_and_styling self.assertAlmostEqual( [...] AssertionError: 102.5 != 512 within 10 delta (409.5 difference) --------------------------------------------------------------------- Ran 3 tests in 9.188s FAILED (failures=1) ---- That's the expected failure. Still, this kind of FT is easy to get wrong, so let's use a quick-and-dirty "cheat" solution, to check that the FT definitely passes when the input box is centered. We'll delete this code again almost as soon as we've used it to check the FT: [role="sourcecode small-code"] .lists/templates/home.html (ch08l002) ==== [source,html] ----

{% csrf_token %} ---- ==== That passes, which means the FT works. Let's extend it to make sure that the input box is also center-aligned on the page for a new list: [role="sourcecode"] .functional_tests/tests.py (ch08l003) ==== [source,python] ---- # She starts a new list and sees the input is nicely # centered there too inputbox.send_keys("testing") inputbox.send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: testing") inputbox = self.browser.find_element(By.ID, "id_new_item") self.assertAlmostEqual( inputbox.location["x"] + inputbox.size["width"] / 2, 512, delta=10, ) ---- ==== That gives us another test failure: ---- File "...goat-book/functional_tests/tests.py", line 131, in test_layout_and_styling self.assertAlmostEqual( AssertionError: 102.5 != 512 within 10 delta (409.5 difference) ---- Let's commit just the FT: [subs="specialcharacters,quotes"] ---- $ *git add functional_tests/tests.py* $ *git commit -m "first steps of FT for layout + styling"* ---- [role="pagebreak-before"] Now it feels like we're justified in finding a "proper" solution to improve the styling for our site. We can back out our hacky `text-align: center`: [subs="specialcharacters,quotes"] ---- $ *git reset --hard* ---- WARNING: `git reset --hard` is the "take off and nuke the site from orbit" Git command, so be careful with it--it blows away all your un-committed changes. Unlike almost everything else you can do with Git, there's no way of going back after this one.((("Git", "reset --hard")))((("", startref="DLTtargets08"))) === Prettification: Using a CSS Framework ((("design and layout testing", "CSS frameworks", id="DLTcssframe08"))) ((("CSS (Cascading Style Sheets)", "CSS frameworks", id="CSSframe08"))) ((("Bootstrap", "downloading"))) UI design is hard, and doubly so now that we have to deal with mobile, tablets, and so forth. That's why many programmers, particularly lazy ones like me, turn to CSS frameworks to solve some of those problems for them. There are lots of frameworks out there, but one of the earliest and most popular still, is Bootstrap. Let's use that. You can find Bootstrap at https://getbootstrap.com[getbootstrap.com]. We'll download it and put it in a new folder called _static_ inside the `lists` app:footnote:[On Windows, you may not have `wget` and `unzip`, but I'm sure you can figure out how to download Bootstrap, unzip it, and put the contents of the _dist_ folder into the _lists/static/bootstrap_ folder.] [subs="specialcharacters,quotes"] ---- $ *wget -O bootstrap.zip https://github.com/twbs/bootstrap/releases/download/\ v5.3.5/bootstrap-5.3.5-dist.zip* $ *unzip bootstrap.zip* $ *mkdir lists/static* $ *mv bootstrap-5.3.5-dist lists/static/bootstrap* $ *rm bootstrap.zip* ---- Bootstrap comes with a plain, uncustomised installation in the 'dist' folder. We're going to use that for now, but you should really never do this for a real site--vanilla Bootstrap is instantly recognisable, and a big signal to anyone in the know that you couldn't be bothered to style your site. Learn how to use Sass and change the font, if nothing else! There is info in Bootstrap's docs, or read an https://www.freecodecamp.org/news/how-to-customize-bootstrap-with-sass[introductory guide]. [role="pagebreak-before"] Our 'lists' folder will end up looking like this: [subs="specialcharacters,macros"] ---- [...] ├── lists │   ├── __init__.py │   ├── admin.py │   ├── apps.py │   ├── migrations │   │   ├── [...] │   ├── models.py │   ├── static │   │   └── bootstrap │   │   ├── css │   │   │   ├── bootstrap-grid.css │   │   │   ├── bootstrap-grid.css.map │   │   │   ├── [...] │   │   │   └── bootstrap.rtl.min.css.map │   │   └── js │   │   ├── bootstrap.bundle.js │   │   ├── bootstrap.bundle.js.map │   │   ├── [...] │   │   └── bootstrap.min.js.map │   ├── templates │   │   ├── home.html │   │   └── list.html │   ├── tests.py │   ├── urls.py │   └── views.py [...] ---- ((("Bootstrap", "documentation"))) Look at the "Getting started" section of the https://getbootstrap.com/docs/5.3/getting-started/introduction[Bootstrap documentation]; you'll see it wants our HTML template to include something like this: [role="skipme"] [source,html] ---- Bootstrap demo

Hello, world!

---- We already have two HTML templates. We don't want to be adding a whole load of boilerplate code to each, so now feels like the right time to apply the "Don't repeat yourself" rule, and bring all the common parts together. Thankfully, the Django template language makes that easy using something called template inheritance. ((("", startref="DLTcssframe08"))) ((("", startref="CSSframe08"))) === Django Template Inheritance ((("design and layout testing", "Django template inheritance"))) ((("templates", "Django template inheritance"))) ((("Django framework", "template inheritance"))) Let's have a little review of what the differences are between 'home.html' and 'list.html': [subs="specialcharacters,macros"] ---- $ pass:quotes[*diff lists/templates/home.html lists/templates/list.html*] <

Start a new To-Do list

<
--- >

Your To-Do list

> [...] > > {% for item in list.item_set.all %} > > {% endfor %} >
{{ forloop.counter }}: {{ item.text }}
---- They have different header texts, and their forms use different URLs. On top of that, 'list.html' has the additional `` element. //IDEA add a note re downsides of inheritance? Now that we're clear on what's in common and what's not, we can make the two templates inherit from a common "superclass" template. We'll start by making a copy of 'list.html': [subs="specialcharacters,quotes"] ---- $ *cp lists/templates/list.html lists/templates/base.html* ---- //006 We make this into a base template, which just contains the common boilerplate, and mark out the "blocks", places where child templates can customise it: [role="sourcecode small-code"] .lists/templates/base.html (ch08l007) ==== [source,html] ---- To-Do lists

{% block header_text %}{% endblock %}

{% csrf_token %} {% block table %} {% endblock %} ---- ==== [role="pagebreak-before"] Let's see how these blocks are used in practice, by changing 'home.html' so that it "inherits" from 'base.html': [role="sourcecode"] .lists/templates/home.html (ch08l008) ==== [source,html] ---- {% extends 'base.html' %} {% block header_text %}Start a new To-Do list{% endblock %} {% block form_action %}/lists/new{% endblock %} ---- ==== You can see that lots of the boilerplate HTML disappears, and we just concentrate on the bits we want to customise. We do the same for 'list.html': [role="sourcecode"] .lists/templates/list.html (ch08l009) ==== [source,html] ---- {% extends 'base.html' %} {% block header_text %}Your To-Do list{% endblock %} {% block form_action %}/lists/{{ list.id }}/add_item{% endblock %} {% block table %}
{% for item in list.item_set.all %} {% endfor %}
{{ forloop.counter }}: {{ item.text }}
{% endblock %} ---- ==== That's a refactor of the way our templates work. We rerun the FTs to make sure we haven't broken anything: ---- AssertionError: 102.5 != 512 within 10 delta (409.5 difference) ---- Sure enough, they're still getting to exactly where they were before. That's worthy of a commit: ((("Git", "diff -w"))) [subs="specialcharacters,quotes"] ---- $ *git diff -w* # the -w means ignore whitespace, useful since we've changed some html indenting $ *git status* $ *git add lists/templates* # leave static, for now $ *git commit -m "refactor templates to use a base template"* ---- === Integrating Bootstrap ((("design and layout testing", "Bootstrap integration"))) ((("Bootstrap", "integrating"))) Now it's much easier to integrate the boilerplate code that Bootstrap wants--we won't add the JavaScript yet, just the CSS: [role="sourcecode"] .lists/templates/base.html (ch08l010) ==== [source,html] ---- To-Do lists [...] ---- ==== ==== Rows and Columns Finally, let's actually use some of the Bootstrap magic! You'll have to read the documentation yourself, but we should be able to use a combination of the grid system and the `justify-content-center` class to get what we want: [role="sourcecode"] .lists/templates/base.html (ch08l011) ==== [source,html] ----

{% block header_text %}{% endblock %}

{% csrf_token %}
{% block table %} {% endblock %}
---- ==== (If you've never seen an HTML tag broken up over several lines, that `` may be a little shocking. It is definitely valid, but you don't have to use it if you find it offensive.) TIP: Take the time to browse through the https://getbootstrap.com/docs/5.3/getting-started/introduction/[Bootstrap documentation], if you've never seen it before. It's a shopping trolley brimming full of useful tools to use in your site. Does that work? Whoops—no, we have an error in our unit tests: ---- FAIL: test_renders_input_form (lists.tests.ListViewTest.test_renders_input_form) [...] AssertionError: False is not true : Couldn't find '` tag, which actually don't matter semantically. Django does provide the `html=True` argument to `assertContains()`, which does help a bit, but it requires exhaustively specifying every attribute of the element we want to check on, like this: [role="sourcecode small-code"] .lists/tests.py (ch08l011-1) ==== [source,python] ---- class HomePageTest(TestCase): def test_uses_home_template(self): [...] def test_renders_input_form(self): response = self.client.get("/") self.assertContains(response, '
') self.assertContains( response, '', html=True, ) [...] class ListViewTest(TestCase): def test_uses_list_template(self): [...] def test_renders_input_form(self): mylist = List.objects.create() response = self.client.get(f"/lists/{mylist.id}/") self.assertContains( response, f'', ) self.assertContains( response, '', html=True, ) ---- ==== That's not entirely satisfactory, because all those extra attributes like `id` and `placeholder` aren't really things we want to nail down in unit tests; we'd rather have the freedom to change them in the template without needing to change the tests as well. They're more of a presentation concern than a true part of the contract between backend and frontend. But it does get the tests to pass: ---- OK ---- [role="pagebreak-before"] So, for now, let's make a note to come back to it: [role="scratchpad"] ***** * _Find a better way to unit test form & input elements._ ***** So, the unit tests are happy. What about the FTs? ---- AssertionError: 102.5 != 512 within 10 delta (409.5 difference) ---- Hmm. No. Why isn't our CSS loading? If you try it manually with `runserver` and look around in DevTools, you'll see the browser 404ing when it tries to fetch _bootstrap.min.css_. If you watch the `runserver` terminal session, you'll also see the 404s there, as in <>. [[bootstrap_css_404_devtools]] .That's a nope on bootstrap.css image::images/tdd3_0802.png["Browser DevTools showing a 404 for css/bootstrap.min.css, but also at the bottom of the screenshot, the terminal window showing the same URL returing a 404 in the runserver session."] To figure out what's happening, let's talk a bit about how Django deals with static files. [role="pagebreak-before less_space"] === Static Files in Django ((("Django framework", "static files in", id="DJFstatic08"))) Django, and indeed any web server, needs to know two things to deal with static files: 1. How to tell when a URL request is for a static file, as opposed to for some HTML that's going to be served via a view function 2. Where to find the static file that the user wants In other words, static files ((("URL mappings", "for static files", secondary-sortas="static")))are a mapping from URLs to files on disk. ((("static files", "URL requests for"))) For item 1, Django lets us define a URL "prefix" to say that any URLs that start with that prefix should be treated as requests for static files. By default, the prefix is [keep-together]#'/static/'#. It's already defined in _settings.py_: [role="sourcecode currentcontents"] .superlists/settings.py ==== [source,python] ---- [...] # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.2/howto/static-files/ STATIC_URL = "static/" ---- ==== ((("static files", "finding"))) The rest of the settings that we will add to this section all have to do with item 2: finding the actual static files on disk. While we're using the Django development server (`manage.py runserver`), we can rely on Django to magically find static files for us--it'll just look in any subfolder of one of our apps called _static_. You now see why we put all the Bootstrap static files into _lists/static_. So, why are they not working at the moment? It's because we're not using the `/static/` URL prefix. Have another look at the link to the CSS in _base.html_: [role="sourcecode currentcontents"] .lists/templates/base.html [source,html] ---- ---- That `href` is just what happened to be in the Bootstrap docs. To get it to work, we need to change it to: [role="sourcecode small-code"] .lists/templates/base.html (ch08l012) ==== [source,html] ---- ---- ==== // DAVID: Django best practice would be to use the static tag instead. // https://docs.djangoproject.com/en/5.2/howto/static-files/#configuring-static-files Now when `runserver` sees the request, it knows that it's for a static file because it begins with `/static/`. It then tries to find a file called _bootstrap/css/bootstrap.min.css_, looking in each of our app folders for subfolders called _static_, and it should find it at _lists/static/bootstrap/css/bootstrap.min.css_. So if you take a look manually, you should see it works, as in <>. [[list-page-centered]] .Our site starts to look a little better... image::images/tdd3_0803.png["The list page with centered header."] ==== Switching to StaticLiveServerTestCase ((("StaticLiveServerTestCase"))) If you run the FT though, annoyingly, it still won't pass: ---- AssertionError: 102.5 != 512 within 10 delta (409.5 difference) ---- That's because, although `runserver` automagically finds static files, +Live⁠S⁠e⁠r⁠v⁠e⁠r​T⁠e⁠s⁠t⁠Case+ doesn't. Never fear, though: the Django developers have made an even [.keep-together]#more magical# test class called `StaticLiveServerTestCase` (see https://oreil.ly/mh-iO[the docs]). // JAN: Maybe you could mention that StaticLiveServerTestCase inherits from LiveServerTestCase - so all previous should work + static files. After reading the name, I imagined StaticLiveServerTestCase as some special test class for testing only static-related stuff Let's switch to that: [role="sourcecode"] .functional_tests/tests.py (ch08l013) ==== [source,diff] ---- @@ -1,14 +1,14 @@ -from django.test import LiveServerTestCase +from django.contrib.staticfiles.testing import StaticLiveServerTestCase from selenium import webdriver from selenium.common.exceptions import WebDriverException from selenium.webdriver.common.keys import Keys import time MAX_WAIT = 10 -class NewVisitorTest(LiveServerTestCase): +class NewVisitorTest(StaticLiveServerTestCase): def setUp(self): ---- ==== //008 [role="pagebreak-before"] And now it will find the new CSS, which will get our test to pass: ((("", startref="DJFstatic08"))) [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests*] Creating test database for alias 'default'... ... --------------------------------------------------------------------- Ran 3 tests in 9.764s ---- // (David): Incidentally, when I ran this the first time I got this error // on the second test case. selenium.common.exceptions.NoSuchElementException: // Message: Unable to locate element: [id="id_new_item"]; // I ran it again and it worked. Hooray! === Using Bootstrap Components to Improve the Look of the Site ((("design and layout testing", "Bootstrap tools")))Let's see if we can do even better, using some of the other tools in Bootstrap's panoply.((("Bootstrap", "using components of to improve looks of site", id="ix_Bootuse"))) ==== Jumbotron! The first version of Bootstrap used to ship with a class called `jumbotron` for things that are meant to be particularly prominent on the page. It doesn't exist anymore, but old-timers like me still pine for it, so they have a specific page in the docs that tells you how to re-create it.((("jumbotron (Bootstrap)"))) Essentially, we massively embiggen the main page header and the input form, putting it into a grey box with nice rounded corners: [role="sourcecode"] .lists/templates/base.html (ch08l014) ==== [source,html] ----

{% block header_text %}{% endblock %}

[...] ---- ==== That ends up looking something like <>. [[jumbotron-header]] .A big grey box at the top of the page image::images/tdd3_0804.png["The home page with a big grey box surrounding the title and input"] TIP: When hacking about with design and layout, it's best to have a window open that we can refresh frequently. Use `python manage.py runserver` to spin up the dev server, and then browse to __http://localhost:8000__ to see your work as we go. // JAN: You could mention force refresh here (Cmd + Shift + R; Ctrl + F5, ...). It comes handy many times when working with CSS etc. ==== Large Inputs ((("Bootstrap", "large inputs"))) ((("form control classes (Bootstrap)"))) The `jumbotron` is a good start, but now the input box has tiny text compared to everything else. Thankfully, Bootstrap's form control classes offer an option to set an input to "large": [role="sourcecode"] .lists/templates/base.html (ch08l015) ==== [source,html] ---- ---- ==== ==== Table Styling ((("Bootstrap", "table styling"))) ((("table styling (Bootstrap)"))) The table text also looks too small compared to the rest of the page now. Adding the Bootstrap `table` class improves things, over in _list.html_: [role="sourcecode"] .lists/templates/list.html (ch08l016) ==== [source,html] ---- ---- ==== ==== Optional: Dark Mode In contrast to my greybeard nostalgia for `jumbotron`, here's something relatively new to Bootstrap: dark mode!((("dark mode (Bootstrap)")))((("Bootstrap", "dark mode"))) [role="sourcecode"] .lists/templates/base.html (ch08l017) ==== [source,html] ---- ---- ==== Take a look at <>. I think that looks great! [[dark-modeee]] .Dark modeeeeeeeeee image::images/tdd3_0805.png["Screenshot of lists page in dark mode. Cool."] But it's very much a matter of personal preference, and my editor will have kittens if I make all the rest of my screenshots use so much ink, so I'm going to revert it for now. You're free to keep dark mode on if you like! [role="pagebreak-before less_space"] ==== A Semi-Decent Page Getting it into shape took me a few goes, but I'm reasonably happy with it now (<>). [[homepage-looking-better]] .The lists page, looking...good enough for now image::images/tdd3_0806.png["Screenshot of lists page in light mode with decent styling."] If you want to go further with customising Bootstrap, you need to get into compiling Sass.((("Sass/SCSS"))) I've said it already, but I _definitely_ recommend taking the time to do that someday. Sass/SCSS is a great improvement on plain old CSS, and a useful tool even if you don't use Bootstrap.((("CSS (Cascading Style Sheets)", "Sass/SCSS improvement on"))) A last run of the FTs, to see if everything still works OK: [role="dofirst-ch08l018"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests*] [...] ... --------------------------------------------------------------------- Ran 3 tests in 10.084s OK ---- That's it! Definitely time for a commit: [subs="specialcharacters,quotes"] ---- $ *git status* # changes tests.py, base.html, list.html, settings.py, # and untracked lists/static $ *git add .* $ *git status* # will now show all the bootstrap additions $ *git commit -m "Use Bootstrap to improve layout"* ---- === Parsing HTML for Less Brittle Tests of Key HTML Content Oh whoops, we nearly forgot our scratchpad: [role="scratchpad"] ***** * _Find a better way to unit test form & input elements._ ***** When working on layout and styling, you expect to spend most of your time in the browser, in a cycle of tweaking your HTML and refreshing to see the effects, with occasional runs of your layout FT, if you have one.((("HTML", "parsing for less brittle tests of content"))) You wouldn't expect to test-drive design with unit tests. And sure enough, we haven't run them in a while. Because if we had done, we'd have noticed that they're failing: ---- FAIL: test_renders_input_form (lists.tests.HomePageTest.test_renders_input_form) [...] AssertionError: False is not true : Couldn't find '' in the following response b'\n\n\n \n To-Do [...] <input\n class="form-control form-control-lg"\n name="item_text"\n id="id_new_item"\n placeholder="Enter a to-do item"\n />\n <input [...] FAIL: test_renders_input_form (lists.tests.ListViewTest.test_renders_input_form) [...] ---- It's also annoyingly hard to see from the tests output, but it happened when we introduced the `class=form-control form-control-lg`. We really don't want this sort of thing breaking our unit tests. Using string matching, even whitespace-aware string matching, is just the wrong tool for the job.footnote:[ As famously explained in a https://oreil.ly/N-cIc[classic Stack Overflow post].] Let's switch to using a proper HTML parser, the venerable https://lxml.de[lxml]. [subs=""] ---- $ <strong>pip install 'lxml[cssselect]'</strong> Collecting lxml[cssselect] [...] Collecting cssselect>=0.7 (from lxml[cssselect]) [...] Installing collected packages: lxml, cssselect Successfully installed [...] ---- (We need the `cssselect` add-on for the nice CSS selectors.)((("lxml parser"))) And here's how we use it to write a more focused version of our test that only cares about the two HTML attributes that actually matter to the integration of frontend and backend: 1. The `<form>` tag's `method` and `action` 2. The `<input>` tag's `name` [role="sourcecode"] .lists/tests.py (ch08l019) ==== [source,python] ---- import lxml.html [...] class HomePageTest(TestCase): def test_uses_home_template(self): [...] def test_renders_input_form(self): response = self.client.get("/") parsed = lxml.html.fromstring(response.content) # <1> [form] = parsed.cssselect("form[method=POST]") # <2><3> self.assertEqual(form.get("action"), "/lists/new") [input] = form.cssselect("input[name=item_text]") # <4> ---- ==== <1> Here's where we parse the HTML into a structured object to represent the DOM (document object model). <2> Here's where we use a CSS selector to find our form, implicitly also checking that it has `method="POST"`. The `cssselect()` method returns a list of matching elements. <3> The `[form] =` is worth a mention. What we're using here is a special assignment syntax called "unpacking", where the lefthand side is a list of variable names and the righthand side is a list of values.((("unpacking")))((("tuple unpacking and multiple assignment"))) It's a bit like saying `form = parsed.cssselect("form[method=POST]")[0]`, but a bit nicer to read, and a bit more strict too. By only putting one element on the left, we're effectively asserting that there is exactly one element on the right; if there isn't, we'll get an error.footnote:[ Read more about tuple unpacking and multiple assignment https://oreil.ly/LMfuB[on Trey Hunner's excellent blog].] <4> We use the same kind of assignment to assert that the form contains exactly one input element with the name `item_text`. Here's the same thing in `ListViewTest`: [role="sourcecode"] .lists/tests.py (ch08l020) ==== [source,python] ---- class ListViewTest(TestCase): def test_uses_list_template(self): [...] def test_renders_input_form(self): mylist = List.objects.create() response = self.client.get(f"/lists/{mylist.id}/") parsed = lxml.html.fromstring(response.content) [form] = parsed.cssselect("form[method=POST]") self.assertEqual(form.get("action"), f"/lists/{mylist.id}/add_item") [input] = form.cssselect("input[name=item_text]") ---- ==== That works! ---- Ran 10 tests in 0.017s OK ---- And as always, for any test you've only ever seen green, it's nice to introduce a deliberate failure: [role="sourcecode"] .lists/templates/base.html (ch08l021) ==== [source,python] ---- @@ -18,7 +18,7 @@ <form method="POST" action="{% block form_action %}{% endblock %}"> <input class="form-control form-control-lg" - name="item_text" + name="geoff" id="id_new_item" placeholder="Enter a to-do item" /> ---- ==== [role="pagebreak-before"] And let's see the error message: ---- [input] = form.cssselect("input[name=item_text]") ^^^^^^^ ValueError: not enough values to unpack (expected 1, got 0) ---- Hmm you know what? I'm actually not happy with that. The `[input] =` syntax is probably another example of me being too clever for my own good. Let's try something else that will give us a clearer message about what _is_ on the page and what isn't: [role="sourcecode"] .lists/tests.py (ch08l022) ==== [source,python] ---- inputs = form.cssselect("input") # <1> self.assertIn("item_text", [input.get("name") for input in inputs]) # <2> ---- ==== <1> We'll get a list of all the inputs in the form. <2> And then we'll assert that at least one of them has the right `name=`. That gives us a more self-explanatory message: ---- self.assertIn("item_text", [input.get("name") for input in inputs]) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'item_text' not found in ['geoff', 'csrfmiddlewaretoken'] ---- Now I feel good about changing our HTML back: [role="sourcecode"] .lists/templates/base.html (ch08l023) ==== [source,diff] ---- @@ -18,7 +18,7 @@ <form method="POST" action="{% block form_action %}{% endblock %}"> <input class="form-control form-control-lg" - name="geoff" + name="item_text" id="id_new_item" placeholder="Enter a to-do item" /> ---- ==== Much better! [subs="specialcharacters,quotes"] ---- $ *git diff* # tests.py $ *git commit -am "use lxml for more specific unit test asserts on html content"* ---- [role="pagebreak-before less_space"] === What We Glossed Over: collectstatic and Other Static Directories ((("design and layout testing", "collecting static files for deployment", id="DLTcollect08"))) ((("static files", "collecting for deployment", id="SFcollect08"))) ((("collectstatic command", id="collect08"))) We saw earlier that the Django dev server will magically find all your static files inside app folders, and serve them for you. That's fine during development, but when you're running on a real web server, you don't want Django serving your static content--using Python to serve raw files is slow and inefficient, and a web server like Apache or nginx can do this all for you. For these reasons, you want to be able to gather all your static files from inside their various app folders and copy them into a single location, ready for deployment. This is what the `collectstatic` command is for. The destination, the place where the collected static files go, needs to be defined in _settings.py_ as `STATIC_ROOT`. In the next chapter, we'll be doing some deployment, so let's actually experiment with that now. A common and straightforward place to put it is in a folder called "static" in the root of our repo: [role="skipme"] ---- . ├── db.sqlite3 ├── functional_tests/ ├── lists/ ├── manage.py ├── static/ └── superlists/ ---- Here's a neat way of specifying that folder, making it relative to the location of the project base directory: [role="sourcecode"] .superlists/settings.py (ch08l024) ==== [source,python] ---- # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.2/howto/static-files/ STATIC_URL = "static/" STATIC_ROOT = BASE_DIR / "static" ---- ==== Take a look at the top of the settings file, and you'll see how that `BASE_DIR` variable is helpfully defined for us, using `pathlib.Path` and `__file__` (both really nice Python built-ins).footnote:[ Notice in the `Pathlib` wrangling of `__file__` that the `.resolve()` happens before anything else. Always follow this pattern when working with `__file__`, otherwise you can see unpredictable behaviours depending on how the file is imported. Thanks to https://github.com/CleanCut/green[Green Nathan] for that tip!] [role="pagebreak-before"] Anyway, let's try running `collectstatic`: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py collectstatic*] 171 static files copied to '...goat-book/static'. ---- And if we look in './static', we'll find all our CSS files: [subs="specialcharacters,quotes"] ---- $ *tree static/* static/ ├── admin │   ├── css │   │   ├── autocomplete.css │   │   ├── [...] [...] │   └── xregexp.min.js └── bootstrap ├── css │   ├── bootstrap-grid.css │   ├── [...] │ └── bootstrap.rtl.min.css.map └── js ├── bootstrap.bundle.js ├── [...] └── bootstrap.min.js.map 17 directories, 171 files ---- `collectstatic` has also picked up all the CSS for the admin site. The admin site is one of Django's powerful features, but we don't need it for our simple site, so let's disable it for now: [role="sourcecode"] .superlists/settings.py (ch08l025) ==== [source,python] ---- INSTALLED_APPS = [ # "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", "lists", ] ---- ==== And we try again: [subs="specialcharacters,macros"] ---- $ pass:quotes[*rm -rf static/*] $ pass:quotes[*python manage.py collectstatic*] 44 static files copied to '...goat-book/static'. ---- Much better. Now we know how to collect all the static files into a single folder, where it's easy for a web server to find them. We'll find out all about that, including how to test it, in the next chapter! ((("", startref="DLTcollect08"))) ((("", startref="SFcollect08"))) ((("", startref="collect08"))) For now, let's save our changes to _settings.py_. We'll also add the top-level static folder to our `gitignore`, because it will only contain copies of files we actually keep in individual apps' static folders: [subs="specialcharacters,quotes"] ---- $ *git diff* # should show changes in settings.py $ *echo /static >> .gitignore* $ *git commit -am "set STATIC_ROOT in settings and disable admin"* ---- === A Few Things That Didn't Make It Inevitably this was only a whirlwind tour of styling and CSS, and there were several topics that I'd considered covering that didn't make it. Here are a few candidates for further study: * The `{% static %}` template tag, for more DRY and fewer hardcoded URLs * Client-side packaging tools, like `npm` and `bower` * Customising Bootstrap with Sass //RITA: Would you want to point readers to any resources, such as a website or another book for example? You don't have to. .Recap: On Testing Design and Layout ******************************************************************************* ((("design and layout testing", "best practices for"))) The tl;dr is: you shouldn't write tests for design and layout per se. It's too much like testing a constant, and the tests you write are often brittle. With that said, the _implementation_ of design and layout involves something quite tricky: CSS and static files. As a result, it is valuable to have some kind of minimal "smoke test" that checks that your static files and CSS are working. As we'll see in the next chapter, it can help pick up problems when you deploy your code to production. Similarly, if a particular piece of styling required a lot of client-side JavaScript code to get it to work (dynamic resizing is one I've spent a bit of time on), you'll definitely want some tests for that (see <<chapter_17_javascript>>). Try to write the minimal tests that will give you the confidence that your design and layout is working, without testing _what_ it actually is. That includes unit tests! Avoid asserting on the cosmetic aspects of your HTML in your unit tests. Aim to leave yourself in a position where you can freely make changes to the design and layout, without having to go back and adjust tests all the time. ******************************************************************************* ================================================ FILE: chapter_09_docker.asciidoc ================================================ [[chapter_09_docker]] == Containerization aka Docker [quote, Malvina Reynolds] ______________________________________________________________ Little boxes, all the same ______________________________________________________________ In this chapter, we'll start by adapting our FTs so that they can run against a container. And then we'll set about containerising our app, and getting those tests passing our code running inside Docker: * We'll build a minimal Dockerfile with everything we need to run our site. * We'll learn how to build and run a container on our machine. * We'll make a few changes to our source code layout, like using a _src_ folder. * We'll start flushing out a few issues around networking and the database. === Docker, Containers, and Virtualization Docker is a commercial product that wraps several free and open source technologies from the world of Linux, sometimes referred to as "containerization".((("Docker")))((("containerization"))) NOTE: Feel free to skip this section if you already know all about Docker. You may have already heard of the idea of "virtualization", which enables a single physical computer to pretend to be several machines.((("virtualization"))) Pioneered by IBM (amongst others) on mainframes in the 1960s, it rose to mainstream adoption in the '90s, where it was sold as a way to optimise resource usage in datacentres. AWS, for example, an offshoot of Amazon, was using virtualization already, and realised it could sell some spare capacity on its servers to customers outside the business.((("Amazon Web Services (AWS)")))((("AWS (Amazon Web Services)"))) So, when you come to deploy your code to a real server in a datacentre, it will be using virtualization. And, actually, you can use virtualization on your own machine, with software like VirtualBox or KVM. You can run Windows "inside" a Mac or Linux laptop, for example. But it can be fiddly to set up!((("virtualization", "containerization and"))) And nowadays, thanks to containerization, we can do better because containerization is a kind of even-more-virtual virtualization.((("VMs (virtual machines)"))) Conceptually, "regular" virtualization works at the hardware level: it gives you multiple virtual machines (VMs) that pretend to be different physical computers, on a single real machine. So you can run multiple operating systems using separate VMs on the same physical box, as in <<virtualization-diagram>>. [[virtualization-diagram]] .Physical versus virtual machines image::images/tdd3_0901.png["A diagram showing a physical machine, with an operating system and a Python virtualenv running inside it, vs multiple virtual machines running different operating systems on a single real machine"] // TODO; remove virtualenvs from this diagram, they just confuse things. // add another diagram later to contrast venvs with dockers. Containerization works at the operating system (OS) level: it gives you multiple virtual operating systems that all run on a single real OS.footnote:[ It's more accurate to say that containers share the same kernel as the host OS.((("operating system (OS), containerization at OS level"))) An operating system is made up of a kernel, and a bunch of utility programs that run on top of it. The kernel is the core of the OS; it's the program that runs all the other programs. Whenever your program needs to interact with the outside world, read a file, or talk to the internet, or start another program, it actually asks the kernel to do it. Starting about 15 years ago, the Linux kernel grew the ability to show different filesystems to different programs, as well as isolate them into different network and process namespaces; these are the capabilities that underpin Docker and containerization.] Containers let us pack the source code((("containers"))) and the system dependencies (like Python or system libraries) together, and our programs run inside separate virtual systems, using a single real host OS and kernel.footnote:[ Because containers all share the same kernel, while virtualization can let you run Windows and Linux on the same machine, containers on Linux hosts all run Linux, and ones on Windows hosts all run Windows. If you're running Linux containers on a Mac or a PC, it's because you're actually running them on a Linux VM under the hood.] See <<containers-diagram>> for an illustration. The upshot of this is that containers are much "cheaper". You can start one up in milliseconds, and you can run hundreds on the same machine. NOTE: If you're new to all this, I know it's a lot to wrap your head around! It takes a while to build a good mental model of what's happening.((("Docker", "resources on containers")))((("containers", "Docker resources on"))) Have a look at https://www.docker.com/resources/what-container[Docker's resources on containers] for more explanation. Hopefully, following along with these chapters and seeing them working in practice will help you to better understand the theory. [[containers-diagram]] .Containers share a kernel in the host operating system image::images/tdd3_0902.png["Diagram showing one or more containers running on a single host operating system, showing that each container uses the kernel from the host OS, but is able to have its own filesystem, based on an image, but also possibly mounting directories from the host filesystem"] ==== Why Not Just Use a Virtualenv? You might be thinking that this sounds a lot like a virtualenv—and you'd be right! Virtualenvs already let us run different versions of Python, with different Python packages, on the same machine.((("virtualenv (virtual environment)", "capabilities of Docker and containers versus")))((("Docker", "capabilities of containers versus virtualenvs")))((("containers", "capabilities of versus virtualenvs"))) What Docker containers give us over and above virtualenvs, is the ability to have different _system_ dependencies too; things you can't `pip install`, in other words. In the Python world, this could be C libraries, like `libpq` for PostgreSQL, or `libxml2` for parsing XML. But you could also run totally different programming languages in different containers, or even different Linux distributions. So, server administrators or platform people like them because it's one system for running any kind of software, and they don't need to understand the intricacies of any particular language's packaging systems. ==== Docker and Your CV That's all well and good for the _theoretical_ justification, but let's get to the _real_ reason for using this technology, which, as always, is: "it's fashionable so it's going to look good on my CV".((("Docker", "use of, asset on your CV")))((("CV (curriculum vitae), Docker and"))) For the purposes of this book, that's not such a bad justification really! Yes, it's going to be a nice way to have a "pretend" deployment on our own machine, before we try the real one--but also, containers are so popular nowadays, that it's very likely that you're going to encounter them at work (if you haven't already). For many working developers, a container image is the final artifact of their work; it's what they "deliver", and often the rest of the deployment process is something they rarely have to think about. In any case, without further ado, let's get into it! === As Always, Start with a Test ((("environment variables")))((("LiveServerTestCase"))) Let's adapt our functional tests (FTs) so that they can run against a standalone server, instead of the one that `LiveServerTestCase` creates for us. Do you remember I said that `LiveServerTestCase` had certain limitations? Well, one is that it always assumes you want to use its own test server, which it makes available at `self.live_server_url`. I still want to be able to do that _sometimes_, but I also want to be able to selectively tell it not to bother, and to use a real server instead.((("TEST_SERVER environment variable"))) [role="pagebreak-before"] We'll do it by checking for an environment variable called `TEST_SERVER`: //IDEA; the word "server" is overloaded. // here we mean docker containers, later we mean a real server. TEST_HOST?? [role="sourcecode"] .functional_tests/tests.py (ch09l001) ==== [source,python] ---- import os [...] class NewVisitorTest(StaticLiveServerTestCase): def setUp(self): self.browser = webdriver.Firefox() if test_server := os.environ.get("TEST_SERVER"): # <1><2> self.live_server_url = "http://" + test_server # <3> ---- ==== <1> Here's where we check for the env var.((("walrus operator (:=)")))(((":= (walrus) operator"))) <2> If you haven't seen this before, the `:=` is known as the "walrus operator" (more formally, it's the operator for an "assignment expression"), which was a controversial new feature from Python 3.8footnote:[ The feature was a favourite of Guido van Rossum's, but the discussion around it was so toxic that Guido stepped down from his role as Python's BDFL, or "Benevolent Dictator for Life".] and it's not often useful, but it is quite neat for cases like this, where you have a variable and want to do a conditional on it straight away. See https://oreil.ly/oDyYs[this article] for more explanation. <3> Here's the hack: we replace `self.live_server_url` with the address of our "real" server. NOTE: A clarification: when we say we run tests _against_ our Docker container, or _against_ our staging server, that doesn't mean we run the tests _from_ Docker or _from_ our staging server.((("Test-Driven Development (TDD)", "concepts", "running tests against"))) We still run the tests from our own laptop, but they target the place that's running our code. We test that said hack hasn't broken anything by running the FTs [keep-together]#"normally"#: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python manage.py test functional_tests*] [...] Ran 3 tests in 8.544s OK ---- And now we can try them against our Docker server URL—which, once we've done the right Docker magic, will be at _http://localhost:8888_. TIP: I'm deliberately choosing a different port to run Dockerised Django on (8888) from the default port that a local `manage.py runserver` would choose (8080). This is to avoid getting in the situation where I (or the tests) _think_ we're looking at Docker, when we're actually looking at a local `runserver` that I've left running in some terminal somewhere.((("Django framework", "running Dockerized Django"))) .Ports ******************************************************************************* Ports are what let you have multiple connections open at the same time on a single machine; the reason you can load two different websites at the same time, for example.((("ports")))((("network adapters, range of ports"))) Each network adapter has a range of ports, numbered from 0 to 65535. In a client/server connection, the client knows the port of the server, and the client OS chooses a random local port for its side of the connection. When a server is "listening" on a port, no other service can bind to that port at the same time. That's why you can't run `manage.py runserver` in two different terminals at the same time, because both want to use port `8080` by default. ******************************************************************************* We'll use the `--failfast` option to exit as soon as((("--failfast option", primary-sortas="failfast"))) a single test fails: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 ./manage.py test functional_tests --failfast*] [...] E ====================================================================== ERROR: test_can_start_a_todo_list (functional_tests.tests.NewVisitorTest.test_can_start_a_todo_list) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/functional_tests/tests.py", line 38, in test_can_start_a_todo_list self.browser.get(self.live_server_url) [...] selenium.common.exceptions.WebDriverException: Message: Reached error page: abo ut:neterror?e=connectionFailure&u=http%3A//localhost%3A8888/[...] Ran 1 tests in 5.518s FAILED (errors=1) ---- NOTE: If, on Windows, you see an error saying something like "TEST_SERVER is not recognized as a command", it's probably because you're not using Git Bash.((("Git Bash")))((("Bash shell (Git Bash)"))) Take another look at the “<<pre-requisites>>” section. You can see that our tests are failing, as expected, because we're not running Docker yet. Selenium reports that Firefox is seeing an error and "cannot establish connection to the server", and you can see _localhost:8888_ in there too. The FT seems to be testing the right things, so let's commit: [subs="specialcharacters,quotes"] ---- $ *git diff* # should show changes to functional_tests.py $ *git commit -am "Hack FT runner to be able to test docker"* ---- TIP: Don't use `export` to set the `TEST_SERVER`` environment variable; otherwise, all your subsequent test runs in that terminal will be against staging, and that can be very confusing if you're not expecting it. Setting it explicitly inline each time you run the FTs is best. ==== Making a src Folder When preparing a codebase for deployment, it's often convenient to separate out the actual source code of our production app from the rest of the files that you need in the project. A folder called _src_ is a common convention.((("src folder"))) Currently, all our code is source code really, so we move everything into _src_ (we'll be seeing some new files appearing outside _src_ shortly):footnote:[ A common thing to find outside of the _src_ folder is a folder called _tests_. We won't be doing that while we're relying on the standard Django test framework, but it can be a good thing to do if you're using pytest, for example.] //002 [subs="specialcharacters,quotes"] ---- $ *mkdir src* $ *git mv functional_tests lists superlists manage.py src* $ *git commit -m "Move all our code into a src folder"* ---- === Installing Docker The https://docs.docker.com/get-docker[Docker documentation] is pretty good, and you'll find detailed installation instructions for Windows, Mac, and Linux.((("Docker", "installing"))) TIP: Choose WSL (Windows Subsystem for Linux) as your backend on Windows, as we'll need it in the next chapter.((("WSL (Windows Subsystem for Linux)")))((("Windows Subsystem for Linux (WSL)"))) You can find installation instructions https://learn.microsoft.com/en-us/windows/wsl/install[on the Microsoft website]. This doesn't mean you have to switch your development environment to being "inside" WSL; Docker just uses WSL as a virtualization engine in the background. You should be able to run all the `docker` CLI commands from the same Git Bash console you've been using so far. // TODO: appendix or link to more detailed instructions for WSL use? [[docker-alternatives]] .Docker Alternatives: Podman, nerdctl, etc ***************************************************************************************** Impartiality commands me to also mention https://podman.io[Podman] and https://github.com/containerd/nerdctl[nerdctl], both like-for-like replacements for Docker. They are both((("Docker", "alternatives to")))((("Podman")))((("nerdctl"))) pretty much exactly the same as Docker, arguably with a few advantages even.footnote:[ Docker uses a central "daemon" to manage containers, which Podman and nerdctl don't.] I actually tried Podman out on early drafts of this chapter (on Linux) and it worked perfectly well. But they are both a little less well established and documented; the Windows installation instructions are a little more DIY, for example. So in the end, although I'm always a fan of a plucky noncommercial upstart, I decided to stick with Docker for now. After all, the core of it is still open source, to its credit! But you could definitely check out one of the alternatives if you feel like it. You can follow along all the instructions in the book by just substituting the `docker` binary for `podman` or `nerdctl` in all the CLI instructions: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *docker run busybox echo hello* # becomes $ *podman run busybox echo hello* # or $ *nerdctl run busybox echo hello* # similarly with podman build, nerdtcl build, podman ps, etc. ---- ***************************************************************************************** .Colima: An Alternative Docker Runtime for macOS ***************************************************************************************** If you're on macOS, you might find the Docker Dekstop licensing terms don't work for you.((("Colima, alternative container runtime for MacOS"))) In that case, you can try https://github.com/abiosoft/colima[Colima], which is a "container runtime", essentially the backend for Docker. You still use the Docker CLI tools, but Colima provides the server to run the containers: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *docker run busybox echo hello* docker: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?. See 'docker run --help'. $ *colima start* INFO[0001] starting colima INFO[0001] runtime: docker INFO[0001] starting ... context=vm INFO[0014] provisioning ... context=docker INFO[0016] starting ... context=docker INFO[0017] done $ *docker run busybox echo hello* hello ---- I used Colima for most of the writing of this book, and it worked fine for me.((("DOCKER_HOST environment variable"))) The only thing I needed to do was set the `DOCKER_HOST` environment variable, and that only came up in <<chapter_12_ansible>>: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *export DOCKER_HOST=unix:///$HOME/.colima/default/docker.sock ---- NOTE: On macOS, you can use Colima as a backend for nerdctl. Podman ships with its own runtime, for both Mac and Windows (there is no need for a runtime on Linux). At the time of writing, Apple had just announced its own container runner, https://github.com/apple/container[_container_], but it was in beta and I didn't have time to try it out. ***************************************************************************************** Test your installation by running: [subs="specialcharacters,macros"] ---- $ pass:quotes[*docker run busybox echo hello world*] Unable to find image 'busybox:latest' locally [...] latest: Pulling from library/busybox [...]: Pull complete Digest: sha256:[...] Status: Downloaded newer image for busybox:latest hello world ---- What's happened there is that Docker has: * Searched for a local copy of the "busybox" image and not found it * Downloaded the image from Docker Hub * Created a container based on that image * Started up that container, telling it to run `echo hello world` * And we can see it worked! Cool! We'll find out more about all of these steps as the chapter progresses. NOTE: On macOS, if you get errors saying `command not found: docker`, obviously the first thing you should do is Google for "macOS command not found Docker", but at least one reader has reported that the solution was Docker Desktop > Settings > Advanced > Change from User to System. === Building a Docker Image and Running a Docker Container Docker has the concepts of _images_ as well as containers. An image is essentially a pre-prepared root filesystem, including the OS, dependencies, and any code you want to run.((("images (container)")))((("Docker", "building image and running a container", id="ix_Dckimg"))) Once you have an image, you can run one or more containers that use the same image. It's a bit like saying, once you've installed your OS and software, you can start up your computer and run that software any number of times, without needing to change anything else. Another way of thinking about it is: images are like classes, and containers are like instances. ==== A First Cut of a Dockerfile Think of a Dockerfile as instructions for setting up a brand new computer that we're going to use to run our Django server on.((("Docker", "building image and running a container", "first draft of Dockerfile")))((("Dockerfiles"))) What do we need to do? Something like this, right? 1. Install an operating system. 2. Make sure it has Python on it. 3. Get our source code onto it. 4. Run `python manage.py runserver`. We create a new file called _Dockerfile_ in the base folder of our repo, next to the _src/_ directory we made earlier: [role="sourcecode"] .Dockerfile (ch09l003) ==== [source,dockerfile] ---- FROM python:3.14-slim # <1> COPY src /src # <2> WORKDIR /src # <3> CMD ["python", "manage.py", "runserver"] # <4> ---- ==== [role="pagebreak-before"] <1> The `FROM` line is usually the first thing in a Dockerfile, and it says which _base image_ we are starting from. Docker images are built from other Docker images! It's not quite turtles all the way down, but almost. So this is the equivalent of choosing a base OS, but images can actually have lots of software preinstalled too. You can browse various base images on Docker Hub. We're using https://hub.docker.com/_/python[one that's published by the Python Software Foundation], called "slim" because it's as small as possible. It's based on a popular version of Linux called Debian, and of course it comes with Python already installed on it. <2> The `COPY` instruction (the uppercase words are called "instructions") lets you copy files from your own computer into the container image. We use it to copy all our source code from the newly created _src_ folder, into a similarly named folder at the root of the container image. <3> `WORKDIR` sets the current working directory for all subsequent commands. It's a bit like doing `cd /src`. <4> Finally, the `CMD` instruction tells Docker which command you want it to run by default, when you start a container based on that image. The syntax is a bit like a Python list (although it's actually parsed as a JSON array, so you _have_ to use double quotes). It's probably worth just showing a directory tree, to make sure everything is in the right place, right? All our source code is in a folder called _src_, next to our `Dockerfile`: [[tree-with-src-and-dockerfile]] [subs="specialcharacters,macros"] ---- . ├── Dockerfile ├── db.sqlite3 ├── src │   ├── functional_tests │   │   ├── [...] │   ├── lists │   │   ├── [...] │   ├── manage.py │   └── superlists │   ├── [...] └── static └── [...] ---- // TODO: figure out what to do with the /static folder [role="pagebreak-before less_space"] ==== Docker Build You build an image with `docker build <path-containing-dockerfile>` and we'll use the `-t <tagname>` argument to "tag" our image with a memorable name.((("Docker", "building image and running a container", "docker build command"))) It's typical to invoke `docker build` from the folder that contains your Dockerfile, so the last argument is usually `.`: [subs="specialcharacters,macros"] ---- $ pass:quotes[*docker build -t superlists .*] [+] Building 1.2s (8/8) FINISHED docker:default => [internal] load build definition from Dockerfile 0.0s => => transferring dockerfile: 115B 0.0s => [internal] load .dockerignore 0.1s => => transferring context: 2B 0.0s => [internal] load metadata for docker.io/library/python:slim 3.4s => [internal] load build context 0.2s => => transferring context: 68.54kB 0.1s => [1/3] FROM docker.io/library/python:3.14-slim@sha256:858[...] 4.4s => => resolve docker.io/library/python:3.14-slim@sha256:858[...] 0.0s => => sha256:72ba3400286b233f3cce28e35841ed58c9e775d69cf11f[...] 0.0s => => sha256:3a72e7f66e827fbb943c494df71d2ae024d0b1db543bf6[...] 0.0s => => sha256:a7d9a0ac6293889b2e134861072f9099a06d78ca983d71[...] 0.5s => => sha256:426290db15737ca92fe1ee6ff4f450dd43dfc093e92804[...] 4.0s => => sha256:e8b685ab0b21e0c114aa94b28237721d66087c2bb53932[...] 0.5s => => sha256:85824326bc4ae27a1abb5bc0dd9e08847aa5fe73d8afb5[...] 0.0s => => extracting sha256:a7d9a0ac6293889b2e134861072f9099a06[...] 0.1s => => extracting sha256:426290db15737ca92fe1ee6ff4f450dd43d[...] 0.4s => => extracting sha256:e8b685ab0b21e0c114aa94b28237721d660[...] 0.0s => [internal] load build context 0.0s => => transferring context: 7.56kB 0.0s => [2/3] COPY src /src 0.2 => [3/3] WORKDIR /src 0.1s => exporting to image 0.0s => => exporting layers 0.0s => => writing image sha256:7b8e1c9fa68e7bad7994fa41e2aca852ca79f01a 0.0s => => naming to docker.io/library/superlists 0.0s ---- [role="pagebreak-before"] Now we can see our image in the list of Docker images on the system: // IDEA, this listing was hard to test due to column widths but there must be a way [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *docker images* REPOSITORY TAG IMAGE ID CREATED SIZE superlists latest 522824a399de 2 minutes ago 164MB [...] ---- NOTE: If you see an error about `failed to solve / compute cache key` and `src: not found`, it may be because you saved the Dockerfile in the wrong place. Have another look at the directory tree from earlier. ==== Docker Run Once you've built an image, you can run one or more containers based on that image, using `docker run`. What happens when we run ours?((("Docker", "building image and running a container", "docker run command"))) [role="ignore-errors"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*docker run superlists*] Traceback (most recent call last): File "/src/manage.py", line 11, in main from django.core.management import execute_from_command_line ModuleNotFoundError: No module named 'django' The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/src/manage.py", line 22, in <module> main() ~~~~^^ File "/src/manage.py", line 13, in main raise ImportError( ...<3 lines>... ) from exc ImportError: Couldn't import Django. Are you sure it's installed and available on your PYTHONPATH environment variable? Did you forget to activate a virtual environment? ---- Ah, we forgot that we need to install Django.((("Docker", "building image and running a container", startref="ix_Dckimg"))) [role="pagebreak-before less_space"] === Installing Django in a Virtualenv in Our Container Image Just like on our own machine, a virtualenv((("Django framework", "installing in virtualenv in container image", id="ix_Djainctnrimg")))((("containers", "installing Django in virtualenv in container image", id="ix_cntnrDja")))((("virtualenv (virtual environment)", "installing Django in virtualenv in container image"))) is useful in a deployed environment to make sure we have full control over the packages installed for a particular project.footnote:[ Even a completely fresh Linux install might have odd things installed in its system site packages. A virtualenv is a guaranteed clean slate.] We can create a virtualenv in our Dockerfile just like we did on our own machine with `python -m venv`, and then we can use `pip install` to get Django: [role="sourcecode"] .Dockerfile (ch09l004) ==== [source,dockerfile] ---- FROM python:3.14-slim RUN python -m venv /venv <1> ENV PATH="/venv/bin:$PATH" <2> RUN pip install "django<6" <3> COPY src /src WORKDIR /src CMD ["python", "manage.py", "runserver"] ---- ==== <1> Here's where we create our virtualenv. We use the `RUN` Dockerfile directive, which is how you run arbitrary shell commands as part of building your Docker image. <2> You can't really "activate" a virtualenv inside a Dockerfile, so instead we change the system path so that the venv versions of `pip` and `python` become the default ones (this is actually one of the things that `activate` does, under the hood). <3> We install Django with `pip install`, just like we do locally. [role="pagebreak-before less_space"] ==== Successful Run Let's do the `build` and `run` in a single line. This is a pattern I used quite often when developing a Dockerfile, to be able to quickly rebuild and see the effect of a change: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run -it superlists* [+] Building 0.2s (11/11) FINISHED docker:default [...] => [internal] load .dockerignore 0.1s => => transferring context: 2B 0.0s => [internal] load build definition from Dockerfile 0.0s => => transferring dockerfile: 246B 0.0s => [internal] load metadata for docker.io/library/python:slim 0.0s => CACHED [1/5] FROM docker.io/library/python:slim 0.0s => [internal] load build context 0.0s => => transferring context: 4.75kB 0.0s => [2/5] RUN python -m venv /venv 0.0s => [3/5] pip install "django<6" 0.0s => [4/5] COPY src /src 0.0s => [5/5] WORKDIR /src 0.0s => exporting to image 0.0s => => exporting layers 0.0s => => writing image sha256:[...] 0.0s => => naming to docker.io/library/superlists 0.0s Watching for file changes with StatReloader Performing system checks... System check identified no issues (0 silenced). You have 19 unapplied migration(s). Your project may not [...] [...] Django version 5.2, using settings 'superlists.settings' Starting development server at http://127.0.0.1:8000/ Quit the server with CONTROL-C. ---- OK, scanning through that, it looks like the server is running! WARNING: Make sure you use the `-it` flags to the Docker `run` command when running `runserver`, or any other tool that expects to be run in an interactive terminal session, otherwise you'll get strange behaviour, including not being able to interrupt the Docker process with Ctrl+C. See the following sidebar for an escape hatch. [role="pagebreak-before less_space"] [[how-to-stop-a-docker-container]] .How to Stop a Docker Container ******************************************************************************* If you've got a container that's "hanging" in a terminal window, you can stop it from another terminal. The Docker daemon lets you list all the currently running containers with `docker ps`: [role="skipme small-code"] [subs="quotes"] ---- $ *docker ps* CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 0818e1b8e9bf superlists "/bin/sh -c 'python …" 4 seconds ago Up 4 seconds hardcore_moore ---- This tells us a bit about each container, including a unique ID and a randomly-generated name (you can override that if you want to). We can use the ID or the name to terminate the container with `docker stop`:footnote:[ There is also a `docker kill` if you're in a hurry. But `docker stop` will send a `SIGKILL` if its initial `SIGTERM` doesn't work within a certain timeout (more info in https://docs.docker.com/reference/cli/docker/container/stop[the Docker docs]).] [role="skipme"] [subs="quotes"] ---- $ *docker stop 0818e1b8e9bf* 0818e1b8e9bf ---- And if you go back to your other terminal window, you should find that the Docker process has been terminated.((("containers", "installing Django in virtualenv in container image", startref="ix_cntnrDja")))((("Django framework", "installing in virtualenv in container image", startref="ix_Djainctnrimg"))) ******************************************************************************* === Using the FT to Check That Our Container Works Let's see what our FTs think about ((("containers", "checking that Docker container works")))this Docker version of our site: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 ./src/manage.py test src/functional_tests --failfast*] [...] selenium.common.exceptions.WebDriverException: Message: Reached error page: abo ut:neterror?e=connectionFailure&u=http%3A//localhost%3A8888/[...] ---- What's going on here? Time for a little debugging. [role="pagebreak-before less_space"] === Debugging Container Networking Problems ((("debugging", "of container networking problems", secondary-sortas="container"))) ((("containers", "debugging networking problems for"))) First, let's try and take a look ourselves, in our browser, by going to http://localhost:8888/, as in <<firefox-unable-to-connect-screenshot>>. [[firefox-unable-to-connect-screenshot]] .Cannot connect on that port image::images/tdd3_0903.png["Firefox showing the 'Unable to connect' error"] Now, let's take another look at the output from our `docker run`. Here's what appeared right at the end: [role="skipme"] ---- Starting development server at http://127.0.0.1:8000/ Quit the server with CONTROL-C. ---- Aha! We notice that we're using the wrong port, the default `8000` instead of the `8888` that we specified in the `TEST_SERVER` environment variable (or, "env var"). Let's fix that by amending the `CMD` instruction in the Dockerfile: [role="sourcecode"] .Dockerfile (ch09l005) ==== [source,dockerfile] ---- [...] WORKDIR /src CMD ["python", "manage.py", "runserver", "8888"] ---- ==== Ctrl+C the current Dockerized container process if it's still running in your terminal, then give it another `build && run`: [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run -it superlists* [...] Starting development server at http://127.0.0.1:8888/ ---- [role="pagebreak-before less_space"] ==== Debugging Web Server Connectivity with curl A quick run of the FT or check in our browser will show us that nope, that doesn't work either.((("debugging", "of web server connectivity using curl", secondary-sortas="web")))((("curl utility"))) Let's try an even lower-level smoke test, the traditional Unix utility `curl`. It's a command-line tool for making HTTP requests.footnote:[ `curl` can do FTP (File Transfer Protocol) and many other types of network requests too! Check out the https://man7.org/linux/man-pages/man1/curl.1.html[`curl` manual].] Try it on your own computer first: [role="ignore-errors"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*curl -iv localhost:8888*] * Trying 127.0.0.1:8888... * connect to 127.0.0.1 port 8888 [...] * Trying [::1]:8888... * connect to ::1 port 8888 [...] * Failed to connect to localhost port 8888 after 0 ms: [...] * Closing connection [...] curl: (7) Failed to connect to localhost port 8888 after 0 ms: [...] ---- TIP: The `-iv` flag to `curl` is useful for debugging. It prints verbose output, as well as full HTTP headers. === Running Code "Inside" the Container with docker exec So, we can't see Django running on port `8888` when we're _outside_ the container. What do we see if we run things from _inside_ the container?((("containers", "running code inside with docker exec", id="ix_cntnrrun")))((("Docker", "running code inside container with docker exec", id="ix_Dckexec"))) We can use `docker exec` to run commands inside a running container. First, we need to get the name or ID of the container: // TODO use --name arg to docker run?? [role="skipme small-code"] [subs="specialcharacters,quotes"] ---- $ *docker ps* CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 5ed84681fdf8 superlists "/bin/sh -c 'python …" 12 minutes ago Up 12 minutes trusting_wu ---- Your values for `CONTAINER_ID` and `NAMES` will be different from mine, because they're randomly generated. But make a note of one or the other, and then run `docker exec -it <container-id> bash`. On most platforms, you can use tab completion for the container ID or name. Let's try it now. Notice that the shell prompt will change from your default Bash prompt to `root@container-id`. Watch out for those in future listings, so that you can be sure of what's being run inside versus outside containers. [role="skipme"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*docker exec -it container-id-or-name bash*] root@5ed84681fdf8:/src# pass:specialcharacters,quotes[*apt-get update && apt-get install -y curl*] Get:1 pass:[http://deb.debian.org/debian] bookworm InRelease [151 kB] Get:2 pass:[http://deb.debian.org/debian] bookworm-updates InRelease [52.1 kB] [...] Reading package lists... Done Building dependency tree... Done Reading state information... Done The following additional packages will be installed: libbrotli1 libcurl4 libldap-2.5-0 libldap-common libnghttp2-14 libpsl5 [...] root@5ed84681fdf8:/src# pass:quotes[*curl -iv http://localhost:8888*] * Trying [...] * Connected to localhost [...] > GET / HTTP/1.1 > Host: localhost:8888 > User-Agent: curl/8.6.0 > Accept: */* > < HTTP/1.1 200 OK HTTP/1.1 200 OK [...] <!doctype html> <html lang="en"> <head> <title>To-Do lists [...] ---- TIP: Use Ctrl+D to exit from the `docker exec` bash shell inside the container. That's definitely some HTML! And the `To-Do lists` looks like it's our HTML, too. So, we can see Django is serving our site _inside_ the container. Why can't we see it _outside_? ==== Docker Port Mapping The (highly, highly recommend) PythonSpeed guide to Docker's very first section is called https://oreil.ly/e3gYQ[Connection refused?], so I'll refer you there once again for an _excellent_, detailed explanation.((("ports", "Docker port mapping"))) But in short: Docker runs in its own little world; specifically, it has its own little network, so the ports _inside_ the container are different from the ports _outside_ the container, the ones we can see on our host machine. So, we need to tell Docker to connect the internal ports to the outside ones—to "publish" or "map" them, in Docker terminology. `docker run` takes a `-p` argument, with the syntax `OUTSIDE:INSIDE`. So, you can actually map a different port number on the inside and outside. But we're just mapping `8888` to `8888`, and that will look like this: [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run -p 8888:8888 -it superlists* ---- Now that will _change_ the error we see, but only quite subtly (see <>).footnote:[ Tip: If you use Chrome as your web browser, its error is something like "localhost didn’t send any data. ERR_EMPTY_RESPONSE".] Things clearly aren't working yet. [[firefox-connection-reset]] .Cannot connect on that port image::images/tdd3_0904.png["Firefox showing the 'Connection reset' error"] // FT would show this // selenium.common.exceptions.WebDriverException: Message: Reached error page: about:neterror?e=netReset&u=http%3A//localhost%3A8888/&c=UTF-8&d=The%20connection%20to%20the%20server%20was%20reset%20while%20the%20page%20was%20loading. [role="pagebreak-before"] Similarly, if you try our `curl -iv` (outside the container) once again, you'll see the error has changed from "Failed to connect", to "Empty reply": // CI consistently says "connection reset by peer", // locally it's empty reply, no matter what curl version [role="ignore-errors skipme"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*curl -iv localhost:8888*] * Trying [...] * Connected to localhost (127.0.0.1) port 8888 > GET / HTTP/1.1 > Host: localhost:8888 > User-Agent: curl/8.6.0 > Accept: */* [...] * Empty reply from server * Closing connection curl: (52) Empty reply from server ---- NOTE: Depending on your system, instead of `(52) Empty reply from server`, you might see `(56) Recv failure: Connection reset by peer`. They mean the same thing: we can connect but we don't get a response. ==== Essential Googling the Error Message The need to map ports and the `-p` argument to `docker run` are something you just pick up, fairly early on in learning Docker.((("error messages", "Django runserver inside Docker, access problem")))(((""Googling the error message" technique", primary-sortas="Googling"))) But the next debugging step is quite a bit more obscure—although admittedly Itamar does address it in his https://oreil.ly/VAQhF[Docker networking article] (did I already mention how excellent it is?). But if we haven't read that, we can always resort to the tried and tested "Googling the error message" technique instead (<>). [[googling-the-error]] .An indispensable publication (source: https://oreil.ly/2WptY[Hacker News]) image::images/tdd3_0905.png["Cover of a fake O'Reilly book called Essential Googling the Error Message",400] [role="pagebreak-before"] Everyone's search results are a little different, and mine are perhaps shaped by years of working with Docker and Django, but I found the answer in my very first result (see <>), when I searched for "cannot access Django runserver inside Docker". The result was was a https://oreil.ly/E_4ed[Stack Overflow post], saying something about needing to specify `0.0.0.0` as the IP address.footnote:[ Kids these days will probably ask an AI right? I have to say, I tried it out, with the prompt being "I'm trying to run Django inside a Docker container, and I've mapped port 8888, but I still can't connect. Can you suggest what the problem might be?", and it come up with a pretty good answer.] [[google-results-screenshot]] .Google can still deliver results image::images/tdd3_0906.png["Google results with a useful stackoverflow post in first position",1000] We're nearing the edges of my understanding of Docker now, but as I understand it, `runserver` binds to `127.0.0.1` by default. However, that IP address doesn't correspond to a network adapter _inside_ the container, which is actually connected to the outside world via the port mapping we defined earlier. [role="pagebreak-before"] The long and short of it is that we need use the long-form `ipaddr:port` version of the `runserver` command, using the magic "wildcard" IP address, `0.0.0.0`: [role="sourcecode"] .Dockerfile (ch09l007) ==== [source,dockerfile] ---- [...] WORKDIR /src CMD ["python", "manage.py", "runserver", "0.0.0.0:8888"] ---- ==== Rebuild and rerun your server, and if you have eagle eyes, you'll spot it's binding to `0.0.0.0` instead of `127.0.0.1`: [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run -p 8888:8888 -it superlists* [...] Starting development server at http://0.0.0.0:8888/ ---- We can verify it's working with `curl`: [subs="specialcharacters,macros"] ---- $ pass:quotes[*curl -iv localhost:8888*] * Trying [...] * Connected to localhost [...] [...] ---- Looking good!((("Docker", "running code inside container with docker exec", startref="ix_Dckexec")))((("containers", "running code inside with docker exec", startref="ix_cntnrrun"))) .On Debugging ******************************************************************************* Let me let you in on a little secret: I'm actually not that good at debugging. We all have our psychological strengths and weaknesses, and one of my weaknesses is that when I run into a problem that I can't see an obvious solution to, I want to throw up my hands way too soon and say "well, this is hopeless; it can't be fixed", and give up.((("debugging", "patience and tenacity in"))) Thankfully I have had some good role models over the years who are much better at it than me (hi, Glenn!). Debugging needs the patience and tenacity of a bloodhound. If at first you don't succeed, you need to systematically rule out options, check your assumptions, eliminate various aspects of the problem, simplify things down, and find the parts that do and don't work, until you eventually find the cause. It might seems hopeless at first! But you usually get there eventually. ******************************************************************************* [role="pagebreak-before less_space"] === Database Migrations ((("database migrations", "into Docker container", secondary-sortas="Docker", id="ix_DBmigDck")))((("Docker", "testing database migrations in", id="ix_Dcktstdb"))) A quick visual inspection confirms--the site is up (<>)! [[site-in-docker-is-up]] .The site in Docker is up! image::images/tdd3_0907.png["The front page of the site, at least, is up"] Let's see what our functional tests say: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 ./src/manage.py test src/functional_tests --failfast*] [...] E ====================================================================== ERROR: test_can_start_a_todo_list (functional_tests.tests.NewVisitorTest.test_can_start_a_todo_list) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/functional_tests/tests.py", line 56, in test_can_start_a_todo_list self.wait_for_row_in_list_table("1: Buy peacock feathers") ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "...goat-book/src/functional_tests/tests.py", line 26, in wait_for_row_in_list_table table = self.browser.find_element(By.ID, "id_list_table") [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_list_table"]; For documentation [...] ---- Although the FTs can connect happily and interact with our site, they are failing as soon as they try to submit a new item. You might have spotted the yellow Django debug page (<>) telling us why. It's because we haven't set up the database (which, as you may remember, we highlighted as one of the "danger areas" of deployment). [[django-debug-screen]] .But the database isn't image::images/tdd3_0908.png["Django DEBUG page showing database error"] NOTE: The tests saved us from potential embarrassment there. The site _looked_ fine when we loaded its front page. If we'd been a little hasty and only tested manually, we might have thought we were done, and it would have been the first users that discovered that nasty Django debug page. Okay, slight exaggeration for effect—maybe we _would_ have checked, but what happens as the site gets bigger and more complex? You can't check everything. The tests can. To be fair, if you look back through the `runserver` command output each time we've been starting our container, you'll see it's been warning us about this issue: [role="skipme"] ---- You have 19 unapplied migration(s). Your project may not work properly until you apply the migrations for app(s): auth, contenttypes, lists, sessions. Run 'python manage.py migrate' to apply them. ---- NOTE: If you don't see this error, it's because your _src_ folder had the database file in it, unlike mine. For the sake of argument, run `rm src/db.sqlite3` and rerun the build and run commands, and you should be able to reproduce the error. I promise it's instructive! ==== Should We Run migrate Inside the Dockerfile? No. So, should we include `manage.py migrate` in our Dockerfile? If you try it, you'll find it certainly _seems_ to fix the((("Dockerfiles", "database migrations and"))) problem: [role="sourcecode"] .Dockerfile (ch09l008) ==== [source,dockerfile] ---- [...] WORKDIR /src RUN python manage.py migrate --noinput <1> CMD ["python", "manage.py", "runserver", "0.0.0.0:8888"] ---- ==== <1> We run `migrate` using the `--noinput` argument to suppress any little "are you sure" prompts. If we rebuild the image... [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run -p 8888:8888 -it superlists* [...] Starting development server at http://0.0.0.0:8888/ ---- ...and try our FTs again, they all pass! [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 ./src/manage.py test src/functional_tests --failfast*] [...] ... --------------------------------------------------------------------- Ran 3 tests in 26.965s OK ---- The problem is that this saves our database file into our system image, which is not what we want, because the system image is meant to be something fixed and stateless (whereas the database is living, stateful data that should change over time). [role="pagebreak-before less_space"] .What Would Happen if We Kept the Database File in the Image ******************************************************************************* You can try this as a little experiment. Assuming you've got the `manage.py migrate` line in your Dockerfile: 1. Create a new to-do list and keep a note of its URL (e.g., at _http://localhost:8888/lists/1_). 2. Now, `docker stop` your container, and rebuild a new one with the same `build && run` command we used earlier. 3. Go back and try to retrieve your old list. It's gone! This is because rebuilding the image will give us a brand new database each time. What we actually want is for our database storage to be "outside" the container somehow, so it can persist between different versions of our Docker image.((("database migrations", "into Docker container", secondary-sortas="Docker", startref="ix_DBmigDck")))((("Docker", "testing database migrations in", startref="ix_Dcktstdb"))) ******************************************************************************* === Mounting Files Inside the Container We want the database on the server to be totally separate data from the data in the system image. ((("Docker", "mounting files in")))((("containers", "mounting files in Docker")))In most deployments, you'd probably be talking to a separate database server, like PostgreSQL. For the purposes of this book, the easiest analogy for a database that's "outside" our container is to access the database from the filesystem outside the container. That also gives us a convenient excuse to talk about mounting files in Docker, which is a very Useful Thing to be Able to Do (TM). First, let's revert our change: [role="sourcecode"] .Dockerfile (ch09l009) ==== [source,dockerfile] ---- [...] COPY src /src WORKDIR /src CMD ["python", "manage.py", "runserver", "0.0.0.0:8888"] ---- ==== Then, let's make sure we _do_ have the database on our local filesystem, by running `migrate` (when we moved everything into _./src_, we left the database file behind): [subs="specialcharacters,quotes"] ---- $ *./src/manage.py migrate --noinput* Operations to perform: Apply all migrations: auth, contenttypes, lists, sessions Running migrations: Applying contenttypes.0001_initial... OK [...] Applying sessions.0001_initial... OK ---- Let's make sure to _.gitignore_ the new location of the database file, and we'll also use a file called https://docs.docker.com/reference/dockerfile/#dockerignore-file[_.dockerignore_] to make sure we can't copy our local dev database into our Docker image during Docker builds: [subs="specialcharacters,quotes"] ---- $ *echo src/db.sqlite3 >> .gitignore* $ *echo src/db.sqlite3 >> .dockerignore* ---- //ch09l010, ch09l011 Now we rebuild, and try mounting our database file. The extra flag to add to the Docker run command is `--mount`, where we specify `type=bind`, the `source` path on our machine,footnote:[ If you're wondering about the `$PWD` in the listing, it's a special environment variable that represents the current directory. The initials echo the `pwd` command, which stands for "print working directory". Docker requires mount paths to be absolute paths.] and the `target` path _inside_ the container: [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 \ -it superlists* ---- TIP: You're likely to come across the old syntax for mounts, which was `-v`. One of the advantages of the new `--mount` version is that it will fail hard if the path you're trying to mount does not exist—it says something like `bind source path does not exist`. This avoids a lot of pain (ask me how I know this). [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 ./src/manage.py test src/functional_tests --failfast*] [...] ... --------------------------------------------------------------------- Ran 3 tests in 26.965s OK ---- AMAZING, IT ACTUALLY WORKSSSSSSSS. Ahem, that's definitely good enough for now! Let's commit: [subs="specialcharacters,quotes"] ---- $ *git add -A .* # add Dockerfile, .dockerignore, .gitignore $ *git commit -am"First cut of a Dockerfile"* ---- Phew. Well, it took a bit of hacking about, but now we can be reassured that the basic Docker plumbing works. Notice that the FT was able to guide us incrementally towards a working config, and spot problems early on (like the missing database). But we really can't be using the Django dev server in production, or running on port `8888` forever. In the next chapter, we'll make our hacky image more production-ready. But first, time for a well-earned tea break I think, and perhaps a https://oreil.ly/GtL7w[chocolate biscuit]. .Docker Recap ******************************************************************************* Docker lets us reproduce a server environment on our own machine:: For developers, ops and infra work is always "fun", by which I mean a process full of fear, uncertainty, and surprises—and painfully slow too. Docker helps to minimise this pain by giving us a mini server on our own machine, which we can try things out with and get feedback quickly, as well as enable us to work in small steps. `docker build && docker run`:: We've learned the core tools for working with Docker. The Dockerfile specifies our image, `docker build` builds it, and `docker run` runs it. `build && run` together give us a "start again from scratch" cycle, which we use every time we make a code change in _src_, or a change in the Dockerfile.footnote:[ There's a common pattern of mounting the whole _src_ folder into your Docker containers in local dev. It means you don't need to rebuild for every source code change. I didn't wan't to introduce that here because it also leads to subtle behaviours that can be hard to wrap your head around, like the _db.sqlite3_ file being shared with the container. For this book, the `build && run` cycle is fast enough, but by all means try out mounting _src_ in your own projects.] Debugging network issues:: We've seen how to use `curl` both outside and inside the container with `docker exec`. We've also seen the `-p` argument to bind ports inside and outside, and the idea of needing to bind to `0.0.0.0`. Mounting files:: We've also had a brief intro to mounting files from outside the container, into the inside. It's an insight into the difference between the "stateless" system image, and the stateful world outside of Docker. ******************************************************************************* ================================================ FILE: chapter_10_production_readiness.asciidoc ================================================ [[chapter_10_production_readiness]] == Making Our App Production-Ready Our container is working fine but it's not production-ready. Let's try to get it there, using the tests to keep us safe.((("containers", "making production-ready", id="ix_cntnrprd"))) In a way we're applying the red/green/refactor cycle to our productionisation process. Our hacky container config got us to green, and now we're going to refactor, working incrementally (just as we would while coding), trying to move from working state to working state, and using the FTs to detect any regressions. === What We Need to Do What's wrong with our hacky container image? A few things: first, we need to host our app on the "normal" port `80` so that people can access it using a regular URL. Perhaps more importantly, we shouldn't use the Django dev server for production; it's not designed for real-life workloads. Instead, we'll use the popular Gunicorn Python WSGI HTTP server. NOTE: Django's `runserver` is built and optimised for local development and debugging. It's designed to handle one user at((("Django framework", "runserver, limitations of"))) a time; it handles automatic reloading upon saving of the source code, but it isn't optimised for performance, nor has it been hardened against security vulnerabilities. ((("DEBUG settings"))) In addition, several options in _settings.py_ are currently unacceptable. `DEBUG=True` is strongly discouraged for production, we'll want to set a unique `SECRET_KEY` and, as we'll see, other things will come up. WARNING: `DEBUG=True` is considered a security risk, because the Django debug page will display sensitive information like the values of variables, and most of the settings in _settings.py_. Let's go through and see if we can fix things one by one. === Switching to Gunicorn ((("production-ready deployment", "using Gunicorn", secondary-sortas="Gunicorn"))) ((("Gunicorn", "switching to"))) Do you know why the Django mascot is a pony? The story is that Django comes with so many things you want: an ORM, all sorts of middleware, the admin site...“What else do you want, a pony?” Well, Gunicorn stands for "Green Unicorn", which I guess is what you'd want next if you already had a pony... We'll need to first install Gunicorn into our container, and then use it instead of `runserver`: [subs="specialcharacters,quotes"] ---- $ *python -m pip install gunicorn* Collecting gunicorn [...] Successfully installed gunicorn-2[...] ---- Gunicorn will need to know a path to a "WSGI server"footnote:[ WSGI stands for Web Server Gateway Interface and it's the protocol for communication((("Web Server Gateway Interface (WSGI)")))((("WSGI (Web Server Gateway Interface)"))) between a web server and a Python web application. Gunicorn is a web server that uses WSGI to interact with Django, and so is the web server you get from `runserver`.] which is usually a function called `application`. Django provides one in 'superlists/wsgi.py'. Let's change the command that our image runs: [role="sourcecode"] .Dockerfile (ch10l001) ==== [source,dockerfile] ---- [...] RUN pip install "django<6" gunicorn # <1> COPY src /src WORKDIR /src CMD ["gunicorn", "--bind", ":8888", "superlists.wsgi:application"] # <2> ---- ==== <1> Installation is a standard `pip install`. <2> Gunicorn has its own command line, `gunicorn`. Here's where we invoke it, including telling it which port to use, and supplying the dot-notation path to the WSGI server provided by Django. As in the previous chapter, we can use the `docker build && docker run` pattern to try out our changes by rebuilding and rerunning our container: [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 \ -it superlists* ---- TIP: If you see an error saying `Bind for 0.0.0.0:8888 failed: port is already allocated.`, it'll be because you still have a container running from the previous chapter. Do you remember how to use `docker ps` and `docker stop`? If not, have another look at <>. ==== The FTs Catch a Problem with Static Files As we run the FTs, you'll see them warning us of a problem, once again. The test for adding list items passes happily, but the test for layout and styling fails.((("static files", "Gunicorn's problem with")))((("Gunicorn", "static files, problem with")))((("CSS (Cascading Style Sheets)", "challenges of static files"))) Good job, tests! [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests --failfast*] [...] AssertionError: 102.5 != 512 within 10 delta (409.5 difference) FAILED (failures=1) ---- And indeed, if you take a look at the site, you'll find the CSS is all broken, as in <>. The reason that we have no CSS is that although the Django dev server will serve static files magically for you, Gunicorn doesn't. [[site-with-broken-css]] .Broken CSS image::images/tdd3_1001.png["The site is up, but CSS is broken"] One step forwards, one step backwards, but once again we've identified the problem nice and early. Moving on! === Serving Static Files with WhiteNoise Serving static files is very different from serving dynamically rendered content from Python and Django.((("static files", "serving with WhiteNoise")))((("WhiteNoise library, serving static files with"))) There are many ways to serve them in production: you can use a web server like nginx, or a content delivery network (CDN) like Amazon S3. But in our case, the most straightforward thing to do is to use https://whitenoise.readthedocs.io[WhiteNoise], a Python library expressly designed for serving staticfootnote:[ Believe it or not, this pun didn't actually hit me until I was rewriting this chapter. For 10 years, it was right under my nose. I think that makes it funnier actually.] files from Python. // DAVID: It might be worth pointing out what Whitenoise is actually doing. // From what I understand, we're still using Django to serve static files. First, we install WhiteNoise into our local environment: [subs="specialcharacters,quotes"] ---- *pip install whitenoise* ---- Then we tell Django to enable it, in __settings.py__footnote:[ Find out more about Django middleware in https://docs.djangoproject.com/en/5.2/topics/http/middleware[the docs]. ]: [role="sourcecode"] .src/superlists/settings.py (ch10l002) ==== [source,python] ---- MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "whitenoise.middleware.WhiteNoiseMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", [...] ---- ==== And then ((("Django framework", "middleware")))we need to add it to our ++pip install++s in the Dockerfile: [role="sourcecode"] .Dockerfile (ch10l003) ==== [source,dockerfile] ---- RUN pip install "django<6" gunicorn whitenoise ---- ==== This manual list of ++pip install++s is getting a little fiddly! We'll come back to that in a moment. First let's rebuild and try rerunning our FTs: [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 \ -it superlists* ---- And if you take another manual look at your site, things should look much healthier. [role="pagebreak-before"] Let's rerun our FTs to confirm: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests --failfast*] [...] ... --------------------------------------------------------------------- Ran 3 tests in 10.718s OK ---- Phew. Let's commit that: [subs="specialcharacters,quotes"] ---- $ *git commit -am"Switch to Gunicorn and Whitenoise"* ---- === Using requirements.txt Let's deal with that fiddly list of ++pip install++s.((("dependency management tools")))((("requirements.txt", id="ix_reqr"))) To reproduce our local virtualenv, rather than just manually ++pip install++ing things one by one and having to remember to sync things between local dev and Docker, we can "save" the list of packages we're using by creating a _requirements.txt_ file.footnote:[ There are many other dependency management tools these days so _requirements.txt_ is not the only way to do it, although it is one of the oldest and best established. As you continue your Python adventures, I'm sure you'll come across many others.] The `pip freeze` command will show us everything that's installed((("virtualenv (virtual environment)", "pip freeze command showing all contents"))) in our virtualenv at the moment: // version numbers change too much [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *pip freeze* asgiref==3.8.1 attrs==25.3.0 certifi==2025.4.26 Django==5.2.3 gunicorn==23.0.0 h11==0.16.0 idna==3.10 outcome==1.3.0.post0 packaging==25.0 PySocks==1.7.1 selenium==4.31.0 sniffio==1.3.1 sortedcontainers==2.4.0 sqlparse==0.5.3 trio==0.30.0 trio-websocket==0.12.2 typing_extensions==4.13.2 urllib3==2.4.0 websocket-client==1.8.0 whitenoise==6.11.0 wsproto==1.2.0 ---- That shows _all_ the packages in our virtualenv, along with their version numbers. Let's pull out just the "top-level" dependencies—Django, Gunicorn, and WhiteNoise: [subs="specialcharacters,quotes"] ---- $ *pip freeze | grep -i django* Django==5.2[...] $ *pip freeze | grep -i django >> requirements.txt* $ *pip freeze | grep -i gunicorn >> requirements.txt* $ *pip freeze | grep -i whitenoise >> requirements.txt* ---- That should give us a _requirements.txt_ file that looks like this: [role="sourcecode skipme"] .requirements.txt (ch10l004) ==== [source,python] ---- django==5.2.3 gunicorn==23.0.0 whitenoise==6.11.0 ---- ==== Let's try it out! To install things from a _requirements.txt_ file, you use the `-r` flag, like this: [subs="specialcharacters,quotes"] ---- $ *pip install -r requirements.txt* Requirement already satisfied: Django==5.2.[...] ./.venv/lib/python3.14/site-packages (from -r requirements.txt (line 1)) (5.2.[...] Requirement already satisfied: gunicorn==23.0.0 in ./.venv/lib/python3.14/site-packages (from -r requirements.txt (line 2)) (23.0.0) Requirement already satisfied: whitenoise==6.11.0 in ./.venv/lib/python3.14/site-packages (from -r requirements.txt (line 3)) (6.11.0) Requirement already satisfied: asgiref[...] Requirement already satisfied: sqlparse[...] [...] ---- As you can see, it's a no-op because we already have everything installed. That's expected! TIP: Forgetting the `-r` and running `pip install requirements.txt` is such a common error, that I recommend you do it _right now_ and get familiar with the error message (which is thankfully much more helpful than it used to be). It's a mistake I still make, _all the time_. Anyway, that's a good first version of a requirements file. Let's commit it: [subs="specialcharacters,quotes"] ---- $ *git add requirements.txt* $ *git commit -m "Add a requirements.txt with Django, gunicorn and whitenoise"* ---- .Dev Dependencies, Transitive Dependencies, and Lockfiles ******************************************************************************* You may be wondering why we didn't add our other key dependency, Selenium, to our requirements.((("dependencies", "dev and transitive"))) Or you might be wondering why we didn't just add _all_ the dependencies, including the "transitive" ones (e.g., Django has its own dependencies like `asgiref` and `sqlparse`, etc.). As always, I have to gloss over some nuance and trade-offs, but the short answer is: Selenium is only a dependency for the tests, not the application code; we're never going to run the tests directly on our production servers.footnote:[ Some people like to separate out test or "dev" dependencies into a separate requirements file called _requirements.dev.txt_, for example. For the record, I think this is a good idea, I just didn't want to add yet another concept to the book.] As for transitive dependencies, they're fiddly to manage without bringing in more tools, and I didn't want to do that for this book. // TODO: revisit this decision When you have a moment, you should probably to do some further reading on "lockfiles", _pyproject.toml_, hard pinning versus soft pinning, and immediate versus transitive dependencies.((("lockfiles")))((("pip-tools, dependency management"))) If I absolutely _had_ to recommend a Python dependency management tool, it would be https://github.com/jazzband/pip-tools[pip-tools], which is a fairly minimal one. ******************************************************************************* Now let's see how we use that requirements file in our Dockerfile: [role="sourcecode"] .Dockerfile (ch10l005) ==== [source,dockerfile] ---- FROM python:3.14-slim RUN python -m venv /venv ENV PATH="/venv/bin:$PATH" COPY requirements.txt /tmp/requirements.txt # <1> RUN pip install -r /tmp/requirements.txt # <2> COPY src /src WORKDIR /src CMD ["gunicorn", "--bind", ":8888", "superlists.wsgi:application"] ---- ==== <1> We copy our requirements file in, just like the _src_ folder. <2> Now instead of just installing Django, we install all our dependencies using `pip install -r`. Let's build and run: [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 \ -it superlists* ---- And then test to check everything still works: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests --failfast*] [...] OK ---- Hooray. That's a commit!((("requirements.txt", startref="ix_reqr"))) [subs="specialcharacters,quotes"] ---- $ *git commit -am "Use requirements.txt in Dockerfile"* ---- === Using Environment Variables to Adjust Settings for Production ((("DEBUG settings")))((("environment variables", "using to adjust production settings", id="ix_envvar")))((("configurations", "dev settings, changing for production"))) We know there are several things in _settings.py_ that we want to change for production: * `DEBUG` mode is all very well for hacking about on your own server, but it https://docs.djangoproject.com/en/5.2/ref/settings/#debug[isn't secure]. For example, exposing raw tracebacks to the world is a bad idea. * `SECRET_KEY` is used by Django for some of its crypto--things like cookies and CSRF protection. It's good practice((("SECRET_KEY setting"))) to make sure the secret key in production is different from the one in your source code repo, because that code might be visible to strangers. We'll want to generate a new, random one but then keep it the same for the foreseeable future (find out more in the https://docs.djangoproject.com/en/5.2/topics/signing[Django docs]). Development, staging, and production sites always have some differences in their configuration. Environment variables are a good place to store those different settings.footnote:[The approach of using environment variables for configuration was originally published by https://oreil.ly/ZdVhR[“The 12-Factor App”] manifesto. Another common way of handling this is to have different versions of _settings.py_ for dev and prod. That can work fine too, but it can get confusing to manage. Environment variables also have the advantage of working for non-Django stuff too.] [role="pagebreak-before less_space"] ==== Setting DEBUG=True and SECRET_KEY There are lots of ways you might set these settings.((("DEBUG settings", "setting DEBUG=True"))) What I propose may seem a little fiddly, but I'll provide a little justification for each choice. Let them be an inspiration (but not a template) for your own choices! Note that this `if` statement replaces the `DEBUG` and `SECRET_KEY` lines that are included by default in the _settings.py_ file: [role="sourcecode"] .src/superlists/settings.py (ch10l006) ==== [source,python] ---- import os [...] # SECURITY WARNING: don't run with debug turned on in production! if "DJANGO_DEBUG_FALSE" in os.environ: #<1> DEBUG = False SECRET_KEY = os.environ["DJANGO_SECRET_KEY"] #<2> else: DEBUG = True #<3> SECRET_KEY = "insecure-key-for-dev" ---- ==== // CSANAD: I think variable names like "something_false" are confusing, since // we need to set something to true so that they mean false. // How about `DJANGO_ENV_PRODUCTION` or something similar? <1> We say we'll use an environment variable called `DJANGO_DEBUG_FALSE` to switch debug mode off and, in effect, require production settings (it doesn't matter what we set it to, just that it's there). <2> And now we say that, if debug mode is off, we _require_ the `SECRET_KEY` to be set by a second environment variable. <3> Otherwise we fall back to the insecure, debug mode settings that are useful for dev. The end result is that you don't need to set any env vars for dev, but production needs both to be set explicitly, and it will error if any are missing. I think this gives us a little bit of protection against accidentally forgetting to set one. TIP: Better to fail hard than allow a typo in an environment variable name to leave you running with insecure settings. // CSANAD: I think it would worth pointing out the development environment // does not use Docker, launching the dev server should be done from // the reader's host system. I think this isn't immediately obvious, e.g. I // thought all along that from now on we would only run the server from Docker. // If we end up making a TIP or similar about it, I think we should also mention // in a development environment relying on containerization, programmers usually // mount the whole /src minimizing the time-consuming rebuilding of their images. [role="pagebreak-before less_space"] ==== Setting Environment Variables Inside the Dockerfile Now let's set ((("Dockerfiles", "setting environment variables in")))((("ENV directive (Dockerfiles)")))that environment variable in our Dockerfile using the `ENV` directive: [role="sourcecode"] .Dockerfile (ch10l007) ==== [source,dockerfile] ---- WORKDIR /src ENV DJANGO_DEBUG_FALSE=1 CMD ["gunicorn", "--bind", ":8888", "superlists.wsgi:application"] ---- ==== And try it out... [role="ignore-errors"] [subs="specialcharacters,macros"] ---- $ pass:specialcharacters,quotes[*docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 \ -it superlists*] [...] File "/src/superlists/settings.py", line 23, in SECRET_KEY = os.environ["DJANGO_SECRET_KEY"] ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ [...] KeyError: 'DJANGO_SECRET_KEY' ---- Oops. I forgot to set said secret key env var, mere seconds after having dreamt it up! ==== Setting Environment Variables at the Docker Command Line We've said we can't keep the secret key in our source code, so the Dockerfile isn't an option; where else can we put it?((("Docker", "setting environment variables at command line"))) For now, we can set it at the command line using the `-e` flag for `docker run`: [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 \ -e DJANGO_SECRET_KEY=sekrit \ -it superlists* ---- With that running, we can use our FT again to see if we're back to a working state. [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests --failfast*] [...] AssertionError: 'To-Do' not found in 'Bad Request (400)' ---- [role="pagebreak-before"] NOTE: The eagle-eyed might spot a message saying `UserWarning: No directory at: /src/static/`. That's a little clue about a problem with static files, which we're going to deal with shortly. Let's deal with this 400 issue first. ==== ALLOWED_HOSTS Is Required When Debug Mode Is Turned Off It's not quite working yet (see <>)! Let's take a look manually.((("ALLOWED_HOSTS setting"))) [[django-400-error]] .An unfriendly 400 error image::images/tdd3_1002.png["Web page showing wth the text 400 Bad Request in default font"] We've set our two environment variables, but doing so seems to have broken things. However, once again, by running our FTs frequently, we're able to identify the problem early, before we've changed too many things at the same time. We've only changed two settings—which one might be at fault? Let's use the "Googling the error message" technique again, with the search terms "Django debug false" and "400 bad request". Well, the very first link in my https://oreil.ly/gVcLz[search results] was Stack Overflow suggesting that a 400 error is usually to do with `ALLOWED_HOSTS`. And the second was the official Django docs, which takes a bit more scrolling, but confirms it (see <>). [[search-results-400-bad-request]] .Search results for "django debug false 400 bad request" image::images/tdd3_1003.png["Duckduckgo search results with stackoverflow and django docs"] `ALLOWED_HOSTS` is a security setting designed to reject requests that are likely to be forged, broken, or malicious because they don't appear to be asking for your site.footnote:[HTTP requests contain the address they were intended for in a header called "host".] When `DEBUG=True`, `ALLOWED_HOSTS` effectively allows _localhost_ (our own machine) by default, so that's why it was working OK until now. There's more information in the https://docs.djangoproject.com/en/5.2/ref/settings/#allowed-hosts[Django docs]. [role="pagebreak-before"] The upshot is that we need to adjust `ALLOWED_HOSTS` in _settings.py_. Let's use another environment variable for that: [role="sourcecode"] .src/superlists/settings.py (ch10l008) ==== [source,python] ---- if "DJANGO_DEBUG_FALSE" in os.environ: DEBUG = False SECRET_KEY = os.environ["DJANGO_SECRET_KEY"] ALLOWED_HOSTS = [os.environ["DJANGO_ALLOWED_HOST"]] else: DEBUG = True SECRET_KEY = "insecure-key-for-dev" ALLOWED_HOSTS = [] ---- ==== This is a setting that we want to change, depending on whether our Docker image is running locally or on a server, so we'll use the `-e` flag again: [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 \ -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -it superlists* ---- ==== Collectstatic Is Required when Debug Is Turned Off An FT run (or just looking at the site) reveals that we've had ((("DEBUG settings", "collectstatic required when DEBUG turned off")))((("static files", "collectstatic required when debug turned off")))((("collectstatic command")))a regression in our static files: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests --failfast*] [...] AssertionError: 102.5 != 512 within 10 delta (409.5 difference) FAILED (failures=1) ---- And you might have seen this warning message in the `docker run` output: [role="skipme"] [subs="specialcharacters"] ---- /venv/lib/python3.14/site-packages/django/core/handlers/base.py:61: UserWarning: No directory at: /src/static/ mw_instance = middleware(adapted_handler) ---- [role="pagebreak-before"] We saw this at the beginning of the chapter, when switching from the Django dev server to Gunicorn, and that was why we introduced WhiteNoise. Similarly, when we switch `DEBUG` off, WhiteNoise stops automagically finding static files in our code, and instead we need to run `collectstatic`: [role="sourcecode"] .Dockerfile (ch10l009) ==== [source,dockerfile] ---- WORKDIR /src RUN python manage.py collectstatic ENV DJANGO_DEBUG_FALSE=1 CMD ["gunicorn", "--bind", ":8888", "superlists.wsgi:application"] ---- ==== // DAVID: Interestingly when I did this I put the RUN directive after the ENV // directive, which led to a KeyError: 'DJANGO_SECRET_KEY' which foxed me for a bit. // Might be worth calling out that we're running collectstatic in debug mode. Well, it was fiddly, but that should get us to passing tests after we build and run the Docker container! [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 \ -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -it superlists* ---- And... [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests --failfast*] [...] OK ---- We're nearly ready to ship to production! Let's quickly adjust our `gitignore`, as the static folder is in a new place, and do another commit to mark this bit of incremental progress: //0010 [subs="specialcharacters,quotes"] ---- $ *git status* # should show dockerfile and untracked src/static folder $ *echo src/static >> .gitignore* $ *git status* # should now be clean $ *git commit -am "Add collectstatic to dockerfile, and new location to gitignore"* ---- [role="pagebreak-before less_space"] === Switching to a Nonroot User Let's do one more!((("environment variables", "using to adjust production settings", startref="ix_envvar"))) By default, Docker containers run as root. Although container security is a very well-tested ground by now, experts agree it's still good practice to use an unprivileged user inside your container.((("SQLite", "dealing with permissions for db.sqlite3 file", id="ix_SQLperm"))) The main fiddly thing, for us, will be dealing with permissions for the _db.sqlite3_ file. It will need to be: . Writable by the nonroot user . In a _directory_ that's writable by the nonroot userfootnote:[ This is surprising. It's due to https://sqlite.org/tempfiles.html[SQLite wanting to write various additional temporary files during operation].] ==== Making the Database Filepath Configurable First, let's make the path to the database file configurable using an environment variable: [role="sourcecode"] .src/superlists/settings.py (ch10l011) ==== [source,python] ---- # SECURITY WARNING: don't run with debug turned on in production! if "DJANGO_DEBUG_FALSE" in os.environ: DEBUG = False SECRET_KEY = os.environ["DJANGO_SECRET_KEY"] ALLOWED_HOSTS = [os.environ["DJANGO_ALLOWED_HOST"]] db_path = os.environ["DJANGO_DB_PATH"] # <1> else: DEBUG = True SECRET_KEY = "insecure-key-for-dev" ALLOWED_HOSTS = [] db_path = BASE_DIR / "db.sqlite3" # <2> [...] # Database # https://docs.djangoproject.com/en/5.2/ref/settings/#databases DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", "NAME": db_path # <3> } } ---- ==== <1> Inside Docker, we'll assume that an environment variable called `DJANGO_DB_PATH` has been set. We save it to a local variable called `db_path`. <2> Outside Docker, we'll use the default path to the database file. <3> And we modify the `DATABASES` entry to use our `db_path` variable. Now let's change the ((("Dockerfiles", "changing to set DJANGO_DB_PATH and to nonroot user")))Dockerfile to set that env var, and to create and switch to our nonroot user, which we may as well call "nonroot" (although it could be anything!): [role="sourcecode "] .Dockerfile (ch10l012) ==== [source,dockerfile] ---- WORKDIR /src RUN python manage.py collectstatic ENV DJANGO_DEBUG_FALSE=1 RUN adduser --uid 1234 nonroot # <1> USER nonroot # <2> CMD ["gunicorn", "--bind", ":8888", "superlists.wsgi:application"] ---- ==== <1> We use the `adduser` command to create our user, explicitly setting its UID to `1234`.footnote:[ A more or less arbitrary number, the first non-system user on a system is usually 1000, so it's nice that this won't be the same as the `elspeth` user outside the container. But other than that it could be any number greater than 1000 really.] <2> The `USER` directive in the Dockerfile tells Docker to run everything as that user by default. ==== Using UIDs to Set Permissions Across Host/Container Mounts Our user will now have a writable home directory at `/home/nonroot`, so we'll put the database file in there. That takes care of the "writable directory" requirement. Because we're mounting the file from outside though, that's not quite enough to make the file itself writable.((("host/container mounts, using UIDs to set permissions")))((("user IDs (UIDs)", "using to set permissions across host/container mounts"))) We'll need to set the _owner_ of the file to be `nonroot` as well. Because of the way Linux permissions work, we're going to use integer user IDs (UIDs). This might seem a bit magical if you're not used to Linux permissions, so you'll have to trust me, I'm afraid.footnote:[ Linux permissions aren't actually implemented using the string names of users; instead they use integer user IDs (called UIDs). The way we map from the UIDs to strings is using a special file called _/etc/passwd_. Because _/etc/passwd_ is not the same inside and outside the container, the UIDs to username mappings inside and outside are not necessarily the same. However, the permission UIDs are just numbers, and they actually are stored inside individual files, so they don't change when you mount files. There's more info here on https://oreil.ly/ceIfE[this Stack Overflow post].] First, let's create a file with the right permissions, outside the container: [subs="specialcharacters,quotes"] ---- $ *touch container.db.sqlite3* # Change the owner to uid 1234 $ *sudo chown 1234 container.db.sqlite3* # This next step is needed on non-Linux dev environments, # to make sure that the container host VM can write to the file. # Change the file to be group-writeable as well as owner-writeable: $ *sudo chmod g+rw container.db.sqlite3* ---- Now let's rebuild and run our container, changing the `--mount` path to our new file, and setting the `DJANGO_DB_PATH` environment variable to match: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/container.db.sqlite3",target=/home/nonroot/db.sqlite3 \ -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -e DJANGO_DB_PATH=/home/nonroot/db.sqlite3 \ -it superlists* ---- As a first check that we can write to the database from inside the container, let's use `docker exec` to populate the database tables using `manage.py migrate`: [subs="specialcharacters,quotes"] ---- $ *docker ps* # note container id $ *docker exec container-id-or-name python manage.py migrate* Operations to perform: Apply all migrations: auth, contenttypes, lists, sessions Running migrations: Applying contenttypes.0001_initial... OK [...] Applying lists.0001_initial... OK Applying lists.0002_item_text... OK Applying lists.0003_list... OK Applying lists.0004_item_list... OK Applying sessions.0001_initial... OK ---- [role="pagebreak-before"] And, as after every incremental change, we rerun our FT suite to make sure everything works: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests --failfast*] [...] OK ---- Great! We wrap up with a bit of housekeeping; we'll add this new database file to our `.gitignore`, and commit: [subs="specialcharacters,quotes"] ---- $ *echo container.db.sqlite3 >> .gitignore* $ *git commit -am"Switch to nonroot user"* ---- // ch10l014 === Configuring Logging One last thing we'll want to do is make sure that we can get logs out of our server.((("SQLite", "dealing with permissions for db.sqlite3 file", startref="ix_SQLperm")))((("logging", "configuring for production-ready container app", id="ix_logcfg"))) If things go wrong, we want to be able to get to the tracebacks. And as we'll soon see, switching `DEBUG` off means that Django's default logging configuration changes. ==== Provoking a Deliberate Error To test this, we'll provoke a deliberate error by corrupting the database file: [subs="specialcharacters,quotes"] ---- $ *echo 'bla' > container.db.sqlite3* ---- Now if you run the tests, you'll see they fail: // TODO: for some reason this wont repro in CI [role="small-code pause-first skipme"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests --failfast*] [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_list_table"]; [...] ---- // DAVID: Got me thinking, I'm not always clear when I need to rebuild the image. // I would have thought I might need to do it here, but I didn't. Might be worth // explaining in the previous chapter when we do. And you might spot in the browser that we just see a minimal error page, with no debug info, as in <> (try it manually if you like). [[minimal-error-page]] .Minimal default server error 500 image::images/tdd3_1004.png["A minimal error page saying just Server error (500)"] [role="pagebreak-before"] But if you look in your Docker terminal, you'll see there is no traceback: [role="skipme"] ---- [2024-02-28 10:41:53 +0000] [7] [INFO] Starting gunicorn 21.2.0 [2024-02-28 10:41:53 +0000] [7] [INFO] Listening at: http://0.0.0.0:8888 (7) [2024-02-28 10:41:53 +0000] [7] [INFO] Using worker: sync [2024-02-28 10:41:53 +0000] [8] [INFO] Booting worker with pid: 8 ---- Where have the tracebacks gone? You might have been expecting that the Django debug page and its tracebacks would disappear from our web browser, but it's more of shock to see that they are no longer appearing in the terminal either! If you're like me, you might find yourself wondering if we really _did_ see them earlier and starting to doubt your own sanity. But the explanation is that Django's https://docs.djangoproject.com/en/5.2/ref/logging/#default-logging-configuration[default logging configuration] changes when `DEBUG` is turned off. This means we need to interact with the standard library's `logging` module, unfortunately one of the most fiddly parts of the Python standard library.footnote:[ It's not necessarily for bad reasons, but it is all very Java-ey and enterprise-y. I mean, yes, separating the concepts of handlers and loggers and filters, and making it all configurable in a nested hierarchy, is all well and good and covers every possible use case, but sometimes you just wanna say "just print stuff to stdout pls", and you wish that configuring the simplest thing was a little easier.] Here's pretty much the simplest possible logging config, which just prints everything to the console (i.e., standard out); I've added this code to the very end of the _settings.py_ file: [role="sourcecode"] .src/superlists/settings.py (ch10l013) ==== [source,python] ---- LOGGING = { "version": 1, "disable_existing_loggers": False, "handlers": { "console": {"class": "logging.StreamHandler"}, }, "loggers": { "root": {"handlers": ["console"], "level": "INFO"}, }, } ---- ==== Rebuild and restart our container... [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 \ -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -e DJANGO_DB_PATH=/home/nonroot/db.sqlite3 \ -it superlists* ---- Then try the FT again (or submitting a new list item manually) and we now should see a clear error message: // TODO: test get from docker logs [role="skipme"] ---- Internal Server Error: /lists/new Traceback (most recent call last): [...] File "/src/lists/views.py", line 10, in new_list nulist = List.objects.create() ^^^^^^^^^^^^^^^^^^^^^ [...] File "/venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py", line 328, in execute return super().execute(query, params) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ django.db.utils.DatabaseError: file is not a database ---- We can fix and re-create the database by doing: [subs="specialcharacters,quotes"] ---- $ *echo > container.db.sqlite3* $ *docker exec -it python manage.py migrate* ---- And rerun the FTs to check we're back to a working state. Let's do a final commit for this change: [subs="specialcharacters,quotes"] ---- $ *git commit -am "Add logging config to settings.py"* ---- === Exercise for the Reader: Using the Django check Command I don't have time in this book to cover every last aspect of production-readiness.((("logging", "configuring for production-ready container app", startref="ix_logcfg"))) Apart from anything else, this is a fast-changing area, and security updates to Django and its best practice recommandations change frequently, so things I write now might be incomplete by the time you read the book. I _have_ given a decent overview of the various different axes along which you'll need to make production-readiness changes, so hopefully you have a toolkit for how to do this sort of work.((("Django framework", "deployment checklist and check --deploy command"))) If you'd like to dig into this a little bit more, or if you're preparing a real project for release into the wild, the next step is to read up on Django's https://docs.djangoproject.com/en/5.2/howto/deployment/checklist[deployment checklist]. [role="pagebreak-before"] The first suggestion is to use Django's "self-check" command, `manage.py check --deploy`. Here's what it reported as outstanding when I ran it in April 2025: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *docker exec python manage.py check --deploy* System check identified some issues: WARNINGS: ?: (security.W004) You have not set a value for the SECURE_HSTS_SECONDS setting. If your entire site is served only over SSL, you may want to consider setting a value and enabling HTTP Strict Transport Security. Be sure to read the documentation first; enabling HSTS carelessly can cause serious, irreversible problems. ?: (security.W008) Your SECURE_SSL_REDIRECT setting is not set to True. Unless your site should be available over both SSL and non-SSL connections, you may want to either set this setting True or configure a load balancer or reverse-proxy server to redirect all connections to HTTPS. ?: (security.W009) Your SECRET_KEY has less than 50 characters, less than 5 unique characters, or it's prefixed with 'django-insecure-' indicating that it was generated automatically by Django. Please generate a long and random value, otherwise many of Django's security-critical features will be vulnerable to attack. ?: (security.W012) SESSION_COOKIE_SECURE is not set to True. Using a secure-only session cookie makes it more difficult for network traffic sniffers to hijack user sessions. ?: (security.W016) You have 'django.middleware.csrf.CsrfViewMiddleware' in your MIDDLEWARE, but you have not set CSRF_COOKIE_SECURE to True. Using a secure-only CSRF cookie makes it more difficult for network traffic sniffers to steal the CSRF token. ---- Why not pick one of these and have a go at fixing it? === Wrap-Up We might not have addressed every last issue that `check --deploy` raised, but we've at least touched on many or most of the things you might need to think about when considering production-readiness. We've worked in small steps and used our tests all the way along, and we're now ready to deploy our container to a real server! Find out how, in our next exciting installment... TIP: One more recommendation for PythonSpeed and its https://pythonspeed.com/docker[Docker Packaging for Python Developers] article—again, I cannot recommend it highly enough. Read it before you're too much older! [role="pagebreak-before less_space"] .Production-Readiness Config ******************************************************************************* ((("production-ready deployment", "configuration, preparing")))((("configurations", "production-ready, issues to consider"))) A few things to think about when trying to prepare a production-ready configuration: Don't use the Django dev server in production:: Something like Gunicorn or uWSGI is a better tool for running Django; it will let you run multiple workers, for example. ((("Gunicorn", "benefits of"))) Decide how to serve your static files:: Static files aren't the same kind of things as the dynamic content that comes from Django and your web app, so they need to be treated differently. WhiteNoise is just one example of how you might do that. Check your settings.py for dev-only config:: `DEBUG=True`, `ALLOWED_HOSTS`, and `SECRET_KEY` are the ones we came across, but you will probably have others (and we'll see more when we start to send emails from the server). Change things one at a time and rerun your tests frequently:: Whenever we make a change to our server configuration, we can rerun the test suite, and either be confident that everything works as well as it did before, or find out immediately if we did something wrong. Think about logging and observability:: When things go wrong, you need to be able to find out what happened. At a minimum, you need a way of getting logs and tracebacks out of your server, and in more advanced environments you'll want to think about metrics and tracing too. But we can't cover all that in this book!((("containers", "making production-ready", startref="ix_cntnrprd"))) Use the Django "check" command:: `python manage.py check --deploy` can give you a list of additional settings to check for production-readiness. ******************************************************************************* ================================================ FILE: chapter_11_server_prep.asciidoc ================================================ [[chapter_11_server_prep]] == Getting a Server Ready for Deployment ((("infrastructure as code (IaC)"))) This chapter is all about getting ready for our deployment. We're going to spin up an actual server, make it accessible on the internet with a real domain name, and set up the authentication and credentials we need to be able to control it remotely with SSH and Ansible. === Manually Provisioning a Server to Host Our Site ((("staging sites", "manual server provisioning", id="SSserver09"))) ((("server provisioning", id="seerver09"))) We can separate our "deployment" into two tasks: . _Provisioning_ a new server to be able to host the code, which includes choosing an operating system, getting basic credentials to log in, and configuring DNS . _Deploying_ our application to an existing server, which includes getting our Docker image onto the server, starting a container, and configuring it to talk to the database and the outside world Infrastructure-as-code tools can let you automate both of these, but the provisioning parts tend to be quite vendor-specific, so for the purposes of this book, we can live with manual provisioning. NOTE: I should probably stress once more that deployment is something that varies a lot and, as a result, there are few universal best practices for how to do it. So, rather than trying to remember the specifics of what I'm doing here, you should be trying to understand the rationale, so that you can apply the same kind of thinking in the specific future circumstances you encounter. ==== Choosing Where to Host Our Site ((("hosting services"))) There are loads of different solutions out there these days, but they broadly fall into two camps: . Running your own (probably virtual) server—aka VPS (virtual private server) . Using a platform as a service (PaaS) offering like Heroku or my old employers, PythonAnywhere ((("platform-as-a-service (PaaS)")))((("VPS (virtual private server)"))) ((("PythonAnywhere"))) With a PaaS, you don't get your own server; instead, you're renting a "service" at a higher level of abstraction. Particularly for small sites, a PaaS offers many advantages over running your own server, and I would definitely recommend looking into them.((("PaaS", see="platform-as-a-service"))) We're not going to use a PaaS in this book, however, for several reasons. The main reason is that I want to avoid endorsing specific commercial providers. Secondly, all the PaaS offerings are quite different, and the procedures to deploy to each vary a lot--learning about one doesn't necessarily tell you about the others. Any one of them might radically change their process or business model by the time you get to read this book. Instead, we'll learn just a tiny bit of good old-fashioned server admin, including SSH and manual debugging. They're unlikely to ever go away, and knowing a bit about them will get you some respect from all the grizzled dinosaurs out there. ==== Spinning Up Our Own Server I'm not going to dictate how you spin up a server--whether you choose Amazon AWS, Rackspace, DigitalOcean, your own server in a datacentre, or a Raspberry Pi in a cupboard under the stairs, any solution should((("server provisioning", "creating a server", id="ix_serprvcr")))((("Ubuntu, server running Ubuntu 22.04"))) be fine, as long as: * Your server is running Ubuntu 22.04 (aka "Jammy/LTS"). * You have root access to it.((("root user"))) * It's on the public internet (i.e., it has a public IP address). * You can SSH into it (I recommend using a nonroot user account, with `sudo` access, and public/private key authentication). I'm recommending Ubuntu as a distro because it's popular and I'm used to it.footnote:[Linux as an operating system comes in lots of different flavours, called "distros" or "distributions".((("Linux", "different flavors or distributions"))) The differences between them and their relative pros and cons are, like any seemingly minor detail, of tremendous interest to the right kind of nerd. We don't need to care about them for this book. As I say, Ubuntu is fine.] If you know what you're doing, you can probably get away with using something else, but I won't be able to help you as much if you get stuck. [[step-by-step-guide]] .Step-by-Step Instructions for Spinning Up a Server ******************************************************************************* ((("server provisioning", "guide to")))((("Linux", "server, creating"))) I appreciate that, if you've never started a Linux server before and you have absolutely no idea where to start, this is a big ask, especially when I'm refusing to "dictate" exactly how to do it. With that in mind, I wrote a https://github.com/hjwp/Book-TDD-Web-Dev-Python/blob/main/server-quickstart.md[very brief guide on GitHub]. I didn't want to include it in the book itself because, inevitably, I do end up specifying a specific commercial provider in there. ******************************************************************************* NOTE: Some people get to this chapter, and are tempted to skip the domain bit and the "getting a real server" bit, and just use a VM on their own PC. Don't do this. It's _not_ the same, and you'll have more difficulty following the instructions, which are complicated enough as it is. If you're worried about cost, have a look at the guide I wrote for free options. ((("getting help"))) .General Tip for Working with Infrastructure ******************************************************************************* The most important lesson to remember over the next few chapters is, as always but more than ever, to work incrementally, make one change at a time, and run your tests frequently.((("infrastructure, working with"))) When things (inevitably) go wrong, resist the temptation to flail about and make other unrelated changes in the hope that things will start working again; instead, stop, go backwards if necessary to get to a working state, and figure out what went wrong before moving forwards again. It's just as easy to fall into the Refactoring Cat trap when working with infrastructure!((("server provisioning", "creating a server", startref="ix_serprvcr"))) ******************************************************************************* === Getting a Domain Name ((("domains", "getting a domain name"))) We're going to need a couple of domain names at this point in the book--they can both be subdomains of a single domain.((("server provisioning", "getting a domain name"))) I'm going to use _superlists.ottg.co.uk_ and _staging.ottg.co.uk_. If you don't already own a domain, this is the time to register one! Again, this is something I really want you to _actually_ do. If you've never registered a domain before, just pick any old registrar and buy a cheap one--it should only cost you $5 or so, and I promise seeing your site on a "real" website will be a thrill. // DAVID: just wondering if it's worth giving them the option to cheat and // specify a domain name in a hosts file? === Configuring DNS for Staging and Live Domains We don't want to be messing about with IP addresses all the time, so we should point our staging and live domains to the server.((("domains", "configuring DNS for staging and live domains"))) At my registrar, the control screens looked a bit like <>. [[registrar-control-screens]] .Domain setup image::images/tdd3_1101.png["Registrar control screen for adding a DNS record"] ((("A-records")))((("AAAA-records (IPv6)"))) In the DNS system, pointing a domain at a specific IP address is referred to as an "A-record".footnote:[ Strictly speaking, A-records are for IPv4, and you can also use AAAA-records for IPv6. Some cheap providers only support IPv6, and there's nothing wrong with that.] All registrars are slightly different, but a bit of clicking around should get you to the right screen in yours. You'll need two A-records: one for the staging address and one for the live one. No need to worry about any other type of record. DNS records take some time to "propagate" around the world (it's controlled by a setting called "TTL", time to live), so once you've set up your A-record, you can check its progress on a "propagation checking" service like this one: https://www.whatsmydns.net/#A/staging.ottg.co.uk. I'm planning to host my staging server at _staging.ottg.co.uk_. === Ansible Infrastructure-as-code tools, also called "configuration management" tools, come in lots of shapes and sizes.((("Ansible")))((("deployment", "automating with Ansible", id="Dfarbric11")))((("infrastructure as code (IaC)", "tools for")))((("configuration management tools"))) Chef and Puppet were two of the original ones, and you'll probably come across Terraform, which is particularly strong on managing cloud services like AWS. // SEBASTIAN: mentioning of too many technologies (e.g. Puppet/Chef - IMHO not necessary in 2024). We're going to use the infrastructure automation tool Ansible—because it's relatively popular, because it can do everything we need it to, because I'm biased that it happens to be written in Python, and because it's probably the one I'm personally most familiar with. Another tool could probably have worked just as well! The main thing to remember is the _concept_, which is that, as much as possible we want to manage our server configuration _declaratively_, by expressing the desired state of the server in a particular configuration syntax, rather than specifying a procedural series of steps to be followed one by one. ==== Ansible Versus SSH: How We'll Talk to Our Server <> shows how we’ll interact with our server using SSH, Ansible, and our FTs. [[ansible-and-ssh]] .Ansible and SSH image::images/tdd3_1102.png["Diagram "] Our objective is to use Ansible((("Ansible", "using with SSH for server interactions", id="ix_AnsSSH")))((("SSH", "using with Ansible to interact with server"))) to automate the process of deploying to our server: making sure that the server has everything it needs to run our app (mostly, Docker and our container image), and then telling it to start or restart our container. Now and again, we'll want to "log on" to the server and have a look around manually; for that, we'll use the `ssh` command line on our computer, which can let us open up an interactive console on the server. Finally, we'll run our FTs against the server, once it's running our app, to make sure it's all working correctly. === Start by Making Sure We Can SSH In At this point and for the rest of the book, I'm assuming that you have a nonroot user account set up, and that it has `sudo` privileges, so whenever we need to do something that requires root access, we use `sudo`, (or "become" in Ansible terminology); I'll be explicit about that in the various instructions that follow.((("SSH", "making sure you can SSH to the server"))) My user is called "elspeth", but you can call yours whatever you like! Just remember to substitute it in all the places I've hardcoded it. See the guide I wrote (<>) if you need tips on creating a `sudo` user. Ansible uses SSH under the hood to talk to the server, so checking we can log in "manually" is a good first step: [role="server-commands"] [subs="specialcharacters,quotes"] ---- $ *ssh elspeth@staging.ottg.co.uk* elspeth@server$: *echo "hello world"* hello world ---- TIP: Look out for that `elspeth@server` in the command-line listings in this chapter. It indicates commands that must be run on the server, as opposed to commands you run on your own PC. .Use WSL on Windows ******************************************************************************* Ansible will not run natively on Windows (see the https://docs.ansible.com/ansible/latest/os_guide/intro_windows.html#using-windows-as-the-control-node[docs]) but you can use the Windows Subsystem for Linux (WSL), a sort of mini-Linux that Microsoft has made to run inside Windows.((("Ansible", "using WSL on Windows with")))((("Windows Subsystem for Linux (WSL)"))) Follow Microsoft's https://learn.microsoft.com/en-us/windows/wsl/setup/environment[instructions for setting up WSL]. Once inside your WSL environment, you can navigate to your project directory on the host Windows filesystem at _/mnt/c/Users/yourusername/Projects/superlists_, for example. You'll need to use a different virtualenv for WSL: [role="skipme"] [subs="specialcharacters,quotes"] ---- yourusername@wsl: *cd /mnt/c/Users/yourusername/Projects/superlists* yourusername@wsl: *python -m venv .venv-wsl* yourusername@wsl: *source .venv-wsl/bin/activate* ---- If you are using public key authentication, it's probably simplest to generate a new SSH keypair, and add it to __home/elspeth/.ssh/authorized_keys__ on the server: [role="skipme"] [subs="specialcharacters,quotes"] ---- yourusername@wsl: *ssh-keygen* [..] yourusername@wsl: *cat ~/.ssh/*.pub* # copy the public key to your clipboard, ---- I'd suggest you _only_ use WSL when you need to use Ansible. The alternative is to switch your whole dev environment to WSL, and move your source code in there, but you might need to overcome a few hurdles around things like networking. ******************************************************************************* ==== Debugging Issues with SSH Here's a few things to try if you ((("SSH", "debugging issues with", id="ix_SSHdbg")))can't SSH in. ===== Debugging network connectivity First, check network connectivity: can we even reach the server?((("network connectivity, debugging"))) [role="skipme"] [subs="quotes"] ---- $ *ping staging.ottg.co.uk* # if that doesn't work, try the IP address $ *ping 193.184.215.14* # or whatever your IP is # also see if the domain name resolves $ *nslookup staging.ottg.co.uk* ---- If the IP works and the domain name doesn't, and/or if the `nslookup` doesn't work, you should go check your DNS config at your registrar. You may just need to wait!((("nslookup")))((("domains", "checking DNS using propagation checker"))) Try a DNS propagation checker like https://www.whatsmydns.net/#A/staging.ottg.co.uk. [role="pagebreak-before less_space"] ===== Debugging SSH auth issues Next, let's try and debug any possible issues with authentication.((("authentication", "SSH, debugging issues with"))) First, your hosting provider might have the option to open a console directly from within their web UI. That's worth trying, and if there are any problems there, then you probably need to restart your server, or perhaps stop it and create a new one. TIP: It's worth double-checking your IP address at this point, in your provider's server control panel pages. Next, we can try debugging our SSH connection: [role="skipme small-code"] [subs="quotes"] ---- # try the -v flag which turn on verbose/debug output $ *ssh -v elspeth@staging.ottg.uk* OpenSSH_9.7p1, LibreSSL 3.3.6 debug1: Reading configuration data ~/.ssh/config debug1: Reading configuration data ~/.colima/ssh_config debug1: Reading configuration data /etc/ssh/ssh_config debug1: /etc/ssh/ssh_config line 21: include /etc/ssh/ssh_config.d/* matched no files debug1: /etc/ssh/ssh_config line 54: Applying options for * debug1: Authenticator provider $SSH_SK_PROVIDER did not resolve; disabling debug1: Connecting to staging.ottg.uk port 22. ssh: Could not resolve hostname staging.ottg.uk: nodename nor servname provided, or not known # oops I made a typo! it should be ottg.co.uk not ottg.uk ---- If that doesn't help, try switching to((("root user", "switching to in SSH debugging"))) the root user instead: [role="skipme"] [subs="quotes"] ---- $ *ssh -v root@staging.ottg.co.uk* [...] debug1: Authentications that can continue: publickey debug1: Next authentication method: publickey debug1: get_agent_identities: bound agent to hostkey debug1: get_agent_identities: agent returned 1 keys debug1: Will attempt key: ~/.ssh/id_ed25519 ED25519 SHA256:gZLxb9zCuGVT1Dm8 [...] debug1: Will attempt key: ~/.ssh/id_rsa debug1: Will attempt key: ~/.ssh/id_ecdsa debug1: Will attempt key: ~/.ssh/id_ecdsa_sk debug1: Will attempt key: ~/.ssh/id_ed25519_sk debug1: Will attempt key: ~/.ssh/id_xmss debug1: Will attempt key: ~/.ssh/id_dsa debug1: Offering public key: ~/.ssh/id_ed25519 [...] debug1: Server accepts key: ~/.ssh/id_ed25519 [...] Authenticated to staging.ottg.co.uk ([165.232.110.81]:22) using "publickey". ---- That one actually worked! But in the verbose output, you can watch to make sure it finds the right SSH keys, for example.((("public/private key pairs", "SSH keys"))) TIP: If root works but your nonroot user doesn't, you may need to add your public key to `/home/yournonrootuser/.ssh/authorized_keys`. If root doesn't work either, you may need to add your public SSH key to your account settings page, via your provider's web UI. That may or may not take effect immediately; you might need to delete your old server and create a new one. Remember, that probably means a new IP address! .Security ******************************************************************************* A serious discussion of server security is beyond the scope of this book, and I'd warn against running your own servers without learning a good bit more about it.((("server provisioning", "learning more about server security"))) (One reason people choose to use a PaaS to host their code is that it means slightly fewer security issues to worry about.) If you'd like a place to start, here's as good a place as any: https://blog.codelitt.com/my-first-10-minutes-on-a-server-primer-for-securing-ubuntu. I can definitely recommend the eye-opening experience of installing Fail2Ban and watching its logfiles to see just how quickly it picks up on random drive-by attempts to brute force your SSH login. The internet is a wild place! ((("Ansible", "using with SSH for server interactions", startref="ix_AnsSSH")))((("SSH", "debugging issues with", startref="ix_SSHdbg")))((("security issues and settings", "server security"))) ((("platform-as-a-service (PaaS)"))) ******************************************************************************* ==== Installing Ansible Assuming we can reliably SSH into the server, it's time to install Ansible and make sure it can talk to our server as well.((("Ansible", "installing"))) Take a look at the https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html[Ansible installation guide] for all the various options, but probably the simplest thing to do is to install Ansible into the virtualenv on our local machine (Ansible doesn't need to be installed on the server): [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *pip install ansible* # we also need the Docker SDK for the ansible/docker integration to work: $ *pip install docker* ---- // TODO: consider introducing an explicit requirements.dev.txt here, // with -r requirements.txt and put ansible, docker, and selenium in there. // or, maybe get that in place in the previous chapter, keep this one shorter. [role="pagebreak-before less_space"] ==== Checking Ansible Can Talk to Our Server This is the last step in ensuring we're ready: making sure Ansible can talk to our server.((("Ansible", "checking interactions with server", id="ix_Ansserint"))) At the core of Ansible is what's called a "playbook", which describes what we want to happen on our server. Let's create one now. It's probably a good idea to keep it in a folder of its own: [subs="quotes"] ---- *mkdir infra* ---- And here's a minimal playbook whose job is just to "ping" the server, to check we can talk to it.((("YAML (yet another markup language)"))) It's in a format called YAML (yet another markup language) which, if you've never come across before, you will soon develop a love-hate relationship for:footnote:[ The "love" part is that YAML is very easy to _read_ and scan through at a glance. The "hate" part is that the actual syntax is surprisingly fiddly to get right: the difference between lists and key/value maps is subtle and I can never quite remember it, honestly.] [role="sourcecode"] .infra/deploy-playbook.yaml (ch11l001) ==== [source,yaml] ---- - hosts: all tasks: - name: Ping to make sure we can talk to our server ansible.builtin.ping: ---- ==== We won't worry too much about the syntax or how it works at the moment; let's just use it to make sure everything works. To invoke Ansible, we use the command `ansible-playbook`, which will have been installed into your virutalenv when we did the `pip install ansible` earlier. Here's the full command we'll use, with an explanation of each part: [role="small-code skipme"] ---- ansible-playbook \ --user=elspeth \ <1> -i staging.ottg.co.uk, \ <2><3> infra/deploy-playbook.yaml \ <4> -vv <5> ---- <1> The `--user=` flag lets us specify the user to use to authenticate with the server. This should be the same user you can SSH with. <2> The `-i` flag specifies what server to run against. <3> Note the trailing comma after the server hostname. Without this, it won't work (it's there because Ansible is designed to work against multiple servers at the same time).footnote:[ The "i" in the `-i` flag stands for "inventory". Using the `-i` flag is actually a little unconventional. If you read the Ansible docs, you'll find they usually recommend having an "inventory file", which lists all your servers, along with various bits of qualifying metadata. That's overkill for our use case though!] <4> Next comes the path to our playbook, as a positional argument. <5> Finally the `-v` or `-vv` flags control how verbose the output will be—useful for debugging! Here's some example output when I run it: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*ansible-playbook --user=elspeth -i staging.ottg.co.uk, infra/deploy-playbook.yaml -vv*] ansible-playbook [core 2.17.5] config file = None configured module search path = ['~/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules'] ansible python module location = ...goat-book/.venv/lib/python3.14/site-packages/ansible ansible collection location = ~/.ansible/collections:/usr/share/ansible/collections executable location = ...goat-book/.venv/bin/ansible-playbook python version = 3.14.0 (main, Oct 11 2024, 22:59:05) [Clang 15.0.0 (clang-1500.3.9.4)] (...goat-book/.venv/bin/python) jinja version = 3.1.4 libyaml = True No config file found; using defaults Skipping callback 'default', as we already have a stdout callback. Skipping callback 'minimal', as we already have a stdout callback. Skipping callback 'oneline', as we already have a stdout callback. PLAYBOOK: deploy-playbook.yaml ************************************************ 1 plays in infra/deploy-playbook.yaml PLAY [all] ******************************************************************** TASK [Gathering Facts] ******************************************************** task path: ...goat-book/infra/deploy-playbook.yaml:1 [WARNING]: Platform linux on host staging.ottg.co.uk is using the discovered Python interpreter at /usr/bin/python3.10, but future installation of another Python interpreter could change the meaning of that path. See https://docs.ansible.com/ansible- core/2.17/reference_appendices/interpreter_discovery.html for more information. ok: [staging.ottg.co.uk] TASK [Ping to make sure we can talk to our server] **************************** task path: ...goat-book/infra/deploy-playbook.yaml:3 ok: [staging.ottg.co.uk] => {"changed": false, "ping": "pong"} PLAY RECAP ******************************************************************** staging.ottg.co.uk : ok=2 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 ---- Looking good! In the next chapter, we'll use Ansible to get our app up and running on our server. It'll be a thrill, I promise!((("Ansible", "checking interactions with server", startref="ix_Ansserint")))((("server provisioning", "recap of"))) [role="pagebreak-before less_space"] .Server Prep Recap ******************************************************************************* VPS versus PaaS:: We discussed the trade-offs of running your own server versus opting for a PaaS. A VPS is great for learning, but you might find the lower admin overhead of a PaaS makes sense for real projects.((("platform-as-a-service (PaaS)", "VPS versus")))((("VPS (virtual private server)", "versus PaaS", secondary-sortas="PaaS"))) Domain name registration and DNS:: This tends to be something you((("domains", "domain name registration and DNS"))) only do once, but buying a domain name and pointing it at your server is an unavoidable part of hosting a web app. Now you know your TTLs from your A-records! SSH:: SSH is the Swiss Army knife of server admin.((("SSH"))) The dream is that everything is automated, but now and again you just gotta open up a shell on that box! Ansible:: Ansible will be our deployment automation tool.((("Ansible"))) We've had the barest of teasers, but we have it installed and we're ready to learn how to use it. ******************************************************************************* ================================================ FILE: chapter_12_ansible.asciidoc ================================================ [[chapter_12_ansible]] == Infrastructure as Code: Automated Deployments with Ansible [quote, 'Cay S. Horstmann'] ______________________________________________________________ Automate, automate, automate. ______________________________________________________________ ((("deployment", "automating with Ansible", id="ix_dplyautAns"))) ((("infrastructure as code (IaC)")))((("IaC", see="infrastructure as code")))((("Ansible", "automated deployments with", id="ix_Ansautd"))) Now that our server is up and running, we want to install our app on it, using our Docker image and container.((("Docker", "installing app on server"))) We _could_ do this manually, but a key insight of modern software engineering is that small, frequent deployments are a must. NOTE: This insight about the importance of frequent deployments we owe to https://nicolefv.com/writing[Nicole Forsgren] and the _State of DevOps_ reports. They are some of the only really firm science we have in the field of software engineering. Frequent deployments rely on automation,footnote:[ Some readers mentioned a worry that using automation tools would leave them with less understanding of the underlying infrastructure. But in fact, using automation requires deep understanding of the things you're automating. So, don't worry; we'll be taking the time to look under the hood and make sure we know how things work.] so we'll use Ansible. [role="pagebreak-before"] Automation is also key to making sure our tests give us true confidence over our deployments.((("automation of tests, giving confidence in deployments")))((("development server", "deploying")))((("staging server", "deploying"))) If we go to the trouble of building a staging server,footnote:[ Depending on where you work, what I'm calling a "staging" server, some people would call a "development" server, and some others would also like to distinguish "preproduction" servers. Whatever we call it, the point is to have somewhere we can try our code out in an environment that's as similar as possible to the real production server. As we'll see, Docker isn't _quite_ enough!] we want to make sure that it's as similar as possible to the production environment. By automating the way we deploy, and using the same automation for staging and prod, we give ourselves much more confidence. The buzzword for automating your deployments these days is "infrastructure as code" (IaC).((("infrastructure as code (IaC)"))) NOTE: Why not ping me a note once your site is live on the web, and send me the URL? It always gives me a warm and fuzzy feeling...Email me at obeythetestinggoat@gmail.com. //// DAVID overall notes I also think we're missing some stuff at the end about how all this might look as a development workflow. Maybe talk about setting up scripts (so we don't have to remember the ansible command?) And what about releasing to production? It doesn't need much, it just feels unfinished to me. //// === A First Cut of an Ansible Playbook for Deployment Let's start using Ansible a little more seriously.((("Ansible", "automated deployments with", "first draft of playbook for deployment", id="ix_Ansautdplybk")))((("Docker", "Ansible running simple container on our server", id="ix_DckAns"))) We're not going to jump all the way to the end though! Baby steps, as always. Let's see if we can get it to run a simple "hello world" Docker container on our server. [role="pagebreak-before"] Let's delete the old content, which had the "ping", and replace it with something like this: [role="sourcecode"] .infra/deploy-playbook.yaml (ch12l001) ==== [source,yaml] ---- --- - hosts: all tasks: - name: Install docker #<1> ansible.builtin.apt: #<2> name: docker.io #<3> state: latest update_cache: true become: true - name: Run test container community.docker.docker_container: name: testcontainer state: started image: busybox command: echo hello world become: true ---- ==== <1> An Ansible playbook is a series of "tasks"; we now have more than one.((("playbooks", seealso="Ansible"))) In that sense, it's still quite sequential and procedural, but the individual tasks themselves are quite declarative. Each one usually has a human-readable `name` attribute. <2> Each task uses an Ansible "module" to do its work. This one uses the +b⁠u⁠i⁠l⁠t​i⁠n⁠.⁠a⁠p⁠t+ module, which provides a wrapper around the `apt` Debian and Ubuntu package management tool.((("modules (Ansible)"))) <3> Each module then provides a bunch of parameters that control how it works. Here, we specify the `name` of the package we want to install ("docker.io"footnote:[ In the official Docker installation instructions, you'll see a recommendation to install Docker via a private package repository. I wanted to avoid that complexity for the book, but you should probably follow those instructions in a real-world scenario, to make sure your version of Docker has all the latest security patches.]) and tell it to update its cache first, which is required on a fresh server. Most Ansible modules have pretty good documentation—check out the `builtin.apt` one for example; I often skip to the https://docs.ansible.com/ansible/latest/collections/ansible/builtin/apt_module.html#examples["Examples" section]. [role="pagebreak-before"] Let's rerun our deployment command, `ansible-playbook`, with the same flags we used in the last chapter: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*ansible-playbook --user=elspeth -i staging.ottg.co.uk, infra/deploy-playbook.yaml -vv*] ansible-playbook [core 2.16.3] config file = None [...] No config file found; using defaults BECOME password: Skipping callback 'default', as we already have a stdout callback. Skipping callback 'minimal', as we already have a stdout callback. Skipping callback 'oneline', as we already have a stdout callback. PLAYBOOK: deploy-playbook.yaml ************************************************ 1 plays in infra/deploy-playbook.yaml PLAY [all] ******************************************************************** TASK [Gathering Facts] ******************************************************** task path: ...goat-book/superlists/infra/deploy-playbook.yaml:2 ok: [staging.ottg.co.uk] PLAYBOOK: deploy-playbook.yaml ************************************************ 1 plays in infra/deploy-playbook.yaml TASK [Install docker] ********************************************************* task path: ...goat-book/superlists/infra/deploy-playbook.yaml:6 ok: [staging.ottg.co.uk] => {"cache_update_time": 1708981325, "cache_updated": true, "changed": false} TASK [Install docker] ********************************************************* task path: ...goat-book/superlists/infra/deploy-playbook.yaml:6 changed: [staging.ottg.co.uk] => {"cache_update_time": [...] "cache_updated": true, "changed": true, "stderr": "", "stderr_lines": [], "stdout": "Reading package lists...\nBuilding dependency tree...\nReading [...] information...\nThe following additional packages will be installed:\n wmdocker\nThe following NEW packages will be installed:\n docker wmdocker\n0 TASK [Run test container] ***************************************************** task path: ...goat-book/superlists/infra/deploy-playbook.yaml:13 changed: [staging.ottg.co.uk] => {"changed": true, "container": {"AppArmorProfile": "docker-default", "Args": ["hello", "world"], "Config": [...] PLAY RECAP ******************************************************************** staging.ottg.co.uk : ok=3 changed=2 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 ---- // DAVID: rather than having to edit the username and domains each time, // what about getting the reader to set them as environment variables at the beginning of the chapter? I don't know about you, but whenever I make a terminal spew out a stream of output, I like to make little _brrp brrp brrp_ noises—a bit like the computer, Mother, in _Alien_. Ansible scripts are particularly satisfying in this regard. TIP: You may need to use the `--ask-become-pass` argument to `ansible-playbook` if you get an error, "Missing sudo password".footnote:[ You can also look into "passwordless sudo" if it's all just too annoying, but that does have security implications.] .Idempotence and Declarative Configuration ******************************************************************************* IaC tools like Ansible aim to be "declarative", meaning that, as much as possible, you specify the desired state that you want, rather than specifying a series of steps to get there.((("declarative IaC tools")))((("infrastructure as code (IaC)", "declarative tools for"))) This concept goes along with the idea of "idempotence", which is when you want a thing that has the same effect, whether it is run just once or multiple times.((("idempotence"))) An example is the `apt` module that we used to install Docker. It doesn't crash if Docker is already installed and, in fact, Ansible is smart enough to check first before trying to install anything. It makes no difference whether you run it once or many times.((("Ansible", "automated deployments with", "first draft of playbook for deployment", startref="ix_Ansautdplybk")))((("Docker", "Ansible running simple container on our server", startref="ix_DckAns"))) In contrast, adding an item to our to-do list is not currently idempotent. If I add "Buy milk" and then I add "Buy milk" again, I end up with two items that both say "Buy milk". (We might fix that later, mind you.) ******************************************************************************* === SSHing Into the Server and Viewing Container Logs Ansible _looks_ like it's doing its job, but let's practice our SSH skills, and do some good old-fashioned system admin.((("SSH", "SSHing into server and viewing container logs", id="ix_SSHser")))((("Docker", "viewing container logs on"))) Let's log in to our server and see if we can see any actual evidence that our container has run. After we `ssh` in, we can use `docker ps`, just like we do on our own machine. We pass the `-a` flag to view _all_ containers, including old/stopped ones. Then we can use `docker logs` to view the output from one of them: [role="server-commands"] [subs="specialcharacters,quotes"] ---- $ *ssh elspeth@staging.superlists.ottg.co.uk* Welcome to Ubuntu 22.04.4 LTS (GNU/Linux 5.15.0-67-generic x86_64) [...] elspeth@server$ *sudo docker ps -a* CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 3a2e600fbe77 busybox "echo hello world" 2 days ago Exited (0) 10 minutes ago testcontainer elspeth@server:$ *sudo docker logs testcontainer* hello world ---- TIP: Look out for that `elspeth@server` in the command-line listings in this chapter. It indicates commands that must be run on the server, as opposed to commands you run on your own PC. SSHing in to check things worked is a key server debugging skill! It's something we want to practice on our staging server, because ideally we'll want to avoid doing it on production machines. .Docker Debugging ******************************************************************************* ((("debugging", "Docker")))((("Docker", "debugging"))) Here's a rundown of some of the debugging tools—some we've already seen and some new ones we'll use in this chapter. When things don't go to plan, they can help shed some light. All of them should be run on the server, inside an SSH session: - You can check the Container logs using `docker logs superlists`. - You can run things "inside" the container with `docker exec `. A couple of useful examples include `docker exec superlists env`, to print environment variables, and just `docker exec -it superlists bash` to open an interactive Bash shell, inside the container. - You can get lots of detailed info on the _container_ using `docker inspect superlists`. This is a good place to go check on environment variables, port mappings, and exactly which image was running, for example. - You can get detailed info on the _image_ with `docker image inspect superlists`. You might need this to check the exact image hash, to make sure it's the same one you built locally. ******************************************************************************* === Allowing Rootless Docker Access Having to use `sudo` or `become=True` to run Docker commands is a bit of a pain.((("SSH", "SSHing into server and viewing container logs", startref="ix_SSHser")))((("root user", "allowing rootless Docker access", id="ix_rootls")))((("Docker", "rootless access, allowing", id="ix_Dckrtl"))) If we add our user to the `docker` group, we can run Docker commands without `sudo`: [role="sourcecode"] .infra/deploy-playbook.yaml (ch12l001-1) ==== [source,yaml] ---- - name: Install docker [...] - name: Add our user to the docker group, so we don't need sudo/become ansible.builtin.user: # <1> name: '{{ ansible_user }}' # <2> groups: docker append: true # don't remove any existing groups. become: true - name: Reset ssh connection to allow the user/group change to take effect ansible.builtin.meta: reset_connection # <3> - name: Run test container # <4> [...] ---- ==== <1> We use the `builtin.user` module to add our user to the `docker` group. <2> The `{{ ... }}` syntax enables us to interpolate some variables into our config file, much like in a Django template. `ansible_user` will be the user we're using to connect to the server—i.e., "elspeth", in my case. <3> As per the task name, we need this for the user/group change to take effect. Strictly speaking, this is only needed the first time we run the script; if you've got some time, you can read up on how to make tasks https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_conditionals.html[conditional] and configure it to only run if the `builtin.user` tasks has actually made a change. <4> We can remove the `become: true` from this task and it should still work. [role="pagebreak-before"] Let's run that: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*ansible-playbook --user=elspeth -i staging.ottg.co.uk, infra/deploy-playbook.yaml -vv*] PLAYBOOK: deploy-playbook.yaml ************************************************ 1 plays in infra/deploy-playbook.yaml PLAY [all] ******************************************************************** TASK [Gathering Facts] ******************************************************** [...] ok: [staging.ottg.co.uk] TASK [Install docker] ********************************************************* [...] ok: [staging.ottg.co.uk] => {"cache_update_time": 1738767216, "cache_updated": true, "changed": false} TASK [Add our user to the docker group, so we don't need sudo/become] ********* [...] changed: [staging.ottg.co.uk] => {"append": false, "changed": true, [...] "", "group": 1000, "groups": "docker", [...] TASK [Reset ssh connection to allow the user/group change to take effect] ***** [...] META: reset connection TASK [Run test container] ***************************************************** [...] changed: [staging.ottg.co.uk] => {"changed": true, "container": [...] PLAY RECAP ******************************************************************** staging.ottg.co.uk : ok=4 changed=2 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 ---- And check that it worked: [role="server-commands"] [subs="specialcharacters,quotes"] ---- elspeth@server$ *docker ps -a* # no sudo yay! CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES bd3114e43f55 busybox "echo hello world" 12 minutes ago Exited (0) 6 seconds ago testcontainer elsepth@server$ *docker logs testcontainer* hello world hello world ---- [role="pagebreak-before"] Sure enough, we no longer need `sudo`, and we can see that a new version of the container just ran. You know, that's worthy of a commit! [subs="specialcharacters,quotes"] ---- $ *git add infra/deploy-playbook.yaml* $ *git commit -m "Made a start on an ansible playbook for deployment"* ---- Let's move on to trying to get our actual Docker container running on the server. As we go through, you'll see that we're going to work through very similar issues to the ones we've already figured our way through in the last couple of chapters: * Configuration * Networking * The database((("root user", "allowing rootless Docker access", startref="ix_rootls")))((("Docker", "rootless access, allowing", startref="ix_Dckrtl"))) === Getting Our Image Onto the Server Typically, you can "push" and "pull" container images to a "container registry"—Docker offers a public one called Docker Hub, and organisations will often run private ones, hosted by cloud providers like AWS.((("Ansible", "automated deployments with", "getting container image onto server", id="ix_Ansautdcntnr")))((("Docker", "getting container image onto our server", id="ix_Dckcntimg")))((("containers", "getting container image onto server", id="ix_cntnrser"))) So your process of getting an image onto a server is usually: 1. Push the image from your machine to the registry. 2. Pull the image from the registry onto the server. Usually this step is implicit, in that you just specify the image name in the format `registry-url/image-name:tag`, and then `docker run` takes care of pulling down the image for you. But I don't want to ask you to create a Docker Hub account, nor implicitly endorse any particular provider, so we're going to "simulate" this process by doing it manually. [role="pagebreak-before"] It turns out you can "export" a container image to an archive format, manually copy that to the server, and then reimport it. In Ansible config, it looks like this: [role="sourcecode"] .infra/deploy-playbook.yaml (ch12l002) ==== [source,yaml] ---- - name: Install docker [...] - name: Add our user to the docker group, so we don't need sudo/become [...] - name: Reset ssh connection to allow the user/group change to take effect [...] - name: Export container image locally # <1> community.docker.docker_image: name: superlists archive_path: /tmp/superlists-img.tar source: local delegate_to: 127.0.0.1 - name: Upload image to server # <2> ansible.builtin.copy: src: /tmp/superlists-img.tar dest: /tmp/superlists-img.tar - name: Import container image on server # <3> community.docker.docker_image: name: superlists load_path: /tmp/superlists-img.tar source: load force_source: true # <4> state: present - name: Run container community.docker.docker_container: name: superlists image: superlists # <5> state: started recreate: true # <6> ---- ==== [role="pagebreak-before"] <1> We export the Docker image to a _.tar_ file by using the `docker_image` module with the `archive_path` set to a tempfile, and setting the `delegate_to` attribute to say we're running that command on our local machine rather than the server. <2> We then use the `copy` module to upload the _.tar_ file to the server. <3> And we use `docker_image` again, but this time with `load_path` and `source: load` to import the image back on the server. <4> The `force_source` flag tells the server to attempt the import, even if an image of that name already exists. <5> We change our "run container" task to use the `superlists` image, and we'll use that as the container name too. <6> Similarly to `source: load`, the `recreate` argument tells Ansible to re-create the container even if there's already one running whose name and image match "superlists". // TODO: consider using commit id as image tag to avoid the force_source. NOTE: If you see an error saying "Error connecting: Error while fetching server API version", it may be because the Python Docker software development kit (SDK) can't find your Docker daemon. Try restarting Docker Desktop if you're on Windows or a Mac.((("DOCKER_HOST environment variable"))) If you're not using the standard Docker engine—with Colima or Podman, for example—you may need to set the `DOCKER_HOST` environment variable (e.g., `DOCKER_HOST=unix:///$HOME/.colima/default/docker.sock`) or use a symlink to point to the right place. See the https://oreil.ly/gPJmq[Colima FAQ] or https://oreil.ly/Hqoma[Podman docs]. [role="pagebreak-before"] Let's run the new version of our playbook, and see if we can upload a Docker image to our server and get it running: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*ansible-playbook --user=elspeth -i staging.ottg.co.uk, infra/deploy-playbook.yaml -vv*] [...] PLAYBOOK: deploy-playbook.yaml ********************************************** 1 plays in infra/deploy-playbook.yaml PLAY [all] ******************************************************************** TASK [Gathering Facts] ******************************************************** task path: ...goat-book/superlists/infra/deploy-playbook.yaml:2 ok: [staging.ottg.co.uk] TASK [Install docker] ********************************************************* task path: ...goat-book/superlists/infra/deploy-playbook.yaml:5 ok: [staging.ottg.co.uk] => {"cache_update_time": 1708982855, "cache_updated": false, "changed": false} TASK [Add our user to the docker group, so we don't need sudo/become] ********* task path: ...goat-book/infra/deploy-playbook.yaml:11 ok: [staging.ottg.co.uk] => {"append": false, "changed": false, [...] TASK [Reset ssh connection to allow the user/group change to take effect] ***** task path: ...goat-book/infra/deploy-playbook.yaml:17 META: reset connection TASK [Export container image locally] ***************************************** task path: ...goat-book/superlists/infra/deploy-playbook.yaml:20 changed: [staging.ottg.co.uk -> 127.0.0.1] => {"actions": ["Archived image superlists:latest to /tmp/superlists-img.tar, overwriting archive with image 11ff3b83873f0fea93f8ed01bb4bf8b3a02afa15637ce45d71eca1fe98beab34 named superlists:latest"], "changed": true, "image": {"Architecture": "amd64", [...] TASK [Upload image to server] ************************************************* task path: ...goat-book/superlists/infra/deploy-playbook.yaml:27 changed: [staging.ottg.co.uk] => {"changed": true, "checksum": "313602fc0c056c9255eec52e38283522745b612c", "dest": "/tmp/superlists-img.tar", [...] TASK [Import container image on server] *************************************** task path: ...goat-book/superlists/infra/deploy-playbook.yaml:32 changed: [staging.ottg.co.uk] => {"actions": ["Loaded image superlists:latest from /tmp/superlists-img.tar"], "changed": true, "image": {"Architecture": "amd64", "Author": "", "Comment": "buildkit.dockerfile.v0", "Config": [...] TASK [Run container] ********************************************************** task path: ...goat-book/superlists/infra/deploy-playbook.yaml:40 changed: [staging.ottg.co.uk] => {"changed": true, "container": {"AppArmorProfile": "docker-default", "Args": ["--bind", ":8888", "superlists.wsgi:application"], "Config": {"AttachStderr": true, "AttachStdin": false, "AttachStdout": true, "Cmd": ["gunicorn", "--bind", ":8888", "superlists.wsgi:application"], "Domainname": "", "Entrypoint": null, "Env": [...] staging.ottg.co.uk : ok=7 changed=4 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 ---- That looks good! For completeness, let's also add a step to explicitly build the image locally (this means we aren't dependent on having run `docker build` locally): [role="sourcecode"] .infra/deploy-playbook.yaml (ch12l003) ==== [source,yaml] ---- - name: Reset ssh connection to allow the user/group change to take effect [...] - name: Build container image locally community.docker.docker_image: name: superlists source: build state: present build: path: .. platform: linux/amd64 # <1> force_source: true delegate_to: 127.0.0.1 - name: Export container image locally [...] ---- ==== <1> I needed this `platform` attribute to work around an issue with compatibility between Apple's new ARM-based chips and our server's x86/AMD64 architecture. You could also use this `platform:` to cross-build Docker images for a Raspberry Pi from a regular PC, or vice versa. It does no harm in any case.((("Ansible", "automated deployments with", "getting container image onto server", startref="ix_Ansautdcntnr")))((("Docker", "getting container image onto our server", startref="ix_Dckcntimg")))((("containers", "getting container image onto server", startref="ix_cntnrser"))) ==== Taking a Look Around Manually Time to take another proverbial look under the hood, to check whether it really worked.((("Ansible", "automated deployments with", "checking if container deployment worked"))) Hopefully we'll see a container that looks like ours: [role="server-commands"] [subs="specialcharacters,quotes"] ---- $ *ssh elspeth@staging.superlists.ottg.co.uk* Welcome to Ubuntu 22.04.4 LTS (GNU/Linux 5.15.0-67-generic x86_64) [...] elspeth@server$ *docker ps -a* CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 3a2e600fbe77 busybox "echo hello world" 2 days ago Exited (0) 10 minutes ago testcontainer 129e36a42190 superlists "/bin/sh -c \'gunicor…" About a minute ago Exited (3) About a minute ago superlists ---- OK! We can see our "superlists" container is there now, both named "superlists" and based on an image called "superlists". The `Status: Exited` is a bit more worrying though. [role="pagebreak-before"] Still, that's a good bit of progress, so let's do a commit (back on your own machine): [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *git commit -am"Build our image, use export/import to get it on the server, try and run it"* ---- ===== Docker logs Now, back on the server, let's take a look at the logs of our new container to see if we can figure ((("Ansible", "automated deployments with", "checking Docker logs on container deployment")))((("Docker", "checking logs of container deployed to server")))out what's happened: [role="server-commands"] [subs="specialcharacters,quotes"] ---- elspeth@server:$ *docker logs superlists* [2024-02-26 22:19:15 +0000] [1] [INFO] Starting gunicorn 21.2.0 [2024-02-26 22:19:15 +0000] [1] [INFO] Listening at: http://0.0.0.0:8888 (1) [2024-02-26 22:19:15 +0000] [1] [INFO] Using worker: sync [...] File "/src/superlists/settings.py", line 22, in SECRET_KEY = os.environ["DJANGO_SECRET_KEY"] ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ File "", line 685, in __getitem__ KeyError: 'DJANGO_SECRET_KEY' [2024-02-26 22:19:15 +0000] [7] [INFO] Worker exiting (pid: 7) [2024-02-26 22:19:15 +0000] [1] [ERROR] Worker (pid:7) exited with code 3 [2024-02-26 22:19:15 +0000] [1] [ERROR] Shutting down: Master [2024-02-26 22:19:15 +0000] [1] [ERROR] Reason: Worker failed to boot. ---- Oh, whoops; it can't find the `DJANGO_SECRET_KEY` environment variable. We need to set those environment variables on the server too.((("DJANGO_SECRET_KEY environment variable"))) === Setting Environment Variables and Secrets When we run our container manually locally with `docker run`, we can pass in environment variables with the `-e` flag.((("secrets", "setting and checking on deployed Docker container", id="ix_scrtcntn")))((("environment variables", "setting and checking on deployed Docker container", id="ix_envvarDckcnt")))((("Ansible", "automated deployments with", "setting environment variables and secrets on Docker container", id="ix_Ansautdenv")))((("Docker", "setting environment variables and secrets", id="ix_Dckenvsec"))) As we'll see, it's fairly straightforward to replicate that with Ansible, using the `env` parameter for the `docker.docker_container` module that we're already using. But there is at least one "secret" value that we don't want to hardcode into our Ansible YAML file: the Django `SECRET_KEY` setting. There are many different ways of dealing with secrets; different cloud providers have their own tools. There's also HashiCorp Vault—it has varying levels of complexity and security. We don't have time to go into detail on those in this book. Instead, we'll generate a one-off secret key value from a random string, and we'll store it to a file on disk on the server. That's a reasonable amount of security for our purposes. [role="pagebreak-before"] So, here's the plan: 1. We generate a random, one-off secret key the first time we deploy to a new server, and we store it in a file on disk. 2. We read the secret key value back from that file to put it into the container's environment variables. 3. We set the rest of the env vars we need as well. Here's what it looks like: [role="sourcecode small-code"] .infra/deploy-playbook.yaml (ch12l005) ==== [source,yaml] ---- - name: Import container image on server [...] - name: Ensure .secret-key file exists # the intention is that this only happens once per server ansible.builtin.copy: # <1> dest: ~/.secret-key content: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters') }}" # <2> mode: 0600 force: false # do not recreate file if it already exists. - name: Read secret key back from file ansible.builtin.slurp: # <3> src: ~/.secret-key register: secret_key - name: Run container community.docker.docker_container: name: superlists image: superlists state: started recreate: true env: # <4> DJANGO_DEBUG_FALSE: "1" DJANGO_SECRET_KEY: "{{ secret_key.content | b64decode }}" # <5> DJANGO_ALLOWED_HOST: "{{ inventory_hostname }}" # <6> DJANGO_DB_PATH: "/home/nonroot/db.sqlite3" ---- ==== <1> The `builtin.copy` module can be used to copy local files up to the server, and also, as we're demonstrating here, to populate a file with an arbitrary string `content`. <2> This `lookup('password')` thing is how we'll get a random string of characters. I copy-pasted it from Stack Overflow. Come on; there's no shame in that. The rest of the `builtin.copy` directive is designed to save the value to disk, but only if the file doesn't already exist. The `0600` permission will ensure that only the "elspeth" user can read it. <3> The `slurp` command reads the contents of a file on the server, and we can `register` its contents into a variable. Slightly annoyingly, it uses base64 encoding (it's so you can also use it to read binary files). Anyway, the idea is, even though we don't _rewrite_ the file on every deploy, we do _reread_ the value on every deploy. <4> Here's the `env` parameter for our container. <5> Here's how we get our original value for the secret key, using the `| b64decode` to decode it back to a regular string. <6> `inventory_hostname` represents the hostname of the current server we're deploying to, so _staging.ottg.co.uk_ in our case. Let's run this latest version of our playbook now: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*ansible-playbook --user=elspeth -i staging.ottg.co.uk, infra/deploy-playbook.yaml -v*] [...] PLAYBOOK: deploy-playbook.yaml ********************************************** 1 plays in infra/deploy-playbook.yaml PLAY [all] ******************************************************************** TASK [Gathering Facts] ******************************************************** ok: [staging.ottg.co.uk] TASK [Install docker] ********************************************************* ok: [staging.ottg.co.uk] => {"cache_update_time": 1709136057, "cache_updated": false, "changed": false} TASK [Build container image locally] ****************************************** changed: [staging.ottg.co.uk -> 127.0.0.1] => {"actions": ["Built image [...] TASK [Export container image locally] ***************************************** changed: [staging.ottg.co.uk -> 127.0.0.1] => {"actions": ["Archived image [...] TASK [Upload image to server] ************************************************* changed: [staging.ottg.co.uk] => {"changed": true, [...] TASK [Import container image on server] *************************************** changed: [staging.ottg.co.uk] => {"actions": ["Loaded image [...] TASK [Ensure .env file exists] ************************************************ changed: [staging.ottg.co.uk] => {"changed": true, [...] TASK [Run container] ********************************************************** changed: [staging.ottg.co.uk] => {"changed": true, "container": [...] PLAY RECAP ******************************************************************** staging.ottg.co.uk : ok=8 changed=6 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 ---- [role="pagebreak-before less_space"] ==== Manually Checking Environment Variables for Running Containers We'll do one more manual check with SSH, to see if those env vars were set correctly.((("secrets", "setting and checking on deployed Docker container", startref="ix_scrtcntn")))((("Docker", "setting environment variables and secrets", "checking environment variables with docker ps"))) There's a couple of ways we can do this. Let's start with a `docker ps` to check whether our container is running: [role="server-commands"] [subs="specialcharacters,quotes"] ---- elspeth@server:$ *docker ps* CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 96d867b42a31 superlists "gunicorn --bind :88…" 6 seconds ago Up 5 seconds superlists ---- Looking good! The `STATUS: Up 5 Seconds` is better than the `Exited` we had before; that means the container is up and running. Let's take a look at the `docker logs` too: [role="server-commands"] [subs="specialcharacters,quotes"] ---- elspeth@server:~$ *docker logs superlists* [2025-05-02 17:55:18 +0000] [1] [INFO] Starting gunicorn 23.0.0 [2025-05-02 17:55:18 +0000] [1] [INFO] Listening at: http://0.0.0.0:8888 (1) [2025-05-02 17:55:18 +0000] [1] [INFO] Using worker: sync [2025-05-02 17:55:18 +0000] [7] [INFO] Booting worker with pid: 7 ---- Also looking good; no sign of an error. Now let's check on those environment variables. There are two ways we can do this: `docker exec env` and `docker inspect`. ===== docker exec env One way is to run the standard shell `env` command, which prints out all environment variables.((("Docker", "setting environment variables and secrets", "checking settings with docker exec env"))) We run it "inside" the container with `docker exec`: [role="server-commands small-code"] [subs="specialcharacters,quotes"] ---- elspeth@server:~$ *docker exec superlists env* PATH=/venv/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=96d867b42a31 DJANGO_DEBUG_FALSE=1 DJANGO_SECRET_KEY=cXACJZTvoPfWFSBSTdixJTlXCWYTnJlC DJANGO_ALLOWED_HOST=staging.ottg.co.uk DJANGO_DB_PATH=/home/nonroot/db.sqlite3 GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305 PYTHON_VERSION=3.14.3 PYTHON_SHA256=40f868bcbdeb8149a3149580bb9bfd407b3321cd48f0be631af955ac92c0e041 HOME=/home/nonroot ---- [role="pagebreak-before less_space"] ===== docker inspect Another option--useful ((("Docker", "setting environment variables and secrets", "checking settings with docker inspect")))for debugging other things too, like image IDs and mounts--is to use `docker inspect`: [role="server-commands small-code"] [subs="specialcharacters,quotes"] ---- elspeth@server:~$ *docker inspect superlists* [ { [...] "Config": { [...] "Env": [ "DJANGO_DEBUG_FALSE=1", "DJANGO_SECRET_KEY=cXACJZTvoPfWFSBSTdixJTlXCWYTnJlC", "DJANGO_ALLOWED_HOST=staging.ottg.co.uk", "DJANGO_DB_PATH=/home/nonroot/db.sqlite3", "PATH=/venv/bin:/usr/local/bin:/usr/local/sbin:/usr/[...] "GPG_KEY=7169605F62C751356D054A26A821E680E5FA6305", "PYTHON_VERSION=3.14.3", "PYTHON_SHA256=40f868bcbdeb8149a3149580bb9bfd407b332[...] ], "Cmd": [ "gunicorn", "--bind", ":8888", "superlists.wsgi:application" ], "Image": "superlists", "Volumes": null, "WorkingDir": "/src", "Entrypoint": null, "OnBuild": null, "Labels": {} }, "NetworkSettings": { [...] } } ] ---- There's a lot of output! It's more or less everything that Docker knows about the container. But if you scroll around, you can usually get some useful info for debugging and diagnostics—like, in this case, the `Env` parameter which tells us what environment variables were set for the container. TIP: `docker inspect` is also useful for checking exactly which image ID a container is using, and which filesystem mounts are configured. Looking good!((("environment variables", "setting and checking on deployed Docker container", startref="ix_envvarDckcnt")))((("Ansible", "automated deployments with", "setting environment variables and secrets on Docker container", startref="ix_Ansautdenv")))((("Docker", "setting environment variables and secrets", startref="ix_Dckenvsec"))) [role="pagebreak-before less_space"] === Running FTs to Check on Our Deploy Enough manual checking via SSH; let's see what our tests think.((("deployment", "running functional tests to check server deployment", id="ix_dplytstser"))) The `TEST_SERVER` adaptation we made in <> can also be used to check against our staging server. // DAVID: I originally just pasted this as-is, which contacted YOUR server. Another // reason to get them to set environment variables at the start of the chapter. Let's see what they think: [role="skipme"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=staging.ottg.co.uk python src/manage.py test functional_tests*] [...] selenium.common.exceptions.WebDriverException: Message: Reached error page: about:neterror?e=connectionFailure&u=http%3A//staging.ottg.co.uk/[...] [...] Ran 3 tests in 5.014s FAILED (errors=3) ---- None of them passed. Hmm. That `neterror` makes me think it's another networking problem. NOTE: If your domain provider puts up a temporary holding page, you may get a 404 rather than a connection error at this point, and the traceback might have "NoSuchElementException" instead. ==== Manual Debugging with curl Against the Staging Server Let's try our standard debugging technique of using `curl` both locally and then from inside the container on the server. First, on((("debugging", "of staging server deployment", "manually, using curl", secondary-sortas="staging")))((("curl utility", "debugging against staging staging server with "))) our own machine: [role="skipme"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*curl -iv staging.ottg.co.uk*] [...] curl: (7) Failed to connect to staging.ottg.co.uk port 80 after 25 ms: Couldn't connect to server ---- NOTE: Similarly, depending on your domain/hosting provider, you may see "Host not found" here instead. Or, if your version of `curl` is different, you might see "Connection refused". Now let's SSH in to our server and take a look at the Docker logs: // TODO: rework server-commands book parser to detect "elsepth@server" instead of manual skips (or role=) [role="server-commands"] [subs="specialcharacters,quotes"] ---- elspeth@server$ *docker logs superlists* [2024-02-28 22:14:43 +0000] [7] [INFO] Starting gunicorn 21.2.0 [2024-02-28 22:14:43 +0000] [7] [INFO] Listening at: http://0.0.0.0:8888 (7) [2024-02-28 22:14:43 +0000] [7] [INFO] Using worker: sync [2024-02-28 22:14:43 +0000] [8] [INFO] Booting worker with pid: 8 ---- No errors there. Let's try our `curl`: [role="server-commands"] [subs="specialcharacters,quotes"] ---- elspeth@server$ *curl -iv localhost* * Trying 127.0.0.1:80... * connect to 127.0.0.1 port 80 failed: Connection refused * Trying ::1:80... * connect to ::1 port 80 failed: Connection refused * Failed to connect to localhost port 80 after 0 ms: Connection refused * Closing connection 0 curl: (7) Failed to connect to localhost port 80 after 0 ms: Connection refused ---- Hmm, `curl` fails on the server too. But all this talk of port `80`, both locally and on the server, might be giving us a clue. Let's check `docker ps`: // CSANAD: Ackchually I'm not sure if it's supposed to work, since we set // `inventory_hostname` for DJANGO_ALLOWED_HOSTS, so `localhost` // would not get through. [role="server-commands"] [subs="specialcharacters,quotes"] ---- elspeth@server:$ *docker ps* CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 1dd87cbfa874 superlists "/bin/sh -c 'gunicor…" 9 minutes ago Up 9 minutes superlists ---- This might be ringing a bell now--we forgot the ports.((("ports", "mapping between container and deployed server"))) We want to map port `8888` inside the container as port `80` (the default web/HTTP port) on the server: [role="sourcecode"] .infra/deploy-playbook.yaml (ch12l006) ==== [source,yaml] ---- - name: Run container community.docker.docker_container: name: superlists image: superlists state: started recreate: true env: DJANGO_DEBUG_FALSE: "1" DJANGO_SECRET_KEY: "{{ secret_key.content | b64decode }}" DJANGO_ALLOWED_HOST: "{{ inventory_hostname }}" DJANGO_DB_PATH: "/home/nonroot/db.sqlite3" ports: 80:8888 ---- ==== NOTE: You can map a different port on the outside to the one that's "inside" the Docker container. In this case, we can map the public-facing standard HTTP port `80` on the host to the arbitrarily chosen port `8888` on the inside. Let's push that up with `ansible-playbook`: [subs="specialcharacters,macros"] ---- $ pass:quotes[*ansible-playbook --user=elspeth -i staging.ottg.co.uk, \ infra/deploy-playbook.yaml -v*] [...] ---- [role="pagebreak-before"] And now give the FTs another go: [role="skipme small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=staging.ottg.co.uk python src/manage.py test functional_tests*] [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_list_table"]; [...] [...] Ran 3 tests in 21.047s FAILED (errors=3) ---- So, 3/3 failed again, but the FTs _did_ get a little further along. If you saw what was happening, or if you go and visit the site manually in your browser, you'll see that the home page loads fine, but as soon as we try and create a new list item, it crashes with a 500 error.((("deployment", "running functional tests to check server deployment", startref="ix_dplytstser"))) === Mounting the Database on the Server and Running Migrations Let's do another bit of ((("databases", "mounting on deployed server and running migrations", id="ix_DBmntcntr")))manual debugging, and take a look at the logs from our container with `docker logs`. You'll see an `OperationalError`: [role="server-commands"] [subs="specialcharacters,quotes"] ---- $ *ssh elspeth@server docker logs superlists* [...] django.db.utils.OperationalError: no such table: lists_list ---- It looks like our database isn't initialised. Aha! Another of those deployment "danger areas". Just like we did on our own machine, we need to mount the `db.sqlite3` file from the filesystem outside the container. We'll also want to run migrations to create the database and, in fact, each time we deploy, so that any updates to the database schema get applied to the database on the server. Here's the plan: 1. On the host machine, we'll store the database in elspeth's home folder; it's as good a place as any. 2. We'll set its UID to `1234`, just like we did in <>, to match the UID of the `nonroot` user inside the container. 3. Inside the container, we'll use the path `/home/nonroot/db.sqlite3`—again, just like in the last chapter. 4. We'll run the migrations with a `docker exec`, or the Ansible equivalent thereof. [role="pagebreak-before"] Here's what that looks like: [role="sourcecode"] .infra/deploy-playbook.yaml (ch12l007) ==== [source,python] ---- - name: Ensure db.sqlite3 file exists outside container ansible.builtin.file: path: "{{ ansible_env.HOME }}/db.sqlite3" # <1> state: touch # <2> owner: 1234 # so nonroot user can access it in container become: true # needed for ownership change - name: Run container community.docker.docker_container: name: superlists image: superlists state: started recreate: true env: DJANGO_DEBUG_FALSE: "1" DJANGO_SECRET_KEY: "{{ secret_key.content | b64decode }}" DJANGO_ALLOWED_HOST: "{{ inventory_hostname }}" DJANGO_DB_PATH: "/home/nonroot/db.sqlite3" mounts: # <3> - type: bind source: "{{ ansible_env.HOME }}/db.sqlite3" # <1> target: /home/nonroot/db.sqlite3 ports: 80:8888 - name: Run migration inside container community.docker.docker_container_exec: # <4> container: superlists command: ./manage.py migrate ---- ==== <1> `ansible_env` gives us access to the environment variables on the server, including `HOME`, which is the path to the home folder (_/home/elspeth/_ in my case). <2> We use `file` with `state=touch` to make sure a placeholder file exists before we try and mount it in. <3> Here is the `mounts` config, which works a lot like the `--mount` flag to `docker run`. <4> And we use the `docker.container_exec` module to give us the functionality of `docker exec`, to run the migration command inside the container. [role="pagebreak-before"] Let's give that playbook a run and... [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*ansible-playbook --user=elspeth -i staging.ottg.co.uk, infra/deploy-playbook.yaml -v*] [...] TASK [Run migration inside container] ***************************************** changed: [staging.ottg.co.uk] => {"changed": true, "rc": 0, "stderr": "", "stderr_lines": [], "stdout": "Operations to perform:\n Apply all migrations: auth, contenttypes, lists, sessions\nRunning migrations:\n Applying contenttypes.0001_initial... OK\n Applying contenttypes.0002_remove_content_type_name... OK\n Applying auth.0001_initial... OK\n Applying auth.0002_alter_permission_name_max_length... OK\n Applying [...] PLAY RECAP ******************************************************************** staging.ottg.co.uk : ok=9 changed=2 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 ---- === It Workssss Try the tests... [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=staging.ottg.co.uk python src/manage.py test functional_tests*] Found 3 test(s). [...] ... --------------------------------------------------------------------- Ran 3 tests in 13.537s OK ---- Hooray!((("databases", "mounting on deployed server and running migrations", startref="ix_DBmntcntr"))) All the tests pass! That gives us confidence that our automated deploy script can reproduce a fully working app, on a server, hosted on the public internet. That's worthy of a commit: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *git diff* # should show our changes in deploy-playbook yaml $ *git commit -am"Save secret key, set env vars, mount db, run migrations. It works :)"* ---- //// old content follows ==== Use Vagrant to Spin Up a Local VM Running tests against the staging site gives us the ultimate confidence that things are going to work when we go live, but we can also use a VM on our local machine. Download Vagrant and Virtualbox, and see if you can get Vagrant to build a dev server on your own PC, using our Ansible playbook to deploy code to it. Rewire the FT runner to be able to test against the local VM. Having a Vagrant config file is particularly helpful when working in a team--it helps new developers to spin up servers that look exactly like yours.((("", startref="ansible29"))) //// === Deploying to Prod Now that we are confident in our deploy script, let's try using it for our live site!((("deployment", "deploying to production")))((("domains", "passing production domain name to Ansible playbook"))) The main change is to the `-i` flag, where we pass in the production domain name, instead of the staging one: [role="small-code against-server"] [subs=""] ---- $ ansible-playbook --user=elspeth -i www.ottg.co.uk, infra/deploy-playbook.yaml -vv [...] Done. Disconnecting from elspeth@www.ottg.co.uk... done. ---- _Brrp brrp brpp_. Looking good? Go take a click around your live site! === Git Tag the Release ((("Git", "tagging releases"))) One final bit of admin. To preserve a historical marker, we'll use Git tags to mark the state of the codebase that reflects what's currently live on the server: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *git tag LIVE* $ *export TAG=$(date +DEPLOYED-%F/%H%M)* # this generates a timestamp $ *echo $TAG* # should show "DEPLOYED-" and then the timestamp $ *git tag $TAG* $ *git push origin LIVE $TAG* # pushes the tags up to GitHub ---- Now it's easy, at any time, to check what the difference is between our current codebase and what's live on the servers. This will come in handy in a few chapters, when we look at database migrations. Have a look at the tag in the history: [subs="specialcharacters,quotes"] ---- $ *git log --graph --oneline --decorate* * 1d4d814 (HEAD -> main) Save secret key, set env vars, mount db, run migrations. It works :) * 95e0fe0 Build our image, use export/import to get it on the server, try and run it * 5a36957 Made a start on an ansible playbook for deployment [...] ---- NOTE: Once again, this use of Git tags isn't meant to be the _one true way_. We just need some sort of way to keep track of what was deployed when. === Tell Everyone! You now have a live website! Tell all your friends! Tell your mum, if no one else is interested! Or, tell me! I'm always delighted to see a new reader's site: obeythetestinggoat@gmail.com! Congratulations again for getting through this block of deployment chapters; I know they can be challenging. I hope you got something out of them—seeing a practical example of how to take these kinds of complex changes and break them down into small, incremental steps, getting frequent feedback from our tests and manual investigations along the way. NOTE: Our next deploy won't be until <>, so you can switch off your servers until then if you want to. If you're using a platform where you only get one month of free hosting, it might run out by then. You might have to shell out a few bucks, or see if there's some way of getting another free month. In the next chapter, it's back to coding again. ((("", startref="Fstage11"))) === Further Reading ((("automated deployment", "additional resources"))) There's no such thing as the _one true way_ in deployment; I've tried to set you off on a reasonably sane path, but there are plenty of things you could do differently—and lots, _lots_ more to learn besides. Here are some resources I used for inspiration, (including a couple I've already mentioned): * The original https://12factor.net[Twelve-Factor App] manifesto from the Heroku team * The official Django docs' https://docs.djangoproject.com/en/5.2/howto/deployment/checklist[Deployment Checklist] * https://oreil.ly/SPDMv["How to Write Deployment-friendly Applications"] by Hynek Schlawack * The deployment chapter of https://oreil.ly/x7PoY[_Two Scoops of Django_] by Daniel and Audrey Roy Greenfield * The PythonSpeed https://pythonspeed.com/docker["Docker packaging for Python developers"] guide .Automated Deployment and IaC Recap ******************************************************************************* Here's a brief recap of ((("infrastructure as code (IaC)", "recap of IaC and automated deployment")))what we've been through, which are a fairly typical set of steps for deployment in general: Provisioning a server:: This tends to be vendor-specific, so we didn't automate it, but you absolutely can! Installing system dependencies:: In our case, it was mainly Docker. But inside the Docker image, we also had some system dependencies too, like Python itself. The installation of both types of dependencies is now automated, and now defined "in code", whether it's the Dockerfile or the Ansible YAML. Getting our application code (or "artifacts") onto the server:: In our case, because we're using Docker, the thing we needed to transfer was a Docker image. Typically, you would do this by pushing and pulling from an image repository—although in our automation, we used a more direct process, purely to avoid endorsing any particular vendor. Setting environment variables and secrets:: Depending on how you need to vary them, you can set environment variables on your local PC, in a Dockerfile, in your Ansible scripts, or on the server itself. Figuring out which to use in which case is a big part of deployment. Attaching to the database:: In our case, we mount a file from the local filesystem. More typically, you'd be supplying some environment variables and secrets to define a host, port, username, and password to use for accessing a database server. Configuring networking and port mapping:: This includes DNS config, as well as Docker configuration. Web apps need to be able to talk to the outside world! Running database migrations:: We'll revisit this later in the book, but migrations are one of the most risky parts of a deployment, and automating them is a key part of reducing that risk. Going live with the new version of our application:: In our case, we stop the old container and start a new one. In more advanced setups, you might be trying to achieve zero downtime deploys, and looking into techniques like blue/green deployments, but those are topics for different books. Every single aspect of deployment can and probably should be automated. Here are a couple of general principles to think about when implementing IaC: Idempotence:: If your deployment script is deploying to existing servers, you need to design them so that they work against a fresh installation _and_ against a server that's already configured. ((("idempotence"))) Declarative:: As much as possible, we want to try and specify _what_ we want the state to be on the server, rather than _how_ we should get there. This goes hand in hand with the idea of idempotence.((("deployment", "automating with Ansible", startref="ix_dplyautAns")))((("Ansible", "automated deployments with", startref="ix_Ansautd"))) ******************************************************************************* ================================================ FILE: chapter_13_organising_test_files.asciidoc ================================================ [[chapter_13_organising_test_files]] == Splitting Our Tests into Multiple Files, [.keep-together]#and a Generic Wait Helper# Back to local development! The next feature we might like to implement is a little input validation. But as we start writing new tests, we'll notice that it's getting hard to find our way around a single _functional_tests.py_, and _tests.py_, so we'll reorganise them into multiple files--a little refactor of our tests, if you will. We'll also build a generic explicit wait helper. === Start on a Validation FT: Preventing Blank Items ((("list items", id="list12"))) ((("user interactions", "preventing blank items", id="UIblank12"))) ((("blank items, preventing", id="blank12"))) ((("form data validation", "preventing blank items", id="FDVblank12"))) ((("validation", see="form data validation; model-level validation"))) ((("functional tests (FTs)", "for validation", secondary-sortas="validation", id="FTvalidat12"))) As our first few users start using the site, we've noticed they sometimes make mistakes that mess up their lists, like accidentally submitting blank list items, or inputting two identical items to a list. Computers are meant to help stop us from making silly mistakes, so let's see if we can get our site to help. [role="pagebreak-before"] Here's the outline of the new FT method, which we will add to `NewVisitorTestCase`: [role="sourcecode"] .src/functional_tests/tests.py (ch13l001) ==== [source,python] ---- def test_cannot_add_empty_list_items(self): # Edith goes to the home page and accidentally tries to submit # an empty list item. She hits Enter on the empty input box # The home page refreshes, and there is an error message saying # that list items cannot be blank # She tries again with some text for the item, which now works # Perversely, she now decides to submit a second blank list item # She receives a similar warning on the list page # And she can correct it by filling some text in self.fail("write me!") ---- ==== That's all very well, but before we go any further--our functional tests (FTs) file is beginning to get a little crowded. Let's split it out into several files, in which each has a single test method. Remember that FTs are closely linked to "user stories" and features. One way of organising your FTs might be to have one per high-level feature. We'll also have one base test class, which they can all inherit from. Here's how to get there step by step. ==== Skipping a Test NOTE: We're back to local development now. Make sure that the `TEST_SERVER` environment variable is unset by executing the command `unset TEST_SERVER` from the terminal. ((("unittest module", "skip test decorator"))) ((("refactoring"))) ((("decorators", "skip test decorator"))) It's always nice, when refactoring, to have a fully passing test suite. We've just written a test with a deliberate failure.((("skip test decorator"))) Let's temporarily switch it off, using a decorator called "skip" from `unittest`: [role="sourcecode"] .src/functional_tests/tests.py (ch13l001-1) ==== [source,python] ---- from unittest import skip [...] @skip def test_cannot_add_empty_list_items(self): ---- ==== This tells the test runner to ignore this test. You can see it works--if we rerun the tests, you'll see it's a pass, but it explicitly mentions the skipped test: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test functional_tests* [...] Ran 4 tests in 11.577s OK (skipped=1) ---- WARNING: Skips are dangerous--you need to remember to remove them before you commit your changes back to the repo. This is why line-by-line reviews of each of your diffs are a good idea! .Don't Forget the "Refactor" in "Red/Green/Refactor" ********************************************************************** ((("Test-Driven Development (TDD)", "concepts", "Red/Green/Refactor"))) ((("Red/Green/Refactor"))) A criticism that's sometimes levelled at TDD is that it leads to badly architected code, as the developer just focuses on getting tests to pass rather than stopping to think about how the whole system should be designed. I think it's slightly unfair. _TDD is no silver bullet_. You still have to spend time thinking about good design. But what often happens is that people forget the "refactor" in "red/green/refactor".((("refactoring", "red/green/refactor"))) The methodology allows you to throw together any old code to get your tests to pass, but it _also_ asks you to then spend some time refactoring it to improve its design. Otherwise, it's too easy to allow https://oreil.ly/57WKw["technical debt"] to build up. Often, however, the best ideas for how to refactor code don't occur to you straight away. They may occur to you days, weeks, even months after you wrote a piece of code, when you're working on something totally unrelated and you happen to see some old code again with fresh eyes. But if you're halfway through something else, should you stop to refactor the old code? The answer is that it depends. In the case at the beginning of the chapter, we haven't even started writing our new code. We know we are in a working state, so we can justify putting a skip on our new FT (to get back to fully passing tests) and do a bit of refactoring straight away. Later in the chapter, we'll spot other bits of code we want to alter. In those cases, rather than taking the risk of refactoring an application that's not in a working state, we'll make a note of the thing we want to change on our scratchpad and wait until we're back to a fully passing test suite before refactoring. Kent Beck has a book-length exploration of the trade-offs of refactor-now versus refactor-later, called pass:[Tidy First?]. ********************************************************************** === Splitting Functional Tests Out into Many Files ((("functional tests (FTs)", "splitting into many files", id="FTsplit12"))) ((("test files", "splitting FTs into many", id="ix_tstfispl"))) We start putting each test into its own class, still in the same file: [role="sourcecode"] .src/functional_tests/tests.py (ch13l002) ==== [source,python] ---- class FunctionalTest(StaticLiveServerTestCase): def setUp(self): [...] def tearDown(self): [...] def wait_for_row_in_list_table(self, row_text): [...] class NewVisitorTest(FunctionalTest): def test_can_start_a_todo_list(self): [...] def test_multiple_users_can_start_lists_at_different_urls(self): [...] class LayoutAndStylingTest(FunctionalTest): def test_layout_and_styling(self): [...] class ItemValidationTest(FunctionalTest): @skip def test_cannot_add_empty_list_items(self): [...] ---- ==== At this point, we can rerun the FTs and see they all still work: ---- Ran 4 tests in 11.577s OK (skipped=1) ---- That's labouring it a little bit, and we could probably get away with doing this stuff in fewer steps, but—as I keep saying—practising the step-by-step method on the easy cases makes it that much easier when we have a complex case. Now we switch from a single tests file to using one for each class, and one "base" file to contain the base class that all the tests will inherit from. We'll make four copies of 'tests.py', naming them appropriately, and then delete the parts we don't need from each: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *git mv src/functional_tests/tests.py src/functional_tests/base.py* $ *cp src/functional_tests/base.py src/functional_tests/test_simple_list_creation.py* $ *cp src/functional_tests/base.py src/functional_tests/test_layout_and_styling.py* $ *cp src/functional_tests/base.py src/functional_tests/test_list_item_validation.py* ---- _base.py_ can be cut down to just the `FunctionalTest` class. We leave the helper method on the base class, because we suspect we're about to reuse it in our new FT: [role="sourcecode"] .src/functional_tests/base.py (ch13l003) ==== [source,python] ---- import os import time from django.contrib.staticfiles.testing import StaticLiveServerTestCase from selenium import webdriver from selenium.common.exceptions import WebDriverException from selenium.webdriver.common.by import By MAX_WAIT = 5 class FunctionalTest(StaticLiveServerTestCase): def setUp(self): [...] def tearDown(self): [...] def wait_for_row_in_list_table(self, row_text): [...] ---- ==== NOTE: Keeping helper methods in a base `FunctionalTest` class is one useful way of preventing duplication in FTs. Later in the book (in <>), we'll use the "page pattern", which is related, but prefers composition over inheritance--always a good thing. Our first FT is now in its own file, and should be just one class and one test method: [role="sourcecode"] .src/functional_tests/test_simple_list_creation.py (ch13l004) ==== [source,python] ---- from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from .base import FunctionalTest class NewVisitorTest(FunctionalTest): def test_can_start_a_todo_list(self): [...] def test_multiple_users_can_start_lists_at_different_urls(self): [...] ---- ==== I used a relative import (`from .base`). Some people like to use them a lot in Django code (e.g., your views might import models using `from .models import List`, instead of `from list.models`). Ultimately, this is a matter of personal preference.((("imports, relative, in Django")))((("relative imports"))) I prefer to use relative imports only when I'm super, super confident that the relative position of the thing I'm importing won't change. That applies in this case because I know for sure that all the tests will sit next to _base.py_, which they inherit from. The layout and styling FT should now be one file and one class: [role="sourcecode"] .src/functional_tests/test_layout_and_styling.py (ch13l005) ==== [source,python] ---- from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from .base import FunctionalTest class LayoutAndStylingTest(FunctionalTest): [...] ---- ==== Lastly, our new validation test is in a file of its own too: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch13l006) ==== [source,python] ---- from unittest import skip from selenium.webdriver.common.by import By # <1> from selenium.webdriver.common.keys import Keys # <1> from .base import FunctionalTest class ItemValidationTest(FunctionalTest): @skip def test_cannot_add_empty_list_items(self): [...] ---- ==== <1> These two will be marked as "unused imports" for now but that's OK; we'll use them shortly. And we can test that everything worked by rerunning `manage.py test functional_tests`, and checking once again that all four tests are run: ---- Ran 4 tests in 11.577s OK (skipped=1) ---- ((("test files", "splitting FTs into many", startref="ix_tstfispl")))((("", startref="FTsplit12")))Now we can remove our skip: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch13l007) ==== [source,python] ---- class ItemValidationTest(FunctionalTest): def test_cannot_add_empty_list_items(self): [...] ---- ==== === Running a Single Test File ((("functional tests (FTs)", "running single test files"))) ((("test files", "running single"))) As a side bonus, we're now able to run an individual test file, like this: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test functional_tests.test_list_item_validation* [...] AssertionError: write me! ---- Brilliant--no need to sit around waiting for all the FTs when we're only interested in a single one. Although, we need to remember to run all of them now and again to check for regressions. Later in the book, we'll set up a continuous integration (CI) server to run all the tests automatically—for example, every time we push to the main branch. For now, a good prompt for running all the tests is "just before you do a commit", so let's get into that habit now: [subs="specialcharacters,quotes"] ---- $ *git status* $ *git add src/functional_tests* $ *git commit -m "Moved FTs into their own individual files"* ---- Great. We've split our FTs nicely out into different files. Next, we'll start writing our FT. But before long, as you may be guessing, we'll do something similar to our unit test files. ((("", startref="list12"))) ((("", startref="blank12"))) ((("", startref="UIblank12"))) ((("", startref="FDVblank12"))) ((("", startref="FTvalidat12"))) === A New FT Tool: A Generic Explicit Wait Helper ((("waits", "generic explicit wait helper", id="ix_waithlp")))((("implicit and explicit waits"))) ((("explicit and implicit waits"))) ((("functional tests (FTs)", "implicit/explicit waits and time.sleeps"))) ((("generic explicit wait helper", id="gewhelper12"))) First, let's start implementing the test—or at least the beginning of it: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch13l008) ==== [source,python] ---- def test_cannot_add_empty_list_items(self): # Edith goes to the home page and accidentally tries to submit # an empty list item. She hits Enter on the empty input box self.browser.get(self.live_server_url) self.browser.find_element(By.ID, "id_new_item").send_keys(Keys.ENTER) # The home page refreshes, and there is an error message saying # that list items cannot be blank self.assertEqual( self.browser.find_element(By.CSS_SELECTOR, ".invalid-feedback").text, #<1> "You can't have an empty list item", #<2> ) # She tries again with some text for the item, which now works self.fail("finish this test!") [...] ---- ==== [role="pagebreak-before"] This is how we might write the test naively: <1> We specify we're going to use a CSS class called `.invalid-feedback` to mark our error text. We'll see that Bootstrap has some useful styling for those. <2> And we can check that our error displays the message we want. But can you guess what the potential problem is with the test as it's written now? OK, I gave it away in the section header, but whenever we do something that causes a page refresh, we need an explicit wait; otherwise, Selenium might go looking for the `.invalid-feedback` element before the page has had a chance to load. TIP: Whenever you submit a form with `Keys.ENTER` or click something that is going to cause a page to load, you probably want an explicit wait for your next assertion. Our first explicit wait was built into a helper method. For this one, we might decide that building a specific helper method is overkill at this stage, but it might be nice to have some generic way of saying in our tests, "wait until this assertion passes". ((("assertions", "wrapping in lambda function and passing to wait helper")))((("lambda functions", "wrapping assertion in and passing to wait helper")))Something like this: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch13l009) ==== [source,python] ---- [...] # The home page refreshes, and there is an error message saying # that list items cannot be blank self.wait_for( lambda: self.assertEqual( #<1> self.browser.find_element(By.CSS_SELECTOR, ".invalid-feedback").text, "You can't have an empty list item", ) ) ---- ==== <1> Rather than calling the assertion directly, we wrap it in a `lambda` function, and we pass it to a new helper method we imagine called `wait_for`. NOTE: If you've never seen `lambda` functions in Python before, see <>. [role="pagebreak-before"] So, how would this magical `wait_for` method work? Let's head over to _base.py_, make a copy of our existing `wait_for_row_in_list_table` method, and we'll adapt it slightly: [role="sourcecode"] .src/functional_tests/base.py (ch13l010) ==== [source,python] ---- def wait_for(self, fn): #<1> start_time = time.time() while True: try: table = self.browser.find_element(By.ID, "id_list_table") #<2> rows = table.find_element(By.TAG_NAME, "tr") self.assertIn(row_text, [row.text for row in rows]) return except (AssertionError, WebDriverException): if time.time() - start_time > MAX_WAIT: raise time.sleep(0.5) ---- ==== <1> We make a copy of the method, but we name it `wait_for`, and we change its argument. It is expecting to be passed a function. <2> For now, we've still got the old code that's checking table rows. Now, how do we transform this into something that works for any generic `fn` that's been passed in? Like this: [[self.wait-for]] [role="sourcecode"] .src/functional_tests/base.py (ch13l011) ==== [source,python] ---- def wait_for(self, fn): start_time = time.time() while True: try: return fn() #<1> except (AssertionError, WebDriverException): if time.time() - start_time > MAX_WAIT: raise time.sleep(0.5) ---- ==== <1> The body of our `try/except`, instead of being the specific code for examining table rows, just becomes a call to the function we passed in. We also `return` its result, to be able to exit the loop immediately if no exception is raised. [role="pagebreak-before less_space"] [[lamdbafunct]] .Lambda Functions ******************************************************************************* ((("lambda functions"))) ((("Python 3", "lambda functions"))) `lambda` in Python is the syntax for making a one-line, throwaway function. It saves you from having to use `def...():` and an indented block: [role="skipme"] [source,python] ---- >>> myfn = lambda x: x+1 >>> myfn(2) 3 >>> myfn(5) 6 >>> adder = lambda x, y: x + y >>> adder(3, 2) 5 ---- In our case, we're using it to transform a bit of code, that would otherwise be executed immediately, into a function that we can pass as an argument, and that can be executed later, and multiple times: [role="skipme"] [source,python] ---- >>> def addthree(x): ... return x + 3 ... >>> addthree(2) 5 >>> myfn = lambda: addthree(2) # note addthree isn't called immediately here >>> myfn at 0x7f3b140339d8> >>> myfn() 5 >>> myfn() 5 ---- ******************************************************************************* Let's see our funky `wait_for` helper in action: [subs="macros,verbatim"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_list_item_validation*] [...] ====================================================================== ERROR: test_cannot_add_empty_list_items (functional_tests.test_list_item_valida tion.ItemValidationTest.test_cannot_add_empty_list_items) --------------------------------------------------------------------- [...] Traceback (most recent call last): File "...goat-book/src/functional_tests/test_list_item_validation.py", line 16, in test_cannot_add_empty_list_items self.wait_for(<1> File "...goat-book/src/functional_tests/base.py", line 25, in wait_for return fn()<2> ^^^^ File "...goat-book/src/functional_tests/test_list_item_validation.py", line 18, in <3> self.browser.find_element(By.CSS_SELECTOR, ".invalid-feedback").text,<3> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .invalid-feedback; [...] --------------------------------------------------------------------- Ran 1 test in 10.575s FAILED (errors=1) ---- The order of the traceback is a little confusing, but we can more or less follow through what happened: <1> In our FT, we call our `self.wait_for` helper, where we pass the `lambda`-ified version of `assertEqual`. <2> We go into `self.wait_for` in _base.py_, where we're calling (and returning) `fn()`, which refers to the passed `lambda` function encapsulating our test assertion. <3> To explain where the exception has actually come from, the traceback takes us back into _test_list_item_validation.py_ and inside the body of the `lambda` function, and tells us that it was attempting to find the `.invalid-feedback` element that failed. ((("functional programming"))) We're into the realm of functional programming now, passing functions as arguments to other functions, and it can be a little mind-bending. I know it took me a little while to get used to! Have a couple of read-throughs of this code, and the code back in the FT, to let it sink in; and if you're still confused, don't worry about it too much, and let your confidence grow from working with it. We'll use it a few more times in this book and make it even more functionally fun; you'll see. ((("", startref="gewhelper12"))) === Finishing Off the FT We'll finish off the FT like this: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch13l012) ==== [source,python] ---- # The home page refreshes, and there is an error message saying # that list items cannot be blank self.wait_for( lambda: self.assertEqual( self.browser.find_element(By.CSS_SELECTOR, ".invalid-feedback").text, "You can't have an empty list item", ) ) # She tries again with some text for the item, which now works self.browser.find_element(By.ID, "id_new_item").send_keys("Purchase milk") self.browser.find_element(By.ID, "id_new_item").send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Purchase milk") # Perversely, she now decides to submit a second blank list item self.browser.find_element(By.ID, "id_new_item").send_keys(Keys.ENTER) # She receives a similar warning on the list page self.wait_for( lambda: self.assertEqual( self.browser.find_element(By.CSS_SELECTOR, ".invalid-feedback").text, "You can't have an empty list item", ) ) # And she can correct it by filling some text in self.browser.find_element(By.ID, "id_new_item").send_keys("Make tea") self.browser.find_element(By.ID, "id_new_item").send_keys(Keys.ENTER) self.wait_for_row_in_list_table("2: Make tea") ---- ==== .Helper Methods in FTs ******************************************************************************* ((("functional tests (FTs)", "helper methods in"))) ((("helper methods"))) ((("self.wait_for helper method"))) ((("wait_for_row_in_list_table helper method"))) We've got two helper methods now: our generic `self.wait_for` helper, and `wait_for_row_in_list_table`. The former is a general utility--any of our FTs might need to do a wait. The latter also helps prevent duplication across your FT code. The day we decide to change the implementation of how our list table works, we want to make sure we only have to change our FT code in one place, not in dozens of places across loads of FTs...((("waits", "generic explicit wait helper", startref="ix_waithlp"))) See also <> and https://www.obeythetestinggoat.com/book/appendix_bdd.html[Online Appendix: BDD] for more on structuring your FT code. ******************************************************************************* I'll let you do your own "first-cut FT" commit. === Refactoring Unit Tests into Several Files ((("unit tests", "refactoring into several files"))) ((("refactoring", "of unit tests into several files", secondary-sortas="unit"))) ((("test files", "splitting unit tests into several"))) When we (finally!) start coding our solution, we're going to want to add another test for our _models.py_. Before we do so, it's time to tidy up our unit tests in a similar way to the functional tests. A difference will be that, because the `lists` app contains real application code as well as tests, we'll separate out the tests into their own folder: [subs=""] ---- $ mkdir src/lists/tests $ touch src/lists/tests/__init__.py $ git mv src/lists/tests.py src/lists/tests/test_all.py $ git status $ git add src/lists/tests $ python src/manage.py test lists [...] Ran 10 tests in 0.034s OK $ git commit -m "Move unit tests into a folder with single file" ---- If you get((("dunderinit")))((("__init__", primary-sortas="init"))) a message saying "Ran 0 tests", you probably forgot to add the dunderinit.footnote:[ "Dunder" is shorthand for double-underscore, so "dunderinit" means +++__init__.py+++.] It needs to be there for the tests folder to be recognised as a regular Python package,footnote:[ Without the dunderinit, a folder with Python files in it is called a https://oreil.ly/V-w3A[namespace package]. Usually, they are exactly the same as regular packages (which _do_ have a +++__init__.py+++), but the Django test runner does not recognise them.] and thus discovered by the test runner. Now we turn _test_all.py_ into two files—one called _test_views.py_, which will only contain view tests, and one called _test_models.py_. I'll start by making two copies: [subs="specialcharacters,quotes"] ---- $ *git mv src/lists/tests/test_all.py src/lists/tests/test_views.py* $ *cp src/lists/tests/test_views.py src/lists/tests/test_models.py* ---- [role="pagebreak-before"] And strip _test_models.py_ down to being just the one test: [role="sourcecode"] .src/lists/tests/test_models.py (ch13l016) ==== [source,python] ---- from django.test import TestCase from lists.models import Item, List class ListAndItemModelsTest(TestCase): [...] ---- ==== Whereas _test_views.py_ just loses one class: [role="sourcecode"] .src/lists/tests/test_views.py (ch13l017) ==== [source,diff] ---- --- a/src/lists/tests/test_views.py +++ b/src/lists/tests/test_views.py 33 +74,3 @@ class NewItemTest(TestCase): ) self.assertRedirects(response, f"/lists/{correct_list.id}/") - - -class ListAndItemModelsTest(TestCase): - def test_saving_and_retrieving_items(self): [...] ---- ==== We rerun the tests to check that everything is still there: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test lists* [...] Ran 10 tests in 0.040s OK ---- Great! That's another small, working step: [subs="specialcharacters,quotes"] ---- $ *git add src/lists/tests* $ *git commit -m "Split out unit tests into two files"* ---- NOTE: Some people like to make their unit tests into a tests folder straight away, as soon as they start a project. That's a perfectly good idea; I just thought I'd wait until it became necessary, to avoid doing too much housekeeping all in the first chapter! Well, that's our FTs and unit tests nicely reorganised. In the next chapter, we'll get down to some validation proper. [role="pagebreak-before less_space"] .Tips on Organising Tests and Refactoring ******************************************************************************* Use a tests folder:: Just as you use multiple files to hold your application code, you should split your tests out into multiple files: * For functional tests, group them into tests for a particular feature or user story. * For unit tests, use a folder called 'tests', with a +++__init__.py+++. * You probably want a separate test file for each tested source code file. For Django, that's typically 'test_models.py', 'test_views.py', and 'test_forms.py'.((("__init__", primary-sortas="init")))((("dunderinit"))) * Have at least a placeholder test for 'every' function and class. ((("test files", "organizing and refactoring"))) Don't forget the "refactor" in "red/green/refactor":: The whole point of having tests is to allow you to refactor your code! Use them, and make your code (including your tests) as clean as you can. ((("Test-Driven Development (TDD)", "concepts", "Red/Green/Refactor"))) ((("Red/Green/Refactor"))) Don't refactor against failing tests:: * The general rule is that you shouldn't mix refactoring and behaviour change. Having green tests is our best guarantee that we aren't changing behaviour. If you start refactoring against failing tests, it becomes much harder to spot when you're accidentally introducing a regression. * This applies strongly to unit tests. With FTs, because we often develop against red FTs anyway, it's sometimes more tempting to refactor against failing tests. My suggestion is to avoid that temptation and use an early return, so that it's 100% clear if, during a refactor, you accidentally introduce a regression that's picked up in your FTs. * You can occasionally put a skip on a test that is testing something you haven't written yet. * More commonly, make a note of the refactor you want to do, finish what you're working on, and do the refactor a little later when you're back to a working state. * Don't forget to remove any skips before you commit your code! You should always review your diffs line by line to catch things like this. ((("refactoring"))) Try a generic wait_for helper:: Having specific helper methods that do explicit waits is great, and it helps to make your tests readable. But you'll also often need an ad-hoc, one-line assertion or Selenium interaction that you'll want to add a wait to. `self.wait_for` does the job well for me, but you might find a slightly different pattern works for you. ((("generic explicit wait helper"))) ((("wait_for helper method"))) ((("self.wait_for helper method"))) ******************************************************************************* ================================================ FILE: chapter_14_database_layer_validation.asciidoc ================================================ [[chapter_14_database_layer_validation]] == Validation at the Database Layer ((("user interactions", "validating inputs at database layer", id="UIdblayer13"))) ((("database testing", "database-layer validation", id="DBTdblayer13"))) Over the next few chapters, we'll talk about testing and implementing validation of user inputs.((("validation", "database layer", id="ix_valDB"))) In terms of content, there's going to be quite a lot of material here that's more about the specifics of Django, and less discussion of TDD philosophy. That doesn't mean you won't be learning anything about testing--there are plenty of little testing tidbits in here, but perhaps it's more about really getting into the swing of things, the rhythm of TDD, and how we get work done. Once we get through these three short chapters, I've saved a bit of fun with JavaScript (!) for the end of <>. Then it's on to <>, where I promise we'll get right back into some of the real nitty-gritty discussions on TDD methodology--unit tests versus integration tests, mocking, and more. Stay tuned! [role="pagebreak-before"] But for now, a little validation. Let's just remind ourselves where our FT is pointing us: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python3 src/manage.py test functional_tests.test_list_item_validation*] [...] ====================================================================== ERROR: test_cannot_add_empty_list_items (functional_tests.test_list_item_valida tion.ItemValidationTest.test_cannot_add_empty_list_items) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/functional_tests/test_list_item_validation.py", line 16, in test_cannot_add_empty_list_items self.wait_for( ~~~~~~~~~~~~~^ lambda: self.assertEqual( ^^^^^^^^^^^^^^^^^^^^^^^^^ ...<2 lines>... ) ^ ) ^ [...] File "...goat-book/src/functional_tests/test_list_item_validation.py", line 18, in self.browser.find_element(By.CSS_SELECTOR, ".invalid-feedback").text, ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .invalid-feedback; For documentation [...] ---- It's expecting to see an error message if the user tries to input an empty item. === Model-Layer Validation ((("model-layer validation", "benefits and drawbacks of"))) In a web app, there are two places you can do validation: on the client side (using JavaScript or HTML5 properties, as we'll see later), and on the server side. The server side is "safer" because someone can always bypass the client side, whether it's maliciously or due to some bug. Similarly on the server side, in Django, there are two levels at which you can do validation.((("Django framework", "validation, layers of"))) One is at the model level, and the other is higher up at the forms level. I like to use the lower level whenever possible, partially because I'm a bit too fond of databases and database integrity rules, and partially because, again, it's safer--you can sometimes forget which form you use to validate input, but you're always going to use the same database. [role="pagebreak-before less_space"] ==== The self.assertRaises Context Manager ((("model-layer validation", "self.assertRaises context manager"))) ((("self.assertRaises context manager"))) Let's go down and write a unit test at the models layer. Add a new test method to [.keep-together]#+ListAndItemModelsTest+#, which tries to create a blank list item. This test is interesting because it's testing that the code under test should raise an exception: [role="sourcecode"] .src/lists/tests/test_models.py (ch14l001) ==== [source,python] ---- from django.db.utils import IntegrityError [...] class ListAndItemModelsTest(TestCase): def test_saving_and_retrieving_items(self): [...] def test_cannot_save_empty_list_items(self): mylist = List.objects.create() item = Item(list=mylist, text="") with self.assertRaises(IntegrityError): item.save() ---- ==== This is a new unit testing technique: when we want to check that doing something will raise an error, we can use the `self.assertRaises` context manager. We could have used something like this instead: [role="skipme"] [source,python] ---- try: item.save() self.fail('The save should have raised an exception') except IntegrityError: pass ---- But the `with` formulation is neater. TIP: If you're new to Python, you may never have seen the `with` statement. It's the special keyword to use with what are called "context managers". Together, they wrap a block of code, usually with some kind of setup, cleanup, or error-handling code. There's a good write-up on https://oreil.ly/z6Eh8[Python Morsels]. ((("with statements"))) ((("Python 3", "with statements"))) ==== Django Model Constraints and Their Interaction with Databases When we run this new unit test, we see((("model-layer validation", "Django model constraints and database interactions")))((("IntegrityErrors")))((("Django framework", "model constraints and interaction with databases"))) the failure we expected: ---- with self.assertRaises(IntegrityError): AssertionError: IntegrityError not raised ---- But all is not quite as it seems, because _this test should already pass_. If you take a look at the https://docs.djangoproject.com/en/5.2/ref/models/fields/#blank[docs for the Django model fields], you'll see under "Field choices" that the default setting for _all_ fields is `blank=False`. Because "text field" is a type of field, it _should_ already disallow empty values. ((("data integrity errors"))) So, why is the test still failing? Why is our database not raising an `IntegrityError` when we try to save an empty string into the `text` column? The answer is a combination of Django's design and the database we're using. ==== Inspecting Our Constraints at the Database Level Let's have a look directly((("model-layer validation", "inspecting constraints at database layer"))) at the database using the `dbshell` command: [role="skipme small-code"] [subs="specialcharacters,quotes"] ---- $ *./src/manage.py dbshell* # (this is equivalent to running sqlite3 src/db.sqlite3) SQLite version 3.[...] Enter ".help" for usage hints. sqlite> *.schema lists_item* CREATE TABLE IF NOT EXISTS "lists_item" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "text" text NOT NULL, "list_id" bigint NOT NULL REFERENCES "lists_list" ("id") DEFERRABLE INITIALLY DEFERRED); ---- The `text` column only has the `NOT NULL` constraint. This means that the database would not allow `None` as a value, but it will actually allow the empty string. Whilst it is https://oreil.ly/kzu65[technically possible] to implement a "not empty string" constraint on a text column in SQLite, the Django developers have chosen not to do this. This is because Django distinguishes between what they call "database-related" and "validation-related" constraints.((("constraints", "database-related and validation-related"))) As well as `empty=False`, all fields get a `null=False` setting, which translates into the database-level `NOT NULL` constraint we saw earlier. Let's see if we can verify that using our test, instead. We'll pass in `text=None` instead of `text=""` (and change the test name): [role="sourcecode"] .src/lists/tests/test_models.py (ch14l002) ==== [source,python] ---- def test_cannot_save_null_list_items(self): mylist = List.objects.create() item = Item(list=mylist, text=None) with self.assertRaises(IntegrityError): item.save() ---- ==== You'll see that _this_ test now passes: ---- Ran 11 tests in 0.030s OK ---- ==== Testing Django Model Validation That's all vaguely interesting, but it's not actually what we set out to do.((("model-layer validation", "testing Django model validation"))) How do we make sure that the "validation-related" constraint is being enforced?((("ValidationErrors"))) The answer is that, while `IntegrityError` comes from the database, Django uses `ValidationError` to signal errors that come from its own validation. Let's write a second test that checks on that: [role="sourcecode"] .src/lists/tests/test_models.py (ch14l003) ==== [source,python] ---- from django.core.exceptions import ValidationError from django.db.utils import IntegrityError [...] class ListAndItemModelsTest(TestCase): def test_saving_and_retrieving_items(self): [...] def test_cannot_save_null_list_items(self): mylist = List.objects.create() item = Item(list=mylist, text=None) with self.assertRaises(IntegrityError): item.save() def test_cannot_save_empty_list_items(self): mylist = List.objects.create() item = Item(list=mylist, text="") # <1> with self.assertRaises(ValidationError): # <2> item.save() ---- ==== <1> This time we pass `text=""`. <2> And we're expecting a `ValidationError` instead of an `IntegrityError`. ==== A Django Quirk: Model Save Doesn't Run Validation We can try running this new unit test, and we'll see its expected failure... ---- with self.assertRaises(ValidationError): AssertionError: ValidationError not raised ---- Wait a minute! We expected this to _pass_ actually! We just got through learning that Django should be enforcing the `blank=False` constraint by default.((("Django framework", "models not running full validation on save"))) Why doesn't this work? ((("model-layer validation", "running full validation"))) We've discovered one of Django's little quirks. For https://oreil.ly/u3N_2[slightly counterintuitive historical reasons], Django models don't run full validation on save. ((("full_clean method"))) Django does have a method to manually run full validation, however, called `full_clean` (more info in https://docs.djangoproject.com/en/5.2/ref/models/instances/#django.db.models.Model.full_clean[the docs]). Let's swap that for the `.save()` and see if it works: [role="sourcecode"] .src/lists/tests/test_models.py (ch14l004) ==== [source,python] ---- with self.assertRaises(ValidationError): item.full_clean() ---- ==== That gets the unit test to pass: ---- Ran 12 tests in 0.030s OK ---- Good. That taught us a little about Django validation, and the test is there to warn us if we ever forget our requirement and set `blank=True` on the `text` field (try it!). .Recap: Database-level and Model-level Validation in Django ********************************************************************** Django distinguishes two types of validation for models: 1. Database-level constraints like `null=False` or `unique=True` (as we'll see an example of in <>), which are enforced by the database itself, using things like `NOT NULL` or `UNIQUE` constraints and bubble up as ++IntegrityError++s if you try to save an invalid object 2. Model-level validations like `blank=False`, which are only enforced by Django, when you call `full_clean()`, and they raise a `ValidationError` The subtlety is that Django also enforces database-level constraints when you call `full_clean()`. So, you'll only see `IntegrityError` if you forget to call `full_clean()` before doing a `.save()`. ********************************************************************** The FTs are still failing, because we're not actually forcing these errors to appear in our actual app, outside of this one unit test. [role="pagebreak-before less_space"] === Surfacing Model Validation Errors in the View ((("model-layer validation", "surfacing errors in the view", id="MLVsurfac13"))) Let's try to enforce our model validation in the views layer and bring it up into our templates so the user can see them. To optionally display an error in our HTML, we check whether the template has been passed an error variable and, if so, we do this: [role="sourcecode"] .src/lists/templates/base.html (ch14l005) ==== [source,html] ---- name="item_text" id="id_new_item" placeholder="Enter a to-do item" /> {% csrf_token %} {% if error %}
{{ error }}
<2> {% endif %} ---- ==== <1> We add the `.is-invalid` class to any form inputs that have validation errors. <2> We use a `div.invalid-feedback` to display any error messages from the server. ((("Bootstrap", "documentation"))) ((("form control classes (Bootstrap)"))) Take a look at the https://getbootstrap.com/docs/5.3/forms/validation/#server-side[Bootstrap docs] for more info on form controls. TIP: However, ignore the Bootstrap docs' advice to prefer client-side validation.((("client-side validation")))((("server-side validation"))) Ideally, having both server- and client-side validation is the best. If you can't do both, then server-side validation is the one you really can't do without. Check the https://oreil.ly/pkFo8[OWASP checklist], if you are not convinced yet. Client-side validation will provide faster feedback on the UI, but https://oreil.ly/xAUt8[it is not a security measure.] Server-side validation is indispensable for handling any input that gets processed by the server--and it will also provide (albeit slower) feedback for the client side. Passing this error to the template is the view function's job. Let's take a look at the unit tests in the `NewListTest` class. I'm going to use two slightly different error-handling patterns here. [role="pagebreak-before"] In the first case, our URL and view for new lists will optionally render the same template as the home page, but with the addition of an error message. Here's a unit test for that: [role="sourcecode"] .src/lists/tests/test_views.py (ch14l006) ==== [source,python] ---- class NewListTest(TestCase): [...] def test_validation_errors_are_sent_back_to_home_page_template(self): response = self.client.post("/lists/new", data={"item_text": ""}) self.assertEqual(response.status_code, 200) self.assertTemplateUsed(response, "home.html") expected_error = "You can't have an empty list item" self.assertContains(response, expected_error) ---- ==== As we're writing this test, we might get slightly offended by the '/lists/new' URL, which we're manually entering as a string. We've got a lot of URLs hardcoded in our tests, in our views, and in our templates, which violates the DRY (don't repeat yourself) principle. I don't mind a bit of duplication in tests, but we should definitely be on the lookout for hardcoded URLs in our views and templates, and make a note to refactor them out. But we won't do that straight away, because right now our application is in a broken state. We want to get back to a working state first. Back to our test, which is failing because the view is currently returning a 302 redirect, rather than a "normal" 200 response: ---- AssertionError: 302 != 200 ---- Let's try calling `full_clean()` in the view: [role="sourcecode"] .src/lists/views.py (ch14l007) ==== [source,python] ---- def new_list(request): nulist = List.objects.create() item = Item.objects.create(text=request.POST["item_text"], list=nulist) item.full_clean() return redirect(f"/lists/{nulist.id}/") ---- ==== //22 As we're looking at the view code, we find a good candidate for a hardcoded URL to get rid of. Let's add that to our scratchpad: [role="scratchpad"] ***** * 'Remove hardcoded URLs from views.py.' ***** Now the model validation raises an exception, which comes up through our view: ---- [...] File "...goat-book/src/lists/views.py", line 13, in new_list item.full_clean() [...] django.core.exceptions.ValidationError: {'text': ['This field cannot be blank.']} ---- So we try our first approach: using a `try/except` to detect errors. Obeying the Testing Goat, we start with just the `try/except` and nothing else. The tests should tell us what to code next. [role="sourcecode"] .src/lists/views.py (ch14l010) ==== [source,python] ---- from django.core.exceptions import ValidationError [...] def new_list(request): nulist = List.objects.create() item = Item.objects.create(text=request.POST["item_text"], list=nulist) try: item.full_clean() except ValidationError: pass return redirect(f"/lists/{nulist.id}/") ---- ==== That gets us back to the `302 != 200`: ---- AssertionError: 302 != 200 ---- Let's return a rendered template then, which should take care of the template check as well: [role="sourcecode"] .src/lists/views.py (ch14l011) ==== [source,python] ---- except ValidationError: return render(request, "home.html") ---- ==== And the tests now tell us to put the error message into the template: ---- AssertionError: False is not true : Couldn't find 'You can't have an empty list item' in the following response ---- We do that by passing a new template variable in: [role="sourcecode"] .src/lists/views.py (ch14l012) ==== [source,python] ---- except ValidationError: error = "You can't have an empty list item" return render(request, "home.html", {"error": error}) ---- ==== Hmm, it looks like that didn't quite work: ---- AssertionError: False is not true : Couldn't find 'You can't have an empty list item' in the following response ---- ((("print", "debugging with"))) ((("debugging", "print-based"))) A little print-based debug... [role="sourcecode"] .src/lists/tests/test_views.py (ch14l013) ==== [source,python] ---- expected_error = "You can't have an empty list item" print(response.content.decode()) self.assertContains(response, expected_error) ---- ==== ...will show us the cause—Django has https://docs.djangoproject.com/en/5.2/ref/templates/builtins/#autoescape[HTML-escaped] the apostrophe: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test lists*] [...]
You can't have an empty list item
---- We could hack something like this into our test: [role="skipme"] [source,python] ---- expected_error = "You can't have an empty list item" ---- But using Django's helper function `html.escape()` is probably a better idea: [role="sourcecode"] .src/lists/tests/test_views.py (ch14l014) ==== [source,python] ---- from django.utils import html [...] expected_error = html.escape("You can't have an empty list item") self.assertContains(response, expected_error) ---- ==== That passes! ---- Ran 13 tests in 0.047s OK ---- ==== Checking That Invalid Input Isn't Saved to the Database ((("invalid input", seealso="model-layer validation"))) ((("database testing", "invalid input"))) Before we go further though, did you notice a little logic error we've allowed to creep into our implementation? We're currently creating an object, even if validation fails: [role="sourcecode currentcontents"] .src/lists/views.py ==== [source,python] ---- item = Item.objects.create(text=request.POST["item_text"], list=nulist) try: item.full_clean() except ValidationError: [...] ---- ==== Let's add a new unit test to make sure that empty list items don't get saved: [role="sourcecode"] .src/lists/tests/test_views.py (ch14l015) ==== [source,python] ---- class NewListTest(TestCase): [...] def test_validation_errors_are_sent_back_to_home_page_template(self): [...] def test_invalid_list_items_arent_saved(self): self.client.post("/lists/new", data={"item_text": ""}) self.assertEqual(List.objects.count(), 0) self.assertEqual(Item.objects.count(), 0) ---- ==== // HARRY: consider assertEqual(Item.objects.all(), [])? dave and csanners tend to agree. That gives: ---- [...] Traceback (most recent call last): File "...goat-book/src/lists/tests/test_views.py", line 43, in test_invalid_list_items_arent_saved self.assertEqual(List.objects.count(), 0) AssertionError: 1 != 0 ---- We fix it like this: [role="sourcecode"] .src/lists/views.py (ch14l016) ==== [source,python] ---- def new_list(request): nulist = List.objects.create() item = Item(text=request.POST["item_text"], list=nulist) try: item.full_clean() item.save() except ValidationError: nulist.delete() error = "You can't have an empty list item" return render(request, "home.html", {"error": error}) return redirect(f"/lists/{nulist.id}/") ---- ==== Do the FTs pass? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_list_item_validation*] [...] File "...goat-book/src/functional_tests/test_list_item_validation.py", line 32, in test_cannot_add_empty_list_items self.wait_for( [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .invalid-feedback; [...] ---- [role="pagebreak-before"] Not quite, but they did get a little further. Checking the line in which the error occurred (line 31 in my case) we can see that we've got past the first part of the test, and are now onto the second check--that submitting a second empty item also shows an error. ((("", startref="MLVsurfac13"))) We've got some working code though, so let's have a commit: [subs="specialcharacters,quotes"] ---- $ *git commit -am "Adjust new list view to do model validation"* ---- ==== Adding an Early Return to Our FT to Let Us Refactor Against Green ((("early return")))((("refactoring", "early return in FT to refactor against green"))) Let's put an early return in the FT to separate what we got working from those that still need to be dealt with: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch14l017) ==== [source,python] ---- class ItemValidationTest(FunctionalTest): def test_cannot_add_empty_list_items(self): [...] self.browser.find_element(By.ID, "id_new_item").send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Purchase milk") return # TODO re-enable the rest of this test. # Perversely, she now decides to submit a second blank list item self.browser.find_element(By.ID, "id_new_item").send_keys(Keys.ENTER) [...] ---- ==== We should also remind ourselves not to forget to remove this early return: [role="scratchpad"] ***** * 'Remove hardcoded URLs from views.py.' * 'Remove the early return from the FT.' ***** And now, we can focus on making our code a little neater. TIP: When working on a new feature, it's common to realise partway through that a refactor of the application is needed. Adding an early return to the FT you're currently working on enables you to perform this refactor against passing FTs, even while the feature is still in progress. === Django Pattern: Processing POST Requests in the Same View That Renders the Form ((("model-layer validation", "POST requests processing", id="MLVpost13"))) ((("POST requests", "Django pattern for processing", id="POSTdjango13"))) ((("HTML", "POST requests", "Django pattern for processing", id="HTMLpostdjango13"))) This time we'll use a slightly different approach—one that's actually a very common pattern in Django—which uses the same view to both process POST requests and render the form that they come from. Whilst this doesn't fit the REST-ful URL model quite as well, it has the important advantage that the same URL can display a form, and display any errors encountered in processing the user's input; see <>. [[single-endpoint-for-forms]] .Existing list, viewing and adding items in the same end point image::images/tdd3_1401.png["A diagram, showing the 3 different user requests for our end point at /lists/list-id/. (1) a GET request, which receives an HTML response containing the list items and the add-item Form. (2) a valid POST request, wich receives a 301 Redirect response to reload /lists/list-id/. (3) an invalid POST request, whose response is HTML including the list and the form, this time with errors"] The current situation is that we have one view and URL for displaying a list, and one view and URL for processing additions to that list. We're going to combine them into one. NOTE: In this section, we're performing a refactor at the application level. We execute our application-level refactor by changing or adding unit tests, and then adjusting our code. We use the functional tests to warn us if we ever go backwards and introduce a regression, and when they're back to green we'll know our refactor is done. Have another look at the diagram from the end of <> if you need to get your bearings. ==== Refactor: Transferring the new_item Functionality into view_list Let's take the two old tests from `NewItemTest`—the ones that are about saving POST requests to existing lists—and move them into `ListViewTest`. As we do so, we also make them point at the base list URL, instead of '.../add_item': [role="sourcecode"] .src/lists/tests/test_views.py (ch14l030) ==== [source,python] ---- class ListViewTest(TestCase): def test_uses_list_template(self): [...] def test_renders_input_form(self): mylist = List.objects.create() response = self.client.get(f"/lists/{mylist.id}/") parsed = lxml.html.fromstring(response.content) [form] = parsed.cssselect("form[method=POST]") self.assertEqual(form.get("action"), f"/lists/{mylist.id}/") # <1> inputs = form.cssselect("input") self.assertIn("item_text", [input.get("name") for input in inputs]) def test_displays_only_items_for_that_list(self): [...] def test_can_save_a_POST_request_to_an_existing_list(self): other_list = List.objects.create() correct_list = List.objects.create() self.client.post( f"/lists/{correct_list.id}/", # <2> data={"item_text": "A new item for an existing list"}, ) self.assertEqual(Item.objects.count(), 1) new_item = Item.objects.get() self.assertEqual(new_item.text, "A new item for an existing list") self.assertEqual(new_item.list, correct_list) def test_POST_redirects_to_list_view(self): other_list = List.objects.create() correct_list = List.objects.create() response = self.client.post( f"/lists/{correct_list.id}/", # <2> data={"item_text": "A new item for an existing list"}, ) self.assertRedirects(response, f"/lists/{correct_list.id}/") ---- ==== <1> We want our form to point at the base URL. <2> And the two tests we've merged in need to target the base URL too. Note that the `NewItemTest` class disappears completely. I've also changed the name of the redirect test to make it explicit that it only applies to POST requests. [role="pagebreak-before"] That gives: ---- FAIL: test_POST_redirects_to_list_view (lists.tests.test_views.ListViewTest.test_POST_redirects_to_list_view) [...] AssertionError: 200 != 302 : Response didn't redirect as expected: Response code was 200 (expected 302) [...] FAIL: test_can_save_a_POST_request_to_an_existing_list (lists.tests.test_views. ListViewTest.test_can_save_a_POST_request_to_an_existing_list) [...] AssertionError: 0 != 1 [...] FAIL: test_renders_input_form (lists.tests.test_views.ListViewTest.test_renders_input_form) [...] AssertionError: '/lists/1/add_item' != '/lists/1/' [...] Ran 14 tests in 0.025s FAILED (failures=3) ---- That last one is something we can fix in the template. Let's go to _list.html_, and change the `action` attribute on our form so that it points at the existing list URL: [role="sourcecode"] .src/lists/templates/list.html (ch14l031) ==== [source,html] ---- {% block form_action %}/lists/{{ list.id }}/{% endblock %} ---- ==== Incidentally, that's another hardcoded URL. Let's add it to our to-do list and, while we're thinking about it, there's one in _home.html_ too: [role="scratchpad"] ***** * 'Remove hardcoded URLs from views.py.' * 'Remove the early return from the FT.' * 'Remove hardcoded URL from forms in list.html and home.html.' ***** //// This will immediately break our original functional test, because the `view_list` page doesn't know how to process POST requests yet: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests*] [...] AssertionError: '2: Use peacock feathers to make a fly' not found in ['1: Buy peacock feathers'] ---- The FTs are warning us that our attempted refactor has introduced a regression. Let's try and finish the refactor as soon as we can, and get back to green. //// [role="pagebreak-before"] We're now down to two failing tests: ---- FAIL: test_POST_redirects_to_list_view (lists.tests.test_views.ListViewTest.test_POST_redirects_to_list_view) [...] AssertionError: 200 != 302 : Response didn't redirect as expected: Response code was 200 (expected 302) [...] FAIL: test_can_save_a_POST_request_to_an_existing_list (lists.tests.test_views. ListViewTest.test_can_save_a_POST_request_to_an_existing_list) [...] AssertionError: 0 != 1 [...] Ran 14 tests in 0.025s FAILED (failures=2) ---- Those are both about getting the list view to handle POST requests. Let's copy some code across from `add_item` view to do just that: [role="sourcecode"] .src/lists/views.py (ch14l032) ==== [source,python] ---- def view_list(request, list_id): our_list = List.objects.get(id=list_id) if request.method == "POST": # <1> Item.objects.create(text=request.POST["item_text"], list=our_list) # <2> return redirect(f"/lists/{our_list.id}/") # <2> return render(request, "list.html", {"list": our_list}) ---- ==== <1> We add a branch for when the method is POST. <2> And we copy the `Item.objects.create()` and `redirect()` lines from the `add_item` view. That gets us passing unit tests! ---- Ran 14 tests in 0.047s OK ---- Now we can delete the `add_item` view, as it's no longer needed...oops, an unexpected failure: [role="dofirst-ch14l033"] ---- [...] AttributeError: module 'lists.views' has no attribute 'add_item' ---- [role="pagebreak-before"] It's because we've deleted the view, but it's still being referred to in _urls.py_. We remove it from there: [role="sourcecode"] .src/lists/urls.py (ch14l034) ==== [source,python] ---- urlpatterns = [ path("new", views.new_list, name="new_list"), path("/", views.view_list, name="view_list"), ] ---- ==== OK, we're back to the green on the unit tests. ---- OK ---- Let's try a full FT run: they're all passing! ---- Ran 4 tests in 9.951s OK ---- Our refactor of the `add_item` functionality is complete. We should commit there: [subs="specialcharacters,quotes"] ---- $ *git commit -am "Refactor list view to handle new item POSTs"* ---- We can remove the((("early return"))) early return now: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch14l035) ==== [source,diff] ---- @@ -24,8 +24,6 @@ class ItemValidationTest(FunctionalTest): self.browser.find_element(By.ID, "id_new_item").send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Purchase milk") - return # TODO re-enable the rest of this test. - # Perversely, she now decides to submit a second blank list item ---- ==== And, let's cross that off our scratchpad too: [role="scratchpad"] ***** * 'Remove hardcoded URLs from views.py.' * '[strikethrough line-through]#Remove the early return from the FT.#' * 'Remove hardcoded URL from forms in list.html and home.html.' ***** [role="pagebreak-before"] Run the FTs again to see what's still there that needs to be fixed: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test functional_tests* [...] ERROR: test_cannot_add_empty_list_items (functional_tests.test_list_item_valida tion.ItemValidationTest.test_cannot_add_empty_list_items) [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .invalid-feedback; [...] Ran 4 tests in 15.276s FAILED (errors=1) ---- We're back to working on this one failure in our new FT. ==== Enforcing Model Validation in view_list We still want the addition of items to existing lists to be subject to our model validation rules. Let's write a new unit test for that; it's very similar to the one for the home page, with just a couple of tweaks: [role="sourcecode"] .src/lists/tests/test_views.py (ch14l036) ==== [source,python] ---- class ListViewTest(TestCase): [...] def test_validation_errors_end_up_on_lists_page(self): list_ = List.objects.create() response = self.client.post( f"/lists/{list_.id}/", data={"item_text": ""}, ) self.assertEqual(response.status_code, 200) self.assertTemplateUsed(response, "list.html") expected_error = html.escape("You can't have an empty list item") self.assertContains(response, expected_error) ---- ==== Because our view currently does not do any validation, this should fail and just redirect for all POSTs: ---- self.assertEqual(response.status_code, 200) AssertionError: 302 != 200 ---- [role="pagebreak-before"] Here's an implementation: [role="sourcecode"] .src/lists/views.py (ch14l037) ==== [source,python] ---- def view_list(request, list_id): our_list = List.objects.get(id=list_id) error = None if request.method == "POST": try: item = Item(text=request.POST["item_text"], list=our_list) # <1> item.full_clean() # <2> item.save() # <2> return redirect(f"/lists/{our_list.id}/") except ValidationError: error = "You can't have an empty list item" return render(request, "list.html", {"list": our_list, "error": error}) ---- ==== <1> Notice we do `Item()` instead of `Item.objects.create()`. <2> Then we call `full_clean()` before we call `save()`. It works: ---- Ran 15 tests in 0.047s OK ---- But it's not deeply satisfying, is it? There's definitely some duplication of code here; that `try/except` occurs twice in _views.py_, and in general things are feeling clunky. Let's wait a bit before we do more refactoring though, because we know we're about to do some slightly different validation coding for duplicate items. We'll just add it to our scratchpad for now: [role="scratchpad"] ***** * 'Remove hardcoded URLs from views.py.' * '[strikethrough line-through]#Remove the early return from the FT.#' * 'Remove hardcoded URL from forms in list.html and home.html.' * 'Remove duplication of validation logic in views.' ***** NOTE: One of the reasons that the "three strikes and refactor" rule exists is that, if you wait until you have three use cases, each might be slightly different, and it gives you a better view for what the common functionality is. If you refactor too early, you may find that the third use case doesn't quite fit with your refactored code. ((("database testing", "three strikes and refactor rule"))) ((("Test-Driven Development (TDD)", "concepts", "three strikes and refactor")))((("refactoring", ""three strikes and refactor" rule", secondary-sortas="three"))) ((("three strikes and refactor rule"))) At least our FTs are back to passing: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test functional_tests* [...] OK ---- We're back to a working state, so we can take a look at some of the items on our scratchpad. This would be a good time for a commit (and possibly a tea break): ((("", startref="MLVpost13")))((("", startref="HTMLpostdjango13")))((("", startref="POSTdjango13"))) [subs="specialcharacters,quotes"] ---- $ *git commit -am "enforce model validation in list view"* ---- === Refactor: Removing Hardcoded URLs ((("{% url %}"))) ((("templates", "tags", "{% url %}"))) ((("model-layer validation", "removing hardcoded URLs", id="MLVhard13"))) ((("URL mappings", id="url13"))) Do you remember those `name=` parameters in _urls.py_? We just copied them across from the default example that Django gave us, and I've been giving them some reasonably descriptive names. Now we find out what they're for: [role="sourcecode currentcontents"] .src/lists/urls.py ==== [source,python] ---- path("new", views.new_list, name="new_list"), path("/", views.view_list, name="view_list"), ---- ==== ==== The {% url %} Template Tag We can replace the hardcoded URL in _home.html_ with a Django template tag that refers to the URL's "name": [role="sourcecode"] .src/lists/templates/home.html (ch14l038) ==== [source,html] ---- {% block form_action %}{% url 'new_list' %}{% endblock %} ---- ==== We check that this doesn't break the unit tests: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test lists*] OK ---- Let's do the other template. This one is more interesting, because we pass it a [keep-together]#parameter#: [role="sourcecode"] .src/lists/templates/list.html (ch14l039) ==== [source,html] ---- {% block form_action %}{% url 'view_list' list.id %}{% endblock %} ---- ==== See the https://docs.djangoproject.com/en/5.2/topics/http/urls/#reverse-resolution-of-urls[Django docs on reverse URL resolution] for more info. We run the tests again, and check that they all pass: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test lists*] OK $ pass:quotes[*python src/manage.py test functional_tests*] OK ---- Excellent! Let's commit our progress: [subs="specialcharacters,quotes"] ---- $ *git commit -am "Refactor hard-coded URLs out of templates"* ---- And don't forget to cross off the "Remove hardcoded URL..." task as well: [role="scratchpad"] ***** * 'Remove hardcoded URLs from views.py.' * '[strikethrough line-through]#Remove the early return from the FT.#' * '[strikethrough line-through]#Remove hardcoded URL from forms in list.html and home.html.#' * 'Remove duplication of validation logic in views.' ***** ==== Using get_absolute_url for Redirects ((("get_absolute_url"))) Now let's tackle _views.py_. One way of doing it is just like in the template, passing in the name of the URL and a positional argument: [role="sourcecode"] .src/lists/views.py (ch14l040) ==== [source,python] ---- def new_list(request): [...] return redirect("view_list", nulist.id) ---- ==== That would get the unit and functional tests passing, but the `redirect` function can do even better magic than that! In Django, because model objects are often associated with a particular URL, you can define a special function called `get_absolute_url` which tells you what page displays the item. It's useful in this case, but it's also useful in the Django admin (which I don't cover in the book, but you'll soon discover for yourself) because it will let you jump from looking at an object in the admin view to looking at the object on the live site. I'd always recommend defining a `get_absolute_url` for a model whenever there is one that makes sense; it takes no time at all. All it takes is a super simple unit test in 'test_models.py': [role="sourcecode"] .src/lists/tests/test_models.py (ch14l041) ==== [source,python] ---- def test_get_absolute_url(self): mylist = List.objects.create() self.assertEqual(mylist.get_absolute_url(), f"/lists/{mylist.id}/") ---- ==== That gives: ---- AttributeError: 'List' object has no attribute 'get_absolute_url' ---- The implementation is to use Django's `reverse` function, which essentially does the reverse of what Django normally does with _urls.py_: [role="sourcecode"] .src/lists/models.py (ch14l042) ==== [source,python] ---- from django.urls import reverse class List(models.Model): def get_absolute_url(self): return reverse("view_list", args=[self.id]) ---- ==== And now we can use it in the view--the `redirect` function just takes the object we want to redirect to, and it uses `get_absolute_url` under the hood automagically! [role="sourcecode"] .src/lists/views.py (ch14l043) ==== [source,python] ---- def new_list(request): [...] return redirect(nulist) ---- ==== There's more info in the https://docs.djangoproject.com/en/5.2/topics/http/shortcuts/#redirect[Django docs]. Quick check that the unit tests still pass: [subs="specialcharacters,macros"] ---- OK ---- Then we do the same to `view_list`: [role="sourcecode"] .src/lists/views.py (ch14l044) ==== [source,python] ---- def view_list(request, list_id): [...] item.save() return redirect(our_list) except ValidationError: error = "You can't have an empty list item" ---- ==== And a full unit test and FT run to assure ourselves that everything still works: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test lists*] OK $ pass:quotes[*python src/manage.py test functional_tests*] OK ---- Time to cross off our to-dos... [role="scratchpad"] ***** * '[strikethrough line-through]#Remove hardcoded URLs from views.py.#' * '[strikethrough line-through]#Remove the early return from the FT.#' * '[strikethrough line-through]#Remove hardcoded URL from forms in list.html and home.html.#' * 'Remove duplication of validation logic in views.' ***** And commit... [subs="specialcharacters,quotes"] ---- $ *git commit -am "Use get_absolute_url on List model to DRY urls in views"* ---- And we're done with that bit! We have working model-layer validation, and we've taken the opportunity to do a few refactors along the way. ((("", startref="MLVhard13")))((("", startref="url13"))) That final scratchpad item will be the subject of the next chapter. [role="pagebreak-before less_space"] .On Database-layer Validation ******************************************************************************* As we saw, the specific "not empty" constraint we're trying to apply here isn't enforceable by SQLite, and so it was actually Django that ended up enforcing it for us. However, I always like to push my validation logic down as low as possible: ((("model-layer validation", "benefits and drawbacks of"))) Validation at the database layer is the ultimate guarantee of data integrity:: It can ensure that, no matter how complex your code gets at the layers above, you have guarantees at the lowest level that your data is valid and consistent. ((("data integrity errors"))) But it comes at the expense of flexibility:: This benefit doesn't come for free! It's now impossible, even temporarily, to have inconsistent data. Sometimes you might have a good reason for temporarily storing data that breaks the rules rather than storing nothing at all. Perhaps you're importing data from an external source in several stages, for example. And it's not designed for user-friendliness:: Trying to store invalid data will cause a nasty `IntegrityError` to come back from your database, and possibly the user will see a confusing 500 error page. As we'll see in later chapters, forms-layer validation is designed with the user in mind, anticipating the kinds of helpful error messages we want to send them.((("validation", "database layer", startref="ix_valDB"))) ((("", startref="UIdblayer13")))((("", startref="DBTdblayer13"))) ******************************************************************************* ================================================ FILE: chapter_15_simple_form.asciidoc ================================================ [[chapter_15_simple_form]] == A Simple Form At the end of the last chapter, we were left with the thought that there was too much duplication in the validation handling bits of our views. Django encourages you to use form classes to do the work of validating user input, and choosing what error messages to display. We'll use tests to explore the way Django forms work, and then we'll refactor our views to use them. As we go along, we'll see our unit tests and functional tests, in combination, will protect us from regressions. === Moving Validation Logic Into a Form TIP: In Django, a complex view is a code smell. Could some of that logic be pushed out to a form? Or to some custom methods on the model class? Or (perhaps best of all) to a non-Django module that represents your business logic? ((("form data validation", "benefits of"))) ((("form data validation", "moving validation logic to forms", id="FDVmoving14"))) ((("user interactions", "form data validation", id="UIform14"))) Forms have several superpowers in Django: * They can process user input and validate it for errors. * They can be used in templates to render HTML input elements, and error messages too. * And, as we'll see later, some of them can even save data to the database for you. You don't have to use all three superpowers in every form. You may prefer to roll your own HTML or do your own saving. But they are an excellent place to keep validation logic. ==== Exploring the Forms API with a Unit Test ((("Forms API", seealso="form data validation")))((("unit tests", "Forms API")))Let's do a little experimenting with forms by using a unit test. My plan is to iterate towards a complete solution, and hopefully introduce forms gradually enough that they'll make sense if you've never seen them before. First we add a new file for our form unit tests, and we start with a test that just looks at the form HTML: [role="sourcecode"] .src/lists/tests/test_forms.py (ch15l001) ==== [source,python] ---- from django.test import TestCase from lists.forms import ItemForm class ItemFormTest(TestCase): def test_form_renders_item_text_input(self): form = ItemForm() self.fail(form.as_p()) ---- ==== `form.as_p()` renders the form as HTML. This unit test uses a `self.fail` for some exploratory coding. You could just as easily use a `manage.py shell` session, although you'd need to keep reloading your code for each change. Let's make a minimal form. It inherits from the base `Form` class, and has a single field called `item_text`: [role="sourcecode"] .src/lists/forms.py (ch15l002) ==== [source,python] ---- from django import forms class ItemForm(forms.Form): item_text = forms.CharField() ---- ==== We now see a failure message that tells us what the autogenerated form HTML will look like: ---- AssertionError:

[...]

---- [role="pagebreak-before"] It's already pretty close to what we have in _base.html_. We're missing the placeholder attribute and the Bootstrap CSS classes. Let's make our unit test into a test for that: [role="sourcecode"] .src/lists/tests/test_forms.py (ch15l003) ==== [source,python] ---- class ItemFormTest(TestCase): def test_form_item_input_has_placeholder_and_css_classes(self): form = ItemForm() rendered = form.as_p() self.assertIn('placeholder="Enter a to-do item"', rendered) self.assertIn('class="form-control form-control-lg"', rendered) ---- ==== That gives us a fail, which justifies some real coding: [subs="specialcharacters"] ---- self.assertIn('placeholder="Enter a to-do item"', rendered) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'placeholder="Enter a to-do item"' not found in [...] ---- How can we customise the input for a form field? We use a "widget". Here it is with just the placeholder: [role="sourcecode"] .src/lists/forms.py (ch15l004) ==== [source,python] ---- class ItemForm(forms.Form): item_text = forms.CharField( widget=forms.widgets.TextInput( attrs={ "placeholder": "Enter a to-do item", } ), ) ---- ==== That gives: ---- AssertionError: 'class="form-control form-control-lg"' not found in '

\n \n \n \n \n \n \n

' ---- And then: [role="sourcecode"] .src/lists/forms.py (ch15l005) ==== [source,python] ---- widget=forms.widgets.TextInput( attrs={ "placeholder": "Enter a to-do item", "class": "form-control form-control-lg", } ), ---- ==== NOTE: Doing this sort of `widget` customisation would get tedious if we had a much larger, more complex form. Check out https://django-crispy-forms.readthedocs.org[django-crispy-forms] for some help. ((("django-crispy-forms"))) .Development-Driven Tests: Using Unit Tests for Exploratory Coding ******************************************************************************* ((("development-driven tests")))((("unit tests", "using for exploratory coding"))) ((("exploratory coding"))) Does this feel a bit like development-driven tests? That's OK, now and again. When you're exploring a new API, you're absolutely allowed to mess about with it for a while before you get back to rigorous TDD. You might use the interactive console, or write some exploratory code (but you have to promise the Testing Goat that you'll throw it away and rewrite it properly later). Here, we're actually using a unit test as a way of experimenting with the forms API. It can be a surprisingly good way of learning how something works. ******************************************************************************* // SEBASTIAN: Small suggestion - I'd appreciate mentioning breakpoint() for use in test // to be able to play with a form instance even more ==== Switching to a Django ModelForm ((("ModelForms"))) What's next? We want our form to reuse the validation code that we've already defined on our model. Django provides a special class that can autogenerate a form for a model, called `ModelForm`. As you'll see, it's configured using a special inner class called `Meta`: [role="sourcecode"] .src/lists/forms.py (ch15l006) ==== [source,python] ---- from django import forms from lists.models import Item class ItemForm(forms.models.ModelForm): class Meta: # <1> model = Item fields = ("text",) # item_text = forms.CharField( #<2> # widget=forms.widgets.TextInput( # attrs={ # "placeholder": "Enter a to-do item", # "class": "form-control form-control-lg", # } # ), # ) ---- ==== <1> In `Meta`, we specify which model the form is for and which fields we want it to use. <2> We'll comment out our manually created field for now. ++ModelForm++ does all sorts of smart stuff, like assigning sensible HTML form input types to different types of field, and applying default validation. Check out the https://docs.djangoproject.com/en/5.2/topics/forms/modelforms[docs] for more info. We now have some different-looking form HTML: ---- AssertionError: 'placeholder="Enter a to-do item"' not found in '

\n \n \n \n \n \n \n

' ---- It's lost our placeholder and CSS class. And you can also see that it's using `name="text"` instead of `name="item_text"`. We can probably live with that. But it's using a `textarea` instead of a normal input, and that's not the UI we want for our app. Thankfully, you can override `widgets` for `ModelForm` fields, similarly to the way we did it with the normal form: [role="sourcecode"] .src/lists/forms.py (ch15l007) ==== [source,python] ---- class ItemForm(forms.models.ModelForm): class Meta: model = Item fields = ("text",) widgets = { # <1> "text": forms.widgets.TextInput( attrs={ "placeholder": "Enter a to-do item", "class": "form-control form-control-lg", } ), } ---- ==== <1> We restore some of our commented-out code here, but modified slightly, from being an attribute declaration to a key in a dict. That gets the test passing. [role="pagebreak-before less_space"] ==== Testing and Customising Form Validation Now let's see if the `ModelForm` has picked up the same validation rules that we defined on the model.((("form data validation", "testing and customizing validation"))) We'll also learn how to pass data into the form, as if it came from the user: [role="sourcecode"] .src/lists/tests/test_forms.py (ch15l008) ==== [source,python] ---- def test_form_item_input_has_placeholder_and_css_classes(self): [...] def test_form_validation_for_blank_items(self): form = ItemForm(data={"text": ""}) form.save() ---- ==== That gives us: ---- ValueError: The Item could not be created because the data didn't validate. ---- Good: the form won't allow you to save if you give it an empty item text. Now let's see if we can get it to use the specific error message that we want. The API for checking form validation 'before' we try to save any data is a function called `is_valid`: [role="sourcecode"] .src/lists/tests/test_forms.py (ch15l009) ==== [source,python] ---- def test_form_item_input_has_placeholder_and_css_classes(self): [...] def test_form_validation_for_blank_items(self): [...] def test_form_validation_for_blank_items(self): form = ItemForm(data={"text": ""}) self.assertFalse(form.is_valid()) self.assertEqual(form.errors["text"], ["You can't have an empty list item"]) ---- ==== Calling `form.is_valid()` returns `True` or `False`, but it also has the side effect of validating the input data and populating the `errors` attribute. It's a dictionary mapping the names of fields to lists of errors for those fields (it's possible for a field to have more than one error). That gives us: ---- AssertionError: ['This field is required.'] != ["You can't have an empty list item"] ---- [role="pagebreak-before"] Django already has a default error message that we could present to the user--you might use it if you were in a hurry to build your web app, but we care enough to make our message special. Customising it means changing `error_messages`—another `Meta` variable: [role="sourcecode small-code"] .src/lists/forms.py (ch15l010) ==== [source,python] ---- class Meta: model = Item fields = ("text",) widgets = { "text": forms.widgets.TextInput( attrs={ "placeholder": "Enter a to-do item", "class": "form-control form-control-lg", } ), } error_messages = {"text": {"required": "You can't have an empty list item"}} ---- ==== ---- OK ---- You know what would be even better than messing about with all these error strings? Having a constant: [role="sourcecode"] .src/lists/forms.py (ch15l011) ==== [source,python] ---- EMPTY_ITEM_ERROR = "You can't have an empty list item" [...] error_messages = {"text": {"required": EMPTY_ITEM_ERROR}} ---- ==== Rerun the tests to see that they pass...OK. Now we can change the tests too. [role="sourcecode"] .src/lists/tests/test_forms.py (ch15l012) ==== [source,python] ---- from lists.forms import EMPTY_ITEM_ERROR, ItemForm [...] def test_form_validation_for_blank_items(self): form = ItemForm(data={"text": ""}) self.assertFalse(form.is_valid()) self.assertEqual(form.errors["text"], [EMPTY_ITEM_ERROR]) ---- ==== TIP: This is a good example of reusing constants in tests. It makes it easier to change the error message later. And the tests still pass: ---- OK ---- ((("", startref="FDVmoving14")))Great. Totes committable: [subs="specialcharacters,quotes"] ---- $ *git status* # should show forms.py and test_forms.py $ *git add src/lists* $ *git commit -m "new form for list items"* ---- === Attempting to Use the Form in Our Views ((("form data validation", "using forms in views", id="FDVviews14"))) At this point, we may be tempted to carry on—perhaps extend the form to capture uniqueness validation and empty-item validation. But there's a sort of corollary to the "deploy as early as possible" Lean methodology, which is "merge code as early as possible". In other words: while building this bit of forms code, it would be easy to go on for ages, adding more and more functionality to the form--I should know, because that's exactly what I did during the drafting of this chapter, and I ended up doing all sorts of work making an all-singing, all-dancing form class before I realised it wouldn't _actually_ work for our most basic use case. So, instead, try to use your new bit of code as soon as possible. This makes sure you never have unused bits of code lying around, and that you start checking your code against "the real world" as soon as possible. We have a form class that can render some HTML and do validation of at least one kind of error--let's start using it! We should be able to use it in our _base.html_ template—so, also, in all of our views. ==== Using the Form in a View with a GET Request ((("GET requests"))) ((("HTML", "GET requests"))) So, let's start using our form in our home page view: [role="sourcecode"] .src/lists/views.py (ch15l013) ==== [source,python] ---- [...] from lists.forms import ItemForm from lists.models import Item, List def home_page(request): return render(request, "home.html", {"form": ItemForm()}) ---- ==== OK, now let's try using it in the template--we replace the old `` with `{{ form.text }}`: [role="sourcecode"] .src/lists/templates/base.html (ch15l014) ==== [source,html] ----
{{ form.text }} <1> {% csrf_token %} {% if error %}
{{ error }}
{% endif %} ---- ==== <1> `{{ form.text }}` renders just the HTML input for the `text` field of the form. That causes our two unit tests that check on the form input to fail: [subs="specialcharacters,callouts"] ---- [...] ====================================================================== FAIL: test_renders_input_form (lists.tests.test_views.HomePageTest.test_renders_input_form) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/lists/tests/test_views.py", line 19, in test_renders_input_form self.assertIn("item_text", [input.get("name") for input in inputs]) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'item_text' not found in ['text', 'csrfmiddlewaretoken'] <1> ====================================================================== FAIL: test_renders_input_form (lists.tests.test_views.ListViewTest.test_renders_input_form) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/lists/tests/test_views.py", line 60, in test_renders_input_form self.assertIn("item_text", [input.get("name") for input in inputs]) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'item_text' not found in ['csrfmiddlewaretoken'] <2> Ran 18 tests in 0.022s FAILED (failures=2) ---- <1> The test for the home page is failing because the `name` attribute of the input box is now `text`, not `item_text`. <2> The test for the list view is failing because because we're not instantiating a form in that view, so there's no `form` variable in the template. The input box isn't even being rendered. [role="pagebreak-before"] Let's fix things one at a time. First, let's back out our change and restore the hand-crafted HTML input in cases where `{{ form }}` is not defined: [role="sourcecode small-code"] .src/lists/templates/base.html (ch15l015) ==== [source,html] ----
{% if form %} {{ form.text }} {% else %} {% endif %} {% csrf_token %} {% if error %}
{{ error }}
{% endif %} ---- ==== That takes us down to one failure: ---- AssertionError: 'item_text' not found in ['text', 'csrfmiddlewaretoken'] ---- Let's make a note to come back and tidy this up, and then we'll talk about what's happened and how to deal with it: [role="scratchpad"] ***** * _Remove duplication of validation logic in views._ * _Remove if branch and hardcoded input tag from base.html._ ***** ==== The Trade-offs of Django ModelForms: The Frontend Is Coupled to the Database This highlights one of the trade-offs of using `ModelForm`: by auto-generating the form from the model, we tie the `name=` attribute of our form's HTML `` to the name of the model field in the database.((("ModelForms", "tradeoffs of"))) In a simple CRUD (create, read, update, and delete) app like ours, that's probably a good deal. But it does mean we need to go back and change our assumptions about what the `name=` attribute of the input box is going to be. While we're at it, it's worth doing an FT run too: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests*] [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [id="id_new_item"]; [...] [...] FAILED (errors=4) ---- Looks like something else has changed. If you pause the FTs or inspect the HTML manually in a browser, you'll see that the `ModelForm` also changes the `id` attribute to being `id_text`.footnote:[It's actually possible to customise this attribute via the `widgets` attribute we used earlier, even on a `ModelForm`, but because you cannot change the `name` one, we may as well just accept this too.] === A Big Find-and-Replace ((("find and replace"))) ((("grep command"))) If we want to change our assumption about these two attributes, we'll need to embark on a couple of big find-and-replaces basically: [role="scratchpad"] ***** * _Remove duplication of validation logic in views._ * _Remove if branch and hardcoded input tag from base.html._ * _Change input name attribute from item_text to just text._ * _Change input id from id_new_item to id_text._ ***** But before we do that, let's back out the rest of our changes and get back to a working state. [role="pagebreak-before less_space"] ==== Backing Out Our Changes and Getting to a Working State The simplest way to back out changes is with `git`. But in this case, leaving a couple of placeholders does no harm, and they'll be helpful to come back to later.((("commented-out code")))((("Git", "commented-out code and if branches, caution with"))) So we can leave the `{{ form.text }}` in the HTML but, by backing out the change in the view, we'll make sure that branch is never actually exercised. Again, to leave ourselves a little placeholder, we'll comment out our code rather than deleting it: [role="sourcecode"] .src/lists/views.py (ch15l016) ==== [source,python] ---- def home_page(request): # return render(request, "home.html", {"form": ItemForm()}) return render(request, "home.html") ---- ==== WARNING: Be very cautious about leaving commented-out code and unused `if` branches lying around. Do so only if you're sure you're coming back to them very soon, otherwise your codebase will soon get messy! Now we can do a full unit test and FT run to confirm we're back to a working state: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test lists* Found 18 test(s). [...] OK $ *python src/manage.py test functional_tests* Found 4 test(s). [...] OK ---- And let's do a commit to be able to separate out the rename from anything else: [subs="specialcharacters,quotes"] ---- $ *git diff* # changes in base.html + views.py $ *git commit -am "Placeholders for using form in view+template, not in use yet"* ---- [role="pagebreak-before"] And pop an item on the to-do list: [role="scratchpad"] ***** * _Remove duplication of validation logic in views._ * _Remove if branch and hardcoded input tag from base.html._ * _Change input name attribute from item_text to just text._ * _Change input id from id_new_item to id_text._ * _Uncomment use of form in home_page() view_item to id_text._ * _Use form in other views._ ***** ==== Renaming the name Attribute So, let's have a look for `item_text` in the codebase: [subs="specialcharacters,macros"] ---- $ pass:quotes[*grep -Ir item_text src*] src/lists/migrations/0003_list.py: ("lists", "0002_item_text"), src/lists/tests/test_views.py: self.assertIn("item_text", [input.get("name") for input in inputs]) [...] src/lists/templates/base.html: name="item_text" src/lists/views.py: item = Item(text=request.POST["item_text"], list=nulist) src/lists/views.py: item = Item(text=request.POST["item_text"], list=our_list) ---- We can ignore the migration, which is just using `item_text` as metadata. So the changes we need to make are in three places: 1. _views.py_ 2. _test_views.py_ 3. _base.html_ [role="pagebreak-before"] Let's go ahead and make those. I'm sure you can manage your own find-and-replace! They should look something like this: [role="sourcecode"] .src/lists/tests/test_views.py (ch15l017) ==== [source,diff] ---- @@ -16,12 +16,12 @@ class HomePageTest(TestCase): [form] = parsed.cssselect("form[method=POST]") self.assertEqual(form.get("action"), "/lists/new") inputs = form.cssselect("input") - self.assertIn("item_text", [input.get("name") for input in inputs]) + self.assertIn("text", [input.get("name") for input in inputs]) class NewListTest(TestCase): def test_can_save_a_POST_request(self): - self.client.post("/lists/new", data={"item_text": "A new list item"}) + self.client.post("/lists/new", data={"text": "A new list item"}) self.assertEqual(Item.objects.count(), 1) new_item = Item.objects.get() self.assertEqual(new_item.text, "A new list item") [...] ---- ==== Or, in _views.py_: [role="sourcecode dofirst-ch15l018"] .src/lists/views.py (ch15l019) ==== [source,diff] ---- @@ -12,7 +12,7 @@ def home_page(request): def new_list(request): nulist = List.objects.create() - item = Item(text=request.POST["item_text"], list=nulist) + item = Item(text=request.POST["text"], list=nulist) try: item.full_clean() item.save() @@ -29,7 +29,7 @@ def view_list(request, list_id): if request.method == "POST": try: - item = Item(text=request.POST["item_text"], list=our_list) + item = Item(text=request.POST["text"], list=our_list) item.full_clean() item.save() return redirect(our_list) ---- ==== [role="pagebreak-before"] Finally, in _base.html_: [role="sourcecode small-code"] .src/lists/templates/base.html (ch15l020) ==== [source,diff] ---- @@ -21,7 +21,7 @@ {% else %} ---- ==== Once you're done, rerun the unit tests to confirm that the application is self-consistent: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test lists*] [...] Ran 18 tests in 0.126s OK ---- And rerun the FTs too: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests*] [...] Ran 4 tests in 12.154s OK ---- Good! One down: [role="scratchpad"] ***** * _Remove duplication of validation logic in views._ * _Remove if branch and hardcoded input tag from base.html._ * _[strikethrough line-through]#Change input name attribute from item_text to just text.#_ * _Change input id from id_new_item to id_text._ * _Uncomment use of form in home_page() view_item to id_text._ * _Use form in other views._ ***** ==== Renaming the id Attribute Now for the `id=` attribute. A quick `grep` shows us that `id_new_item` appears in the template, and in all three FT files: [subs=""] ---- $ grep -r id_new_item src/lists/templates/base.html: id="id_new_item" src/functional_tests/test_list_item_validation.py: self.browser.find_element(By.ID, "id_new_item").send_keys(Keys.ENTER) src/functional_tests/test_list_item_validation.py: self.browser.find_element(By.ID, "id_new_item").send_keys("Purchase milk") [...] ---- That's a good call for a refactor within the FTs too. Let's make a new helper method in _base.py_: [role="sourcecode"] .src/functional_tests/base.py (ch15l021) ==== [source,python] ---- class FunctionalTest(StaticLiveServerTestCase): [...] def get_item_input_box(self): return self.browser.find_element(By.ID, "id_new_item") # <1> ---- ==== <1> We'll keep the old `id` for now. Working state to working state! And then we use it throughout--I had to make four changes in _test_simple_list_creation.py_, two in _test_layout_and_styling.py_, and six in _test_list_item_validation.py_, for example: [role="sourcecode dofirst-ch15l022 currentcontents"] .src/functional_tests/test_simple_list_creation.py ==== [source,python] ---- # She is invited to enter a to-do item straight away inputbox = self.get_item_input_box() ---- ==== Or: [role="sourcecode currentcontents"] .src/functional_tests/test_list_item_validation.py ==== [source,python] ---- # an empty list item. She hits Enter on the empty input box self.browser.get(self.live_server_url) self.get_item_input_box().send_keys(Keys.ENTER) ---- ==== I won't show you every single one; I'm sure you can manage this for yourself! You can redo the `grep` to check that you've caught them all: [subs="specialcharacters,quotes"] ---- $ *grep -r id_new_item* src/lists/templates/base.html: id="id_new_item" src/functional_tests/base.py: return self.browser.find_element(By.ID, "id_new_item") ---- [role="pagebreak-before"] And we can do an FT run too, to make sure we haven't broken anything: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests*] [...] Ran 4 tests in 12.154s OK ---- Good! FT refactor complete—now hopefully we can make the application-level refactor of the `id` attribute in just two places, and we've been in a working state the whole way through. In the FT helper method: [role="sourcecode"] .src/functional_tests/base.py (ch15l023) ==== [source,diff] ---- @@ -43,4 +43,4 @@ class FunctionalTest(StaticLiveServerTestCase): time.sleep(0.5) def get_item_input_box(self): - return self.browser.find_element(By.ID, "id_new_item") + return self.browser.find_element(By.ID, "id_text") ---- ==== And in the template: [role="sourcecode small-code"] .src/lists/templates/base.html (ch15l024) ==== [source,diff] ---- @@ -22,7 +22,7 @@ {% endif %} ---- ==== And an FT run to confirm: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests*] [...] Ran 4 tests in 12.154s OK ---- Hooray! [role="scratchpad"] ***** * _Remove duplication of validation logic in views._ * _Remove if branch and hardcoded input tag from base.html._ * _[strikethrough line-through]#Change input name attribute from item_text to just text.#_ * _[strikethrough line-through]#Change input id from id_new_item to id_text.#_ * _Uncomment use of form in home_page() view_item to id_text._ * _Use form in other views._ ***** === A Second Attempt at Using the Form in Our Views Now that we've done the groundwork, hopefully we can drop in our form in the `home_page()` once again: [role="sourcecode"] .src/lists/views.py (ch15l025) ==== [source,python] ---- def home_page(request): return render(request, "home.html", {"form": ItemForm()}) ---- ==== Looking good! [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test lists* Found 18 test(s). [...] OK ---- [role="scratchpad"] ***** * _Remove duplication of validation logic in views._ * _Remove if branch and hardcoded input tag from base.html._ * _[strikethrough line-through]#Change input name attribute from item_text to just text.#_ * _[strikethrough line-through]#Change input id from id_new_item to id_text.#_ * _[strikethrough line-through]#Uncomment use of form in home_page() view_item to id_text.#_ * _Use form in other views._ ***** // TODO at this point the FTs actually start failing, // due to the required=true issue. // could address that here, but it does make all the use-form-for-validation stuff // seem a bit pointless Let's see what happens if we remove that `if` from the template: [role="sourcecode small-code"] .src/lists/templates/base.html (ch15l026) ==== [source,diff] ---- @@ -16,16 +16,7 @@

{% block header_text %}{% endblock %}

- {% if form %} - {{ form.text }} - {% else %} - - {% endif %} + {{ form.text }} {% csrf_token %} {% if error %}
{{ error }}
---- ==== Aha—the unit tests are there to tell us that we need to use the form in `view_list()` too: ---- AssertionError: 'text' not found in ['csrfmiddlewaretoken'] ---- [role="pagebreak-before"] Here's the minimal use of the form--we won't use it for validation yet, just for getting the form into the template: [role="sourcecode"] .src/lists/views.py (ch15l027) ==== [source,python] ---- def view_list(request, list_id): our_list = List.objects.get(id=list_id) error = None form = ItemForm() if request.method == "POST": try: item = Item(text=request.POST["text"], list=our_list) item.full_clean() item.save() return redirect(our_list) except ValidationError: error = "You can't have an empty list item" return render( request, "list.html", {"list": our_list, "form": form, "error": error} ) ---- ==== And the tests are happy with that too: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test lists* Found 18 test(s). [...] OK ---- We're done with the template; what's next? [role="scratchpad"] ***** * _Remove duplication of validation logic in views._ * _[strikethrough line-through]#Remove if branch and hardcoded input tag from base.html.#_ * _[strikethrough line-through]#Change input name attribute from item_text to just text.#_ * _[strikethrough line-through]#Change input id from id_new_item to id_text.#_ * _[strikethrough line-through]#Uncomment use of form in home_page() view_item to id_text.#_ * _Use form in other views._ ***** Right, let's move on to the next view that doesn't use our form yet—`new_list()`. And actually, that'll help us with the first item, which was the whole point of this adventure, really: to see if the forms can help us better handle validation. Let's see how that works now. === Using the Form in a View That Takes POST Requests ((("form data validation", "processing POST requests"))) Here's how we can use the form in the `new_list()` view, avoiding all the manual manipulation of `request.POST` and the error message: [role="sourcecode"] .src/lists/views.py (ch15l028) ==== [source,python] ---- def new_list(request): form = ItemForm(data=request.POST) #<1> if form.is_valid(): #<2> nulist = List.objects.create() Item.objects.create(text=request.POST["text"], list=nulist) return redirect(nulist) else: return render(request, "home.html", {"form": form}) #<3> ---- ==== <1> We pass the `request.POST` data into the form's constructor. <2> We use `form.is_valid()` to determine whether this is a good or a bad submission. <3> In the invalid case, we pass the form down to the template, instead of our hardcoded error string. That view is now looking much nicer! But, we have a regression in the unit tests: ---- ====================================================================== FAIL: test_validation_errors_are_sent_back_to_home_page_template (lists.tests.t est_views.NewListTest.test_validation_errors_are_sent_back_to_home_page_templat e) --------------------------------------------------------------------- [...] self.assertContains(response, expected_error) ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: False is not true : Couldn't find 'You can't have an empty list item' in the following response b'\n\n\n \n To-Do [...] ---- ==== Using the Form to Display Errors in the Template We're failing because we're not yet _using_ the form to display errors in the template.((("templates", "using form to display errors in"))) Here's how to do that: [role="sourcecode"] .src/lists/templates/base.html (ch15l029) ==== [source,html] ---- <form method="POST" action="{% block form_action %}{% endblock %}" > {{ form.text }} {% csrf_token %} {% if form.errors %} <1> <div class="invalid-feedback">{{ form.errors.text }}</div> <2> {% endif %} </form> ---- ==== <1> We change the `if` to look at `form.errors`: it contains a list of all the errors for the form. <2> `form.errors.text` is magical Django template syntax for `form.errors["text"]`—i.e., the list of errors for the text field in particular. What does that do to our unit tests? ---- ====================================================================== FAIL: test_validation_errors_end_up_on_lists_page (lists.tests.test_views.ListV iewTest.test_validation_errors_end_up_on_lists_page) --------------------------------------------------------------------- [...] AssertionError: False is not true : Couldn't find 'You can't have an empty list item' in the following response ---- [role="pagebreak-before"] An unexpected failure--it's actually in the tests for our final view, `view_list()`. Once again, because we've changed the base template, which is used by _all_ views, we've made a change that impacts more places than we intended. Let's follow our standard pattern, get back to a working state, and see if we can dig into this a bit. ==== Get Back to a Working State Let's restore the old `[% if %}` in the template, so we display errors in both old and new cases: [role="sourcecode"] .src/lists/templates/base.html (ch15l029-1) ==== [source,html] ---- <form method="POST" action="{% block form_action %}{% endblock %}" > {{ form.text }} {% csrf_token %} {% if error %} <div class="invalid-feedback">{{ error }}</div> {% endif %} {% if form.errors %} <div class="invalid-feedback">{{ form.errors.text }}</div> {% endif %} </form> ---- ==== And add an item to our stack: [role="scratchpad"] ***** * _Remove duplication of validation logic in views_ * _[strikethrough line-through]#Remove if branch and hardcoded input tag from base.html#_ * _[strikethrough line-through]#Change input name attribute from item_text to just text#_ * _[strikethrough line-through]#Change input id from id_new_item to id_text#_ * _[strikethrough line-through]#Uncomment use of form in home_page() view_item to id_text#_ * _Use form in other views_ * _Remove if error branch from template_ ***** [role="pagebreak-before less_space"] ==== A Helper Method for Several Short Tests Let's take a look at ((("helper methods", "for short form validation tests", secondary-sortas="short")))our tests for both views, particularly the ones that check for invalid inputs: [role="sourcecode currentcontents"] .src/lists/tests/test_views.py ==== [source,python] ---- class NewListTest(TestCase): [...] def test_validation_errors_are_sent_back_to_home_page_template(self): response = self.client.post("/lists/new", data={"text": ""}) self.assertEqual(response.status_code, 200) self.assertTemplateUsed(response, "home.html") expected_error = html.escape("You can't have an empty list item") self.assertContains(response, expected_error) def test_invalid_list_items_arent_saved(self): self.client.post("/lists/new", data={"text": ""}) self.assertEqual(List.objects.count(), 0) self.assertEqual(Item.objects.count(), 0) class ListViewTest(TestCase): [...] def test_validation_errors_end_up_on_lists_page(self): list_ = List.objects.create() response = self.client.post( f"/lists/{list_.id}/", data={"text": ""}, ) self.assertEqual(response.status_code, 200) self.assertTemplateUsed(response, "list.html") expected_error = html.escape("You can't have an empty list item") self.assertContains(response, expected_error) ---- ==== I see a few problems here: 1. We’re explicitly checking that validation errors prevent anything from being saved to the database in `NewListTest`, but not in `ListViewTest`. 2. We're mixing up the test for the status code, the template, and finding the error in the result. Let's be extra meticulous here, and separate out these concerns. Ideally, each test should have one assert. If we used copy-paste, that would start to involve a lot of duplication, so using a couple of helper methods is a good idea here. [role="pagebreak-before"] Here's some better tests in `NewListTest`: [role="sourcecode"] .src/lists/tests/test_views.py (ch15l029-2) ==== [source,python] ---- from lists.forms import EMPTY_ITEM_ERROR [...] class NewListTest(TestCase): def test_can_save_a_POST_request(self): [...] def test_redirects_after_POST(self): [...] def post_invalid_input(self): return self.client.post("/lists/new", data={"text": ""}) def test_for_invalid_input_nothing_saved_to_db(self): self.post_invalid_input() self.assertEqual(Item.objects.count(), 0) def test_for_invalid_input_renders_list_template(self): response = self.post_invalid_input() self.assertEqual(response.status_code, 200) self.assertTemplateUsed(response, "home.html") def test_for_invalid_input_shows_error_on_page(self): response = self.post_invalid_input() self.assertContains(response, html.escape(EMPTY_ITEM_ERROR)) ---- ==== By making a little helper function, `post_invalid_input()`, we can make three separate tests without duplicating lots of lines of code. We've seen this several times now. It often feels more natural to write view tests as a single, monolithic block of assertions--the view should do this and this and this, then return that with this. But breaking things out into multiple tests is often worthwhile; as we saw in previous chapters, it helps you isolate the exact problem you have when you later accidentally introduce a bug. Helper methods are one of the tools that lower the psychological barrier, by reducing boilerplate and keeping the tests readable. [role="pagebreak-before"] Let's do something similar in `ListViewTest`: [role="sourcecode"] .src/lists/tests/test_views.py (ch15l029-3) ==== [source,python] ---- class ListViewTest(TestCase): def test_uses_list_template(self): [...] def test_renders_input_form(self): [...] def test_displays_only_items_for_that_list(self): [...] def test_can_save_a_POST_request_to_an_existing_list(self): [...] def test_POST_redirects_to_list_view(self): [...] def post_invalid_input(self): mylist = List.objects.create() return self.client.post(f"/lists/{mylist.id}/", data={"text": ""}) def test_for_invalid_input_nothing_saved_to_db(self): self.post_invalid_input() self.assertEqual(Item.objects.count(), 0) def test_for_invalid_input_renders_list_template(self): response = self.post_invalid_input() self.assertEqual(response.status_code, 200) self.assertTemplateUsed(response, "list.html") def test_for_invalid_input_shows_error_on_page(self): response = self.post_invalid_input() self.assertContains(response, html.escape(EMPTY_ITEM_ERROR)) ---- ==== // (See <<single-endpoint-for-forms>> in the previous chapter if a diagram would be helpful). // TODO - maybe a little aside saying i'm exaggerating here? // not sure i would do this IRL. // i mean, it's a good idea _in general_, // just maybe not for forms??? And let's rerun all our tests: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test lists* Found 21 test(s). [...] OK ---- Great! We now feel confident that we have a lot of very specific unit tests, which can point us to exactly what goes wrong if we ever make a mistake. [role="pagebreak-before"] So let's have another go at using our form for _all_ views, by fully committing to the `{{ form.errors }}` in the template: [role="sourcecode"] .src/lists/templates/base.html (ch15l029-4) ==== [source,diff] ---- @@ -18,9 +18,6 @@ <form method="POST" action="{% block form_action %}{% endblock %}" > {{ form.text }} {% csrf_token %} - {% if error %} - <div class="invalid-feedback">{{ error }}</div> - {% endif %} {% if form.errors %} <div class="invalid-feedback">{{ form.errors.text }}</div> {% endif %} ---- ==== And we'll see that exactly one test is failing: ---- FAIL: test_for_invalid_input_shows_error_on_page (lists.tests.test_views.ListVi ewTest.test_for_invalid_input_shows_error_on_page) [...] AssertionError: False is not true : Couldn't find 'You can't have an empty list item' in the following response ---- === Using the Form in the Existing Lists View ((("form data validation", "processing POST and GET requests"))) Let's try and work step by step towards fully using our form in this final view. ==== Using the Form to Pass Errors to the Template At the moment, one test is failing because the `view_list()` view for existing lists is not populating `form.errors` in the invalid case. ((("templates", "using form to pass errors to")))Let’s address just that: [role="sourcecode small-code"] .src/lists/views.py (ch15l030-1) ==== [source,python] ---- def view_list(request, list_id): our_list = List.objects.get(id=list_id) error = None form = ItemForm() # <2> if request.method == "POST": form = ItemForm(data=request.POST) # <1> try: item = Item(text=request.POST["text"], list=our_list) item.full_clean() item.save() return redirect(our_list) except ValidationError: error = "You can't have an empty list item" return render( request, "list.html", {"list": our_list, "form": form, "error": error} # <3> ) ---- ==== <1> Let's add this line, in the `method=POST` branch, and instantiate a form using the POST data. <2> We already had this empty form for the GET case, but our new one will override it. <3> And it should now drop through to the template here. That gets us back to a working state! [subs="specialcharacters,quotes"] ---- Found 21 test(s). [...] OK ---- [role="scratchpad"] ***** * _Remove duplication of validation logic in views._ * _[strikethrough line-through]#Remove if branch and hardcoded input tag from base.html.#_ * _[strikethrough line-through]#Change input name attribute from item_text to just text.#_ * _[strikethrough line-through]#Change input id from id_new_item to id_text.#_ * _[strikethrough line-through]#Uncomment use of form in home_page() view_item to id_text.#_ * _Use form in other views._ * _[strikethrough line-through]#Remove if error branch from template.#_ ***** [role="pagebreak-before less_space"] ==== Refactoring the View to Use the Form Fully Now let's start using the form more fully, and remove some of the manual error handling. We remove the `try/except` and replace it with an `if form.is_valid()` check, like the one in `new_list()`: [role="sourcecode"] .src/lists/views.py (ch15l030-2) ==== [source,diff] ---- @@ -26,13 +26,11 @@ def view_list(request, list_id): if request.method == "POST": form = ItemForm(data=request.POST) - try: + if form.is_valid(): item = Item(text=request.POST["text"], list=our_list) item.full_clean() item.save() return redirect(our_list) - except ValidationError: - error = "You can't have an empty list item" return render( request, "list.html", {"list": our_list, "form": form, "error": error} ---- ==== And the tests still pass: ---- OK ---- Next, we no longer need the `.full_clean()`, so we can go back to using `.objects.create()`: [role="sourcecode"] .src/lists/views.py (ch15l030-3) ==== [source,diff] ---- @@ -27,9 +27,7 @@ def view_list(request, list_id): if request.method == "POST": form = ItemForm(data=request.POST) if form.is_valid(): - item = Item(text=request.POST["text"], list=our_list) - item.full_clean() - item.save() + Item.objects.create(text=request.POST["text"], list=our_list) return redirect(our_list) ---- ==== The tests still pass: ---- OK ---- [role="pagebreak-before"] Finally, the `error` variable is always `None`, and is no longer needed in the template anyhow: [role="sourcecode"] .src/lists/views.py (ch15l030-4) ==== [source,diff] ---- @@ -21,7 +21,6 @@ def new_list(request): def view_list(request, list_id): our_list = List.objects.get(id=list_id) - error = None form = ItemForm() if request.method == "POST": @@ -30,6 +29,4 @@ def view_list(request, list_id): Item.objects.create(text=request.POST["text"], list=our_list) return redirect(our_list) - return render( - request, "list.html", {"list": our_list, "form": form, "error": error} - ) + return render(request, "list.html", {"list": our_list, "form": form}) ---- ==== And the tests are happy with that! ---- OK ---- I think our view is in a pretty good shape now. Here it is in non-diff mode, as a recap: [role="sourcecode currentcontents"] .src/lists/views.py ==== [source,python] ---- def view_list(request, list_id): our_list = List.objects.get(id=list_id) form = ItemForm() if request.method == "POST": form = ItemForm(data=request.POST) if form.is_valid(): Item.objects.create(text=request.POST["text"], list=our_list) return redirect(our_list) return render(request, "list.html", {"list": our_list, "form": form}) ---- ==== [role="pagebreak-before"] I think we can give ourselves the satisfaction of doing some crossing-things-out: [role="scratchpad"] ***** * _[strikethrough line-through]#Remove duplication of validation logic in views.#_ * _[strikethrough line-through]#Remove if branch and hardcoded input tag from base.html.#_ * _[strikethrough line-through]#Change input name attribute from item_text to just text.#_ * _[strikethrough line-through]#Change input id from id_new_item to id_text.#_ * _[strikethrough line-through]#Uncomment use of form in home_page() view_item to id_text.#_ * _[strikethrough line-through]#Use form in other views.#_ * _[strikethrough line-through]#Remove if error branch from template.#_ ***** Phew! Hey, it's been a while, what do our FTs think? [subs="specialcharacters,quotes"] ---- [...] ====================================================================== ERROR: test_cannot_add_empty_list_items (functional_tests.test_list_item_valida tion.ItemValidationTest.test_cannot_add_empty_list_items) --------------------------------------------------------------------- [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .invalid-feedback; [...] [...] Ran 4 tests in 14.897s FAILED (errors=1) ---- Oh. All the regression tests are OK, but our validation test seems to be failing—and failing early too! It's on the first attempt to submit an empty item. What happened? [role="pagebreak-before"] === An Unexpected Benefit: Free Client-Side Validation from HTML5 ((("HTML5", "client-side validation from")))((("client-side validation", "from HTML5", secondary-sortas="HTML5"))) How shall we find out what's going on here? One option is to add the usual `time.sleep` just before the error in the FTs, and take a look at what's happening while they run. Alternatively, spin up the site manually with `manage.py runserver` if you prefer. Either way, you should see something like <<html5_popup_screenshot>>. [[html5_popup_screenshot]] .HTML5 validation says no image::images/tdd3_1501.png["The input with a popup saying 'please fill out this field'"] It seems like the browser is preventing the user from even submitting the input when it's empty.((("required attribute (HTML input)"))) It's because Django has added the `required` attribute to the HTML input (take another look at our `as_p()` printouts from earlier if you don't believe me, or have a look at the source in DevTools). This is a https://oreil.ly/z2XiU[feature of HTML5]; browsers nowadays will do some validation at the client side if they can, preventing users from even submitting invalid input. That's actually good news! [role="pagebreak-before"] But, we were working based on incorrect assumptions about what the user experience was going to be. Let's change our FT to reflect this new expectation: [role="sourcecode small-code"] .src/functional_tests/test_list_item_validation.py (ch15l031) ==== [source,python] ---- class ItemValidationTest(FunctionalTest): def test_cannot_add_empty_list_items(self): # Edith goes to the home page and accidentally tries to submit # an empty list item. She hits Enter on the empty input box self.browser.get(self.live_server_url) self.get_item_input_box().send_keys(Keys.ENTER) # The browser intercepts the request, and does not load the list page self.wait_for( lambda: self.browser.find_element(By.CSS_SELECTOR, "#id_text:invalid") #<1> ) # She starts typing some text for the new item and the error disappears self.get_item_input_box().send_keys("Purchase milk") self.wait_for( lambda: self.browser.find_element(By.CSS_SELECTOR, "#id_text:valid") #<2> ) # And she can submit it successfully self.get_item_input_box().send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Purchase milk") # Perversely, she now decides to submit a second blank list item self.get_item_input_box().send_keys(Keys.ENTER) # Again, the browser will not comply self.wait_for_row_in_list_table("1: Purchase milk") self.wait_for( lambda: self.browser.find_element(By.CSS_SELECTOR, "#id_text:invalid") ) # And she can make it happy by filling some text in self.get_item_input_box().send_keys("Make tea") self.wait_for( lambda: self.browser.find_element( By.CSS_SELECTOR, "#id_text:valid", ) ) self.get_item_input_box().send_keys(Keys.ENTER) self.wait_for_row_in_list_table("2: Make tea") ---- ==== <1> Instead of checking for our custom error message, we check using the CSS pseudo-selector `:invalid`, which the browser applies to any HTML5 input that has invalid input.((("CSS (Cascading Style Sheets)", "pseudo-selector :invalid"))) <2> And we check for its converse in the case of valid inputs. See how useful and flexible our `self.wait_for()` function is turning out to be? Our FT does look quite different from how it started though, doesn't it? I'm sure that's raising a lot of questions in your mind right now. Put a pin in them for a moment; I promise we'll talk. Let's first see if we're back to passing tests: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests*] [...] Ran 4 tests in 12.154s OK ---- === A Pat on the Back First, let's give ourselves a massive pat on the back: we've just made a major change to our small app--that input field, with its name and ID, is absolutely critical to making everything work. We've touched seven or eight different files, doing a refactor that's quite involved...this is the kind of thing that, without tests, would seriously worry me. In fact, I might well have decided that it wasn't worth messing with code that works. But, because we have a full test suite, we can delve around, tidying things up, safe in the knowledge that the tests are there to spot any mistakes we make. It just makes it that much more likely that you're going to keep refactoring, keep tidying up, keep gardening, keep tending to your code, and keep everything neat and tidy and clean and smooth and precise and concise and functional and good. And it's definitely time for a commit: [subs="specialcharacters,quotes"] ---- $ *git diff* $ *git commit -am "use form in all views, back to working state"* ---- ==== But Have We Wasted a Lot of Time? ((("form data validation", "benefits of"))) But what about our custom error message? What about all that effort rendering the form in our HTML template? We're not even passing those errors from Django to the user if the browser is intercepting the requests before the user even makes them! And our FT isn't even testing that stuff any more! Well, you're quite right. But there are two or three reasons all our time hasn't been wasted. Firstly, client-side validation isn't enough to guarantee you're protected from bad inputs, so you always need the server side as well if you really care about data integrity; using a form is a nice way of encapsulating that logic. ((("HTML5", "browsers' support for"))) Also, not all browsers fully implement HTML5,footnote:[ Safari was a notable laggard in the last decade; it's up to date now.] so some users might still see our custom error message. And if or when we come to letting users access our data via an API (see https://www.obeythetestinggoat.com/book/appendix_rest_api.html[Online Appendix: Building a REST API]), then our validation messages will come back into use. On top of that, we'll be able to reuse all our validation and forms code when we do some more advanced validation that can't be done by HTML5 magic. But you know, even if all that weren't true, you can’t be too hard on yourself for occasionally barking up the wrong tree while you're coding. None of us can see the future, and we should concentrate on finding the right solution rather than the time "wasted" on the wrong solution. === Using the ModelForm's Own Save Method ((("form data validation", "using form’s own save method", id="FDVsave14")))((("ModelForms", "using save method", id="ix_MdFsave"))) There are a couple more things we can do to make our views even simpler. I've mentioned that forms are supposed to be able to save data to the database for us. Our case won't quite work out of the box, because the item needs to know what list to save to. But it's not hard to fix that! We start, as always, with a test. Just to illustrate what the problem is, let's see what happens if we just try to call `form.save()`: [role="sourcecode"] .src/lists/tests/test_forms.py (ch15l033) ==== [source,python] ---- def test_form_save_handles_saving_to_a_list(self): form = ItemForm(data={"text": "do me"}) new_item = form.save() ---- ==== Django isn't happy, because an item needs to belong to a list: ---- django.db.utils.IntegrityError: NOT NULL constraint failed: lists_item.list_id ---- Our solution is to tell the form's save method what list it should save to: [role="sourcecode"] .src/lists/tests/test_forms.py (ch15l034) ==== [source,python] ---- from lists.models import Item, List [...] def test_form_save_handles_saving_to_a_list(self): mylist = List.objects.create() form = ItemForm(data={"text": "do me"}) new_item = form.save(for_list=mylist) # <1> self.assertEqual(new_item, Item.objects.get()) #<2> self.assertEqual(new_item.text, "do me") self.assertEqual(new_item.list, mylist) ---- ==== <1> We'll imagine that the `.save()` method takes a `for_list=` argument. <2> We then make sure that the item is correctly saved to the database, with the right attributes. [role="pagebreak-before"] The tests fail as expected, because as usual, it's still only wishful thinking: ---- new_item = form.save(for_list=mylist) TypeError: BaseModelForm.save() got an unexpected keyword argument 'for_list' ---- Here's how we can implement a custom save method: [role="sourcecode"] .src/lists/forms.py (ch15l035) ==== [source,python] ---- class ItemForm(forms.models.ModelForm): class Meta: [...] def save(self, for_list): self.instance.list = for_list return super().save() ---- ==== The `.instance` attribute on a form represents the database object that is being modified or created. And I only learned that as I was writing this chapter! There are other ways of getting this to work, including manually creating the object yourself, or using the `commit=False` argument to save, but this way seemed neatest. We'll explore a different way of making a form "know" what list it's for in the next chapter. A quick test run to prove it works: ---- Ran 22 tests in 0.086s OK ---- Finally, we can refactor our views. `new_list()` first: [role="sourcecode"] .src/lists/views.py (ch15l036) ==== [source,python] ---- def new_list(request): form = ItemForm(data=request.POST) if form.is_valid(): nulist = List.objects.create() form.save(for_list=nulist) return redirect(nulist) else: return render(request, "home.html", {"form": form}) ---- ==== Rerun the test to check that everything still passes: ---- Ran 22 tests in 0.086s OK ---- [role="pagebreak-before"] Then, refactor `view_list()`: [role="sourcecode"] .src/lists/views.py (ch15l037) ==== [source,python] ---- def view_list(request, list_id): our_list = List.objects.get(id=list_id) form = ItemForm() if request.method == "POST": form = ItemForm(data=request.POST) if form.is_valid(): form.save(for_list=our_list) return redirect(our_list) return render(request, "list.html", {"list": our_list, "form": form}) ---- ==== We still have full passes: // remove unused imports [role="dofirst-ch15l038"] ---- Ran 22 tests in 0.111s OK ---- And: ---- Ran 4 tests in 14.367s OK ---- Great! Let's commit our changes: [subs="specialcharacters,quotes"] ---- $ *git commit -am "implement custom save method for the form"* ---- Our two views are now looking very much like "normal" Django views: they take information from a user's request, combine it with some custom logic or information from the URL (`list_id`), pass it to a form for validation and possible saving, and then redirect or render a template. Forms and validation are really important in Django—and in web programming, in general—so let's try to make a slightly more complicated one in the next chapter, to learn how to prevent duplicate items. ((("ModelForms", "using save method", startref="ix_MdFsave")))((("", startref="FDVsave14"))) [role="less_space pagebreak-before"] .Tips ******************************************************************************* Thin views:: If you find yourself looking at complex views, and having to write a lot of tests for them, it's time to start thinking about whether that logic could be moved elsewhere: possibly to a form, like we've done here. Another possible place would be a custom method on the model class—and, once the complexity of the app demands it, out of Django-specific files and into your own classes and functions, that capture your core business logic.((("form data validation", "best practices"))) ((("views", "thin versus complex views")))((("thin views versus complex views"))) ((("complex views versus thin views"))) Each test should test one thing:: The heuristic is to be suspicious if there's more than one assertion in a unit test.((("assertions", "one assertion per unit test"))) Sometimes two assertions are closely related, so they belong together. But often your first draft of a test ends up testing multiple behaviours. Therefore, it's worth rewriting it as several tests so that each one can pinpoint specific problems more precisely, and so one failure doesn't mask another. Helper functions can keep your tests from getting too bloated. ((("", startref="UIform14"))) ((("unit tests", "testing only one thing"))) ((("testing best practices"))) Be aware of trade-offs when using frameworks:: When we switched to using a `ModelForm`, we saw that it forced us to change the `name=` attribute in our frontend HTML.((("frameworks", "trade-offs of using"))) Django gave us a lot: it autogenerated the form based on the model, and we have a nice API for doing both validation and saving objects. But we lost something too—we'll revisit this trade-off in the next chapter. ******************************************************************************* ================================================ FILE: chapter_16_advanced_forms.asciidoc ================================================ [[chapter_16_advanced_forms]] == More Advanced Forms Let's look at some more advanced forms usage. We’ve successfully helped our users to avoid blank list items, so now let’s help them to avoid duplicate items as well. Our validation constraint so far has been about preventing blank items, and as you may remember, it turned out that we can enforce that very easily in the frontend. Avoiding duplicate items, however, is less straightforward to do in the frontend (although not impossible, of course), so this chapter will lean more heavily on server-side validation, and bubbling errors from the backend back up to the UI. This chapter goes into the more intricate details of Django's forms framework, so you have my official permission to skim through it if you already know all about customising Django forms and how to display errors in the UI, or if you're reading this book for the TDD rather than for the Django. If you're still learning Django, there's good stuff in here! If you want to just skim-read, that's OK too. Make sure you take a quick look at <<testing-for-silliness>>, and <<what-to-test-in-views>> at the end. [role="pagebreak-before less_space"] === Another FT for Duplicate Items ((("form data validation", "for duplicate items", id="FDVduplicate15"))) ((("functional tests (FTs)", "for duplicate items", secondary-sortas="duplicate items", id="FTduplicate15"))) ((("duplicate items testing", "functional test for", id="DITfunctional15"))) ((("user interactions", "preventing duplicate items", id="UIduplicate15"))) We add a second test method to `ItemValidationTest`, and tell a little story about what we want to see happen when a user tries to enter the same item twice into their to-do list: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch16l001) ==== [source,python] ---- def test_cannot_add_duplicate_items(self): # Edith goes to the home page and starts a new list self.browser.get(self.live_server_url) self.get_item_input_box().send_keys("Buy wellies") self.get_item_input_box().send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Buy wellies") # She accidentally tries to enter a duplicate item self.get_item_input_box().send_keys("Buy wellies") self.get_item_input_box().send_keys(Keys.ENTER) # She sees a helpful error message self.wait_for( lambda: self.assertEqual( self.browser.find_element(By.CSS_SELECTOR, ".invalid-feedback").text, "You've already got this in your list", ) ) ---- ==== Why use two test methods instead of extending one, or instead of creating a new file and class? It's a judgement call. These two feel closely related; they're both about validation on the same input field, so it feels right to keep them in the same file. On the other hand, they're logically separate enough that it's practical to keep them in different methods: // DAVID: This feels a bit hand-wavy. What are we weighing up here? // For example, does 'signal' matter in functional tests? // How about speed? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_list_item_validation*] [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .invalid-feedback; [...] Ran 2 tests in 9.613s ---- // DAVID: Side note: The favicon 404s are getting pretty distracting by this point, I wonder if it would be // worth fixing / silencing that somehow earlier in the book? // HARRY: could do it like this https://stackoverflow.com/a/38917888 OK, so we know the first of the two tests passes now. Is there a way to run just the failing one, I hear you ask? Why, yes indeed: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.\ test_list_item_validation.ItemValidationTest.test_cannot_add_duplicate_items*] [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .invalid-feedback; [...] ---- [role="pagebreak-before"] In any case, let's commit it: [subs="specialcharacters,quotes"] ---- $ *git commit -am"Ft for duplicate item validation"* ---- ==== Preventing Duplicates at the Model Layer ((("model-layer validation", "preventing duplicate items"))) So, if we want to start to implement our actual objective for the chapter, let's write a new test that checks that duplicate items in the same list raise an error: [role="sourcecode"] .src/lists/tests/test_models.py (ch16l002) ==== [source,python] ---- def test_duplicate_items_are_invalid(self): mylist = List.objects.create() Item.objects.create(list=mylist, text="bla") with self.assertRaises(ValidationError): item = Item(list=mylist, text="bla") item.full_clean() ---- ==== And, while it occurs to us, we add another test to make sure we don't overdo it on our integrity constraints: [role="sourcecode"] .src/lists/tests/test_models.py (ch16l003) ==== [source,python] ---- def test_CAN_save_same_item_to_different_lists(self): list1 = List.objects.create() list2 = List.objects.create() Item.objects.create(list=list1, text="bla") item = Item(list=list2, text="bla") item.full_clean() # should not raise ---- ==== I always like to put a little comment for tests that are checking that a particular use case should _not_ raise an error; otherwise, it can be hard to see what's being tested: ---- AssertionError: ValidationError not raised ---- If we want to get it deliberately wrong, we can do this: [role="sourcecode"] .src/lists/models.py (ch16l004) ==== [source,python] ---- class Item(models.Model): text = models.TextField(default="", unique=True) list = models.ForeignKey(List, default=None, on_delete=models.CASCADE) ---- ==== [role="pagebreak-before"] That lets us check that our second test really does pick up on this problem: ---- ERROR: test_CAN_save_same_item_to_different_lists (lists.tests.test_models.List AndItemModelsTest.test_CAN_save_same_item_to_different_lists) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/lists/tests/test_models.py", line 59, in test_CAN_save_same_item_to_different_lists item.full_clean() # should not raise [...] django.core.exceptions.ValidationError: {'text': ['Item with this Text already exists.']} [...] ---- [[testing-for-silliness]] .An Aside on When to Test for Developer Silliness ******************************************************************************* // TODO: i kinda want to back to "stupidity". talk to Rita about it. One of the judgement calls in testing is when you should write tests that sound like "check that we haven't done something weird". ((("developer silliness, when to test for")))In general, you should be wary of these. In this case, we've written a test to check that you can't save duplicate items to the same list. Now, the simplest way to get that test to pass, the way in which you'd write the fewest lines of code, would be to make it impossible to save 'any' duplicate items. That justifies writing another test, despite the fact that it would be a "silly" or "wrong" thing for us to code. But you can't be writing tests for every possible way we could have coded something wrong.footnote:[With that said, you can come pretty close. Once you get comfortable writing tests manually, take a look at https://hypothesis.readthedocs.io[Hypothesis]. It lets you automatically generate input for your tests, covering many more test scenarios than you could realistically type manually. It's not always easy to see how to use it, but for the right kind of problem, it can be very powerful; the very first time I used it, it found a bug!] If you have a function that adds two numbers, you can write a couple of tests: [role="skipme"] [source,python] ---- assert adder(1, 1) == 2 assert adder(2, 1) == 3 ---- But you have the right to assume that the implementation isn't deliberately screwy or perverse: [role="skipme"] [source,python] ---- def adder(a, b): # unlikely code! if a == 3: return 666 else: return a + b ---- One way of putting it is: trust yourself not to do something _deliberately_ silly, but do protect against things that might be _accidentally_ silly. ******************************************************************************* ((("Meta attributes")))((("constraints", "for form input uniqueness, in Meta attributes"))) Just like `ModelForm`, models can use an inner class called `Meta`, and that's where we can implement a constraint that says an item must be unique for a particular list—or, in other words, that `text` and `list` must be unique together: [role="sourcecode"] .src/lists/models.py (ch16l005) ==== [source,python] ---- class Item(models.Model): text = models.TextField(default="") list = models.ForeignKey(List, default=None, on_delete=models.CASCADE) class Meta: unique_together = ("list", "text") ---- ==== And that passes: ---- Ran 24 tests in 0.024s OK ---- You might want to take a quick peek at the https://docs.djangoproject.com/en/5.2/ref/models/options[Django docs on model `Meta` attributes] at this point. [[rewrite-model-test]] ==== Rewriting the Old Model Test That long-winded model test did serendipitously help us find unexpected bugs, but now it's time to rewrite it. I wrote it in a very verbose style to introduce the Django ORM, but in fact, we can get the same coverage from a couple of much shorter tests. Delete `test_saving_and_retrieving_items` and replace it with this: [role="sourcecode"] .src/lists/tests/test_models.py (ch16l006) ==== [source,python] ---- class ListAndItemModelsTest(TestCase): def test_default_text(self): item = Item() self.assertEqual(item.text, "") def test_item_is_related_to_list(self): mylist = List.objects.create() item = Item() item.list = mylist item.save() self.assertIn(item, mylist.item_set.all()) [...] ---- ==== That's more than enough really--a check of the default values of attributes on a freshly initialised model object is enough to sense-check that we've probably set some fields up in 'models.py'. The "item is related to list" test is a real "belt and braces" test to make sure that our foreign key relationship works. While we're at it, we can split this file out into tests for `Item` and tests for `List` (there's only one of the latter, `test_get_absolute_url`): [role="sourcecode"] .src/lists/tests/test_models.py (ch16l007) ==== [source,python] ---- class ItemModelTest(TestCase): def test_default_text(self): [...] class ListModelTest(TestCase): def test_get_absolute_url(self): [...] ---- ==== That's neater and tidier: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test lists*] [...] Ran 25 tests in 0.092s OK ---- ==== Integrity Errors That Show Up on Save ((("data integrity errors")))((("database migrations", "data integrity errors on uniqueness"))) A final aside before we move on. Do you remember the discussion mentioned in <<chapter_14_database_layer_validation>> that some data integrity errors _are_ picked up on save? It all depends on whether the integrity constraint is actually being enforced by the database. Try running `makemigrations` and you'll see that Django wants to add the `unique_together` constraint to the database itself, rather than just having it as an application-layer constraint: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py makemigrations*] Migrations for 'lists': src/lists/migrations/0005_alter_item_unique_together.py ~ Alter unique_together for item (1 constraint(s)) ---- //ch16l005-1 Now let's run the migration: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py migrate*] ---- [role="pagebreak-before less_space"] .What to Do If You See an IntegrityError When Running Migrations ******************************************************************************* When you run the migration, you may encounter the following error: [role="skipme small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py migrate*] Operations to perform: Apply all migrations: auth, contenttypes, lists, sessions Running migrations: Applying lists.0005_alter_item_unique_together... Traceback (most recent call last): [...] sqlite3.IntegrityError: UNIQUE constraint failed: lists_item.list_id, lists_item.text [...] django.db.utils.IntegrityError: UNIQUE constraint failed: lists_item.list_id, lists_item.text ---- The problem is that we have at least one database record that _used_ to be valid but, after introducing our new constraint—the `unique_together`—it's no longer compatible. To fix this problem locally, we can just delete `src/db.sqlite3` and run the migration again. We can do this because the database on our laptop is only used for dev, so the data in it is not important. In <<chapter_18_second_deploy>>, we'll deploy our new code to production, and discuss what to do if we run into migrations and data integrity issues at that point. ******************************************************************************* Now, if we change our duplicate test to do a `.save` instead of a `.full_clean`... [role="sourcecode"] .src/lists/tests/test_models.py (ch16l008) ==== [source,python] ---- def test_duplicate_items_are_invalid(self): mylist = List.objects.create() Item.objects.create(list=mylist, text="bla") with self.assertRaises(ValidationError): item = Item(list=mylist, text="bla") # item.full_clean() item.save() ---- ==== [role="pagebreak-before"] It gives: ---- ERROR: test_duplicate_items_are_invalid (lists.tests.test_models.ItemModelTest.test_duplicate_items_are_invalid) [...] sqlite3.IntegrityError: UNIQUE constraint failed: lists_item.list_id, lists_item.text [...] django.db.utils.IntegrityError: UNIQUE constraint failed: lists_item.list_id, lists_item.text ---- You can see that the error bubbles up from SQLite, and it's a different error from the one we want—an `IntegrityError` instead of a `ValidationError`. Let's revert our changes to the test, and see them all passing again: [role="dofirst-ch16l008-1"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test lists*] [...] Ran 25 tests in 0.092s OK ---- ((("", startref="FTduplicate15")))((("", startref="DITfunctional15")))And now it's time to commit our model-layer changes: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:[<strong>git status</strong>] # should show changes to tests + models and new migration $ pass:[<strong>git add src/lists</strong>] $ pass:[<strong>git diff --staged</strong>] $ pass:[<strong>git commit -m "Implement duplicate item validation at model layer"</strong>] ---- [role="pagebreak-before less_space"] === Experimenting with Duplicate Item Validation at the Views Layer ((("duplicate items testing", "at the views layer", secondary-sortas="views layer"))) Let's try running our FT, to see if that's made any difference. ---- selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: .invalid-feedback; [...] ---- In case you didn't see it as it flew past, the site is 500ing,footnote:[500ing, showing a server error, code 500—of course you can use HTTP status codes as verbs!] as in <<integrity-error-unique-constraint>> (feel free to try it out manually). [[integrity-error-unique-constraint]] .Well, at least it didn't make it into the database image::images/tdd3_1601.png["The Django Debug Page showing an IntegrityError, details 'UNIQUE constraint failed: lists_item.list_id, lists_item.text', and traceback"] [role="pagebreak-before"] We need to be clearer on what we want to happen at the views level. Let's write a unit test to set out our expectations: [role="sourcecode"] .src/lists/tests/test_views.py (ch16l009) ==== [source,python] ---- class ListViewTest(TestCase): [...] def test_for_invalid_input_nothing_saved_to_db(self): [...] def test_for_invalid_input_renders_list_template(self): [...] def test_for_invalid_input_shows_error_on_page(self): [...] def test_duplicate_item_validation_errors_end_up_on_lists_page(self): list1 = List.objects.create() Item.objects.create(list=list1, text="textey") response = self.client.post( f"/lists/{list1.id}/", data={"text": "textey"}, ) expected_error = html.escape("You've already got this in your list") self.assertContains(response, expected_error) # <1> self.assertTemplateUsed(response, "list.html") # <2> self.assertEqual(Item.objects.all().count(), 1) # <3> ---- ==== <1> Here's our main assertion, which is that we want to see a nice error message on the page. <2> Here's where we check that it's landing on the normal list page. <3> And we double-check that we haven't saved anything to the database.footnote:[ Harry, didn't we spend time in the last chapter making sure all the asserts were in different tests? Absolutely yes. Feel free to do that! If I had to justify myself, I'd say that we already have all the granular asserts for _one_ error type, and this really is just a smoke test that an additional error type is also handled. So, arguably, it doesn't need to be so granular.] That test confirms that the `IntegrityError` is bubbling all the way up: ---- File "...goat-book/src/lists/views.py", line 28, in view_list form.save(for_list=our_list) ~~~~~~~~~^^^^^^^^^^^^^^^^^^^ [...] django.db.utils.IntegrityError: UNIQUE constraint failed: lists_item.list_id, lists_item.text ---- We want to avoid integrity errors! Ideally, we want the call to `is_valid()` to somehow notice the duplication error before we even try to save. But to do that, our form will need to know in advance what list it's being used for. Let's put a skip on this test for now: [role="sourcecode"] .src/lists/tests/test_views.py (ch16l010) ==== [source,python] ---- from unittest import skip [...] @skip def test_duplicate_item_validation_errors_end_up_on_lists_page(self): ---- ==== // IDEA: alternatively, try/except on the validation error, // get everything passing, then refactor to use a form. // use the forms tests to explore the api (introduce the idea of a spike) // maybe get it working, show how the forms-layer tests are annoying // and switch to only views-layer tests === A More Complex Form to Handle Uniqueness Validation ((("duplicate items testing", "complex form for"))) ((("uniqueness validation", seealso="duplicate items testing"))) The form to create a new list only needs to know one thing: the new item text. A form validating that list items are unique will need to know what list they're in as well. Just as we overrode the save method on our `ItemForm`, this time we'll override the _constructor_ on our new form class so that it knows what list it applies to. Let's duplicate our tests from the previous form, tweaking them slightly: [role="sourcecode"] .src/lists/tests/test_forms.py (ch16l011) ==== [source,python] ---- [...] from lists.forms import ( DUPLICATE_ITEM_ERROR, EMPTY_ITEM_ERROR, ExistingListItemForm, ItemForm, ) [...] class ExistingListItemFormTest(TestCase): def test_form_renders_item_text_input(self): list_ = List.objects.create() form = ExistingListItemForm(for_list=list_) # <1> self.assertIn('placeholder="Enter a to-do item"', form.as_p()) def test_form_validation_for_blank_items(self): list_ = List.objects.create() form = ExistingListItemForm(for_list=list_, data={"text": ""}) self.assertFalse(form.is_valid()) self.assertEqual(form.errors["text"], [EMPTY_ITEM_ERROR]) def test_form_validation_for_duplicate_items(self): list_ = List.objects.create() Item.objects.create(list=list_, text="no twins!") form = ExistingListItemForm(for_list=list_, data={"text": "no twins!"}) self.assertFalse(form.is_valid()) self.assertEqual(form.errors["text"], [DUPLICATE_ITEM_ERROR]) ---- ==== <1> We're specifying that our new `ExistingListItemForm` will take an argument `for_list=` in its constructor, to be able to specify which list the item is for. Next we iterate through a few TDD cycles until we get a form with a custom constructor, which just ignores its `for_list` argument. (I won't show them all, but I'm sure you'll do them, right? Remember, the Goat sees all.) [role="sourcecode"] .src/lists/forms.py (ch16l012) ==== [source,python] ---- DUPLICATE_ITEM_ERROR = "You've already got this in your list" [...] class ExistingListItemForm(forms.models.ModelForm): def __init__(self, for_list, *args, **kwargs): super().__init__(*args, **kwargs) ---- ==== At this point, our error should be: ---- ValueError: ModelForm has no model class specified. ---- Then, let's see if making it inherit from our existing form helps: [role="sourcecode"] .src/lists/forms.py (ch16l013) ==== [source,python] ---- class ExistingListItemForm(ItemForm): def __init__(self, for_list, *args, **kwargs): super().__init__(*args, **kwargs) ---- ==== Yes, that takes us down to just one failure: ---- FAIL: test_form_validation_for_duplicate_items (lists.tests.test_forms.Existing ListItemFormTest.test_form_validation_for_duplicate_items) [...] self.assertFalse(form.is_valid()) AssertionError: True is not false ---- The next step requires a little knowledge of Django's validation system—you can read up on it in the Django docs on https://docs.djangoproject.com/en/5.2/ref/models/instances/#validating-objects[model validation] and https://docs.djangoproject.com/en/5.2/ref/forms/validation[form validation]. [role="pagebreak-before"] We can customise validation for a field by implementing a `clean_<fieldname>()` method, and raising a `ValidationError` if the field is invalid: [role="sourcecode"] .src/lists/forms.py (ch16l013-1) ==== [source,python] ---- from django.core.exceptions import ValidationError [...] class ExistingListItemForm(ItemForm): def __init__(self, for_list, *args, **kwargs): super().__init__(*args, **kwargs) self.instance.list = for_list def clean_text(self): text = self.cleaned_data["text"] if self.instance.list.item_set.filter(text=text).exists(): raise forms.ValidationError(DUPLICATE_ITEM_ERROR) return text ---- ==== That makes the tests happy: ---- Found 29 test(s). [...] OK (skipped=1) ---- We're there! A quick commit: [role="skipme small-code"] [subs="specialcharacters,quotes"] ---- $ *git diff* $ *git add src/lists/forms.py src/lists/tests/test_forms.py* $ *git commit -m "implement ExistingListItemForm, add DUPLICATE_ITEM_ERROR message"* ---- === Using the Existing List Item Form in the List View ((("duplicate items testing", "in the list view", secondary-sortas="list view", id="DITlist15"))) Now let's see if we can put this form to work in our view. We remove the skip and, while we're at it, we can use our new constant: [role="sourcecode"] .src/lists/tests/test_views.py (ch16l014) ==== [source,python] ---- from lists.forms import ( DUPLICATE_ITEM_ERROR, EMPTY_ITEM_ERROR, ) [...] def test_duplicate_item_validation_errors_end_up_on_lists_page(self): [...] expected_error = html.escape(DUPLICATE_ITEM_ERROR) self.assertContains(response, expected_error) [...] ---- ==== [role="pagebreak-before"] We see our `IntegrityError` once again: ---- django.db.utils.IntegrityError: UNIQUE constraint failed: lists_item.list_id, lists_item.text ---- Our fix for this is to switch to using the new form class: [role="sourcecode"] .src/lists/views.py (ch16l016) ==== [source,python] ---- from lists.forms import ExistingListItemForm, ItemForm [...] def view_list(request, list_id): our_list = List.objects.get(id=list_id) form = ExistingListItemForm(for_list=our_list) # <1> if request.method == "POST": form = ExistingListItemForm(for_list=our_list, data=request.POST) # <1> if form.is_valid(): form.save(for_list=our_list) # <2> [...] [...] ---- ==== <1> We swap out `ItemForm` for `ExistingListItemForm`, and pass in the `for_list=`. <2> This is a bit annoying—we're duplicating the `for_list=` argument. This form should already know this! ==== Customising the Save Method on Our New Form Programming by wishful thinking, as always. Let's specify in our _views.py_ that we wish we could call `save()` without the duplicated argument: [role="sourcecode"] .src/lists/views.py (ch16l016-1) ==== [source,diff] ---- @@ -25,6 +25,6 @@ def view_list(request, list_id): if request.method == "POST": form = ExistingListItemForm(for_list=our_list, data=request.POST) if form.is_valid(): - form.save(for_list=our_list) + form.save() return redirect(our_list) return render(request, "list.html", {"list": our_list, "form": form}) ---- ==== That gives us a failure as expected: ---- File "...goat-book/src/lists/views.py", line 28, in view_list form.save() ~~~~~~~~~^^ TypeError: ItemForm.save() missing 1 required positional argument: 'for_list' ---- [role="pagebreak-before"] Let's drop down to the forms level, and write another unit test for how we want our save method to work: [role="sourcecode"] .src/lists/tests/test_forms.py (ch16l017) ==== [source,python] ---- class ExistingListItemFormTest(TestCase): [...] def test_form_save(self): mylist = List.objects.create() form = ExistingListItemForm(for_list=mylist, data={"text": "hi"}) self.assertTrue(form.is_valid()) new_item = form.save() self.assertEqual(new_item, Item.objects.get()) [...] ---- ==== We can make our form call the grandparent save method: [role="sourcecode"] .src/lists/forms.py (ch16l018) ==== [source,python] ---- class ExistingListItemForm(ItemForm): [...] def save(self): return forms.models.ModelForm.save(self) # <1> ---- ==== <1> This manually calls the grandparent `save()`. Personal opinion here: I could have used `super()`, but I prefer not to use `super()` when it requires arguments, say, to get a grandparent. I find Python 3's `super()` with no arguments is awesome to get the immediate parent. Anything else is too error-prone—and, besides, I find it ugly. YMMV. // SEBASTIAN: IMHO it's actually Django's fault that it handles code reuse using inheritance and methods overriding // Wouldn't do the same thing, but it's your book and your opinion so I shall close my mouth :D OK, how does that look? Yep, both the forms level and views level tests now pass: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test lists*] [...] Ran 30 tests in 0.082s OK ---- Time to see what our FTs think! [role="pagebreak-before less_space"] === The FTs Pick Up an Issue with Bootstrap Classes Unfortunately, the FTs are telling ((("Bootstrap", "uniqueness constraint, failure on")))us we're not done: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_list_item_validation*] [...] FAIL: test_cannot_add_duplicate_items [...] ---------------------------------------------------------------------- [...] AssertionError: '' != "You've already got this in your list" + You've already got this in your list ---- Let's spin up the server with `runserver` and try it out manually—with DevTools open—to see what's going on. If you look through the HTML, you'll see our error `div` is there, with the correct error text, but it's greyed out, indicating that it's hidden (as in <<devtools_error_div_hidden>>). [[devtools_error_div_hidden]] .Our error `div` is there but it's hidden image::images/tdd3_1602.png["Our page has a duplicate item in the table and in the form, and devtools is open showing us that the error IS actually in the page HTML, but it's greyed out, to indicate that it is hidden."] I had to dig through https://getbootstrap.com/docs/5.2/forms/validation/#server-side[the docs] a little, but it turns out that Bootstrap requires form elements with errors to have _another_ custom class, `is-invalid`. You can actually try this out in DevTools! If you double-click, you can edit the HTML and add the class, as in <<devtools_closeup_edit_html>>. [[devtools_closeup_edit_html]] .Hack it in manually—yay image::images/tdd3_1603.png["A close-up on the Devtools HTML inspector, showing one of the HTML elements open for editing. I'm adding the is-invalid class to the main input field."] === Conditionally Customising CSS Classes for Invalid Forms Speaking of hackery, I'm starting to get a bit nervous about the amount of hackery we're doing in our forms now, but let's try getting this to work by doing _even more_ customisation in our forms. We want this behaviour for both types of form really, so it can go in the tests for the parent `ItemForm` class: [role="sourcecode"] .src/lists/tests/test_forms.py (ch16l019-1) ==== [source,python] ---- class ItemFormTest(TestCase): def test_form_item_input_has_placeholder_and_css_classes(self): [...] def test_form_validation_for_blank_items(self): [...] def test_invalid_form_has_bootstrap_is_invalid_css_class(self): form = ItemForm(data={"text": ""}) self.assertFalse(form.is_valid()) field = form.fields["text"] self.assertEqual( field.widget.attrs["class"], # <1> "form-control form-control-lg is-invalid", ) def test_form_save_handles_saving_to_a_list(self): [...] ---- ==== <1> Here's where you can inspect the `class` attribute on the input field `widget`. [role="pagebreak-before"] And here's how we can make it work, by overriding the `is_valid()` method: [role="sourcecode"] .src/lists/forms.py (ch16l019-2) ==== [source,python] ---- class ItemForm(forms.models.ModelForm): class Meta: [...] def is_valid(self): result = super().is_valid() # <1> if not result: self.fields["text"].widget.attrs["class"] += " is-invalid" # <2> return result # <3> def save(self, for_list): [...] ---- ==== <1> We make sure to call the parent `is_valid()` method first, so we can do all the normal built-in validation. <2> Here's how we add the extra CSS class to our `widget`. <3> And we remember to return the result. It's not _too_ bad—but, as I say, I'm getting nervous about the amount of fiddly code in our forms classes. Let's make a note on our scratchpad, and come back to it when our FT is passing perhaps: [role="scratchpad"] ***** * Review amount of hackery in forms.py. ***** [role="pagebreak-before"] Speaking of our FT, let's see how it does now: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_list_item_validation*] [...] ====================================================================== FAIL: test_cannot_add_empty_list_items (functional_tests.test_list_item_validat ion.ItemValidationTest.test_cannot_add_empty_list_items) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/functional_tests/test_list_item_validation.py", line 47, in test_cannot_add_empty_list_items self.wait_for_row_in_list_table("2: Make tea") File "...goat-book/src/functional_tests/base.py", line 37, in wait_for_row_in_list_table self.assertIn(row_text, [row.text for row in rows]) AssertionError: '2: Make tea' not found in ['1: Make tea', '2: Purchase milk'] ---- Ooops; what happened here (<<wrong_order_list>>)? [[wrong_order_list]] .The cart is before the horse image::images/tdd3_1604.png["A screenshot of the todo list from the FT, with Make Tea appearing above Purchase Milk"] ==== A Little Digression on Queryset Ordering and String Representations ((("queryset ordering", id="queryset15"))) ((("string representations", id="triprep15"))) Something seems to be going wrong with the ordering of our list items. Trying to fix this by iterating against an FT is going to be slow, so let's work at the unit test level. We'll add a test that checks that list items are ordered in the sequence they are inserted. You'll have to forgive me if I jump straight to the right answer, using intuition borne of long experience, but I suspect that it might be sorting alphabetically based on list text instead (what else would it sort by after all?), so I'll pick some text values designed to test that hypothesis: [role="sourcecode"] .src/lists/tests/test_models.py (ch16l020) ==== [source,python] ---- class ListModelTest(TestCase): def test_get_absolute_url(self): [...] def test_list_items_order(self): list1 = List.objects.create() item1 = Item.objects.create(list=list1, text="i1") item2 = Item.objects.create(list=list1, text="item 2") item3 = Item.objects.create(list=list1, text="3") self.assertEqual( list1.item_set.all(), [item1, item2, item3], ) ---- ==== TIP: FTs are a slow feedback loop. Switch to unit tests when you want to drill down on edge case bugs. That gives us a new failure, but it's not very readable: ---- AssertionError: <QuerySet [<Item: Item object (3)>, <Item[40 chars]2)>]> != [<Item: Item object (1)>, <Item: Item obj[29 chars](3)>] ---- We need a better string representation for our `Item` model. Let's add another unit test: [role="sourcecode"] .src/lists/tests/test_models.py (ch16l021) ==== [source,python] ---- class ItemModelTest(TestCase): [...] def test_string_representation(self): item = Item(text="some text") self.assertEqual(str(item), "some text") ---- ==== NOTE: Ordinarily, you would be wary of adding more failing tests when you already have some--it makes reading test output that much more complicated, and just generally makes you nervous. Will we ever get back to a working state? In this case, they're all quite simple tests, so I'm not worried. That gives us: ---- AssertionError: 'Item object (None)' != 'some text' ---- [role="pagebreak-before"] And it also gives us the other two failures. Let's start fixing them all now: [role="sourcecode"] .src/lists/models.py (ch16l022) ==== [source,python] ---- class Item(models.Model): [...] def __str__(self): return self.text ---- ==== Now we're down to one failure, and the ordering test has a more readable failure message: ---- AssertionError: <QuerySet [<Item: 3>, <Item: i1>, <Item: item 2>]> != [<Item: i1>, <Item: item 2>, <Item: 3>] ---- That confirms our suspicion that the ordering was alphabetical. We can fix that in the `class Meta`: [role="sourcecode"] .src/lists/models.py (ch16l023) ==== [source,python] ---- class Item(models.Model): [...] class Meta: ordering = ("id",) unique_together = ("list", "text") ---- ==== Does that work? ---- AssertionError: <QuerySet [<Item: i1>, <Item: item 2>, <Item: 3>]> != [<Item: i1>, <Item: item 2>, <Item: 3>] ---- Urp? It has worked; you can see the items _are_ in the same order, but the tests are confused. I keep running into this problem actually--Django QuerySets don't compare well with lists. We can fix it by converting the QuerySet to a list in our test:footnote:[You could also check out `assertSequenceEqual` from `unittest`, and `assertQuerysetEqual` from Django's test tools—although I confess, when I last looked at `assertQuerysetEqual`, I was quite baffled...] [role="sourcecode"] .src/lists/tests/test_models.py (ch16l024) ==== [source,python] ---- self.assertEqual( list(list1.item_set.all()), [item1, item2, item3], ) ---- ==== // SEBASTIAN: If it's not too much of Django internals, maybe it's worth to mention // how models instances are compared (or at least leave a link for curious readers) // That said, if it wasn't shown before in the book // https://docs.djangoproject.com/en/5.2/topics/db/queries/#comparing-objects [role="pagebreak-before"] That works; we get a fully passing unit test suite: ---- Ran 33 tests in 0.034s OK ---- ((("", startref="triprep15"))) ((("", startref="queryset15"))) We do need a migration for that ordering change though: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py makemigrations*] Migrations for 'lists': src/lists/migrations/0006_alter_item_options.py ~ Change Meta options on item ---- //ch16l024-1 And as a final check, we rerun 'all' the FTs: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests*] [...] --------------------------------------------------------------------- Ran 5 tests in 19.048s OK ---- Hooray! Time for a final commit: [subs="specialcharacters,quotes"] ---- *git status* *git add src* *git commit -m "Add is-invalid css class, fix list item ordering"* ---- ((("", startref="DITlist15"))) === On the Trade-offs of Django ModelForms, and Frameworks in General Let's come back to((("ModelForms", "trade-offs of")))((("frameworks", "trade-offs of using"))) our scratchpad item: [role="scratchpad"] ***** * Review amount of hackery in forms.py. ***** [role="pagebreak-before"] Let's take a look at the current state of our forms classes. We've got a real mix of presentation logic, validation logic, and ORM/storage logic: [role="sourcecode currentcontents"] .src/lists/forms.py ==== [source,python] ---- class ItemForm(forms.models.ModelForm): class Meta: model = Item fields = ("text",) widgets = { "text": forms.widgets.TextInput( attrs={ "placeholder": "Enter a to-do item", # <1> "class": "form-control form-control-lg", # <1> } ), } error_messages = {"text": {"required": EMPTY_ITEM_ERROR}} def is_valid(self): result = super().is_valid() if not result: self.fields["text"].widget.attrs["class"] += " is-invalid" # <1> return result def save(self, for_list): # <3> self.instance.list = for_list return super().save() class ExistingListItemForm(ItemForm): def __init__(self, for_list, *args, **kwargs): super().__init__(*args, **kwargs) self.instance.list = for_list # <3> def clean_text(self): text = self.cleaned_data["text"] if self.instance.list.item_set.filter(text=text).exists(): <2> raise forms.ValidationError(DUPLICATE_ITEM_ERROR) <2> return text def save(self): return forms.models.ModelForm.save(self) # <3> ---- ==== <1> Presentation logic <2> Validation logic <3> ORM/storage logic //// I'm also nervous about the fact that we're overriding parts of the forms API, like `is_valid()`, and `save()`. Not only that, but; [role="sourcecode currentcontents"] .src/lists/forms.py ==== [source,python] ---- class ItemForm(forms.models.ModelForm): def save(self, for_list): # <1> [...] class ExistingListItemForm(ItemForm): def __init__(self, for_list, *args, **kwargs): # <2> [...] def save(self): # <3> return forms.models.ModelForm.save(self) ---- ==== <1> Here we not only override the forms API method, but we actually _change_ the API, meaning that `ItemForm` no longer matches the normal forms API <2> It's the same here where we override the constructor to add the `for_list` argument. <3> And in this one, we change the `save()` API _again_, so the API isn't even consistent within our own inheritance hierarchy. Without wanting to get all OO-nerdy, this is a violation of the Liskov Substitution Principle, which basically says that subclasses should look like their parents.footnote:[ Read a better write-up here: https://realpython.com/solid-principles-python/] //// I think what's happened is that we've reached the limits of the Django forms framework's sweet spot. `ModelForms` can be great _because_ they can do presentation, validation, and database storage all in one go, so you can get a lot done without much code. But once you want to customise the default behaviours for each of those things, the code you _do_ end up writing starts to get hard to understand. Let's see what things would look like if we tried to: . Move the responsibility for presentation and the rendering of HTML back into the template. . Stop using `ModelForm` and do any database logic more explicitly, with less magic. // 3. Tried to remove some of the Liskov violations .Another Flip-flop! ******************************************************************************* We spent most of the last chapter switching from handcrafted HTML to having our form autogenerated by Django, and now we're switching back. It's a little frustrating, and I could have gone back and changed the book's outline to avoid the back and forth, but I prefer to show software development as it really is. We often try things out and end up changing our minds. Particularly with frameworks like Django, you can find yourself taking advantage of autogenerated shortcuts for as long as they work. But at some point, you meet the limits of what the framework designers have anticipated, and it's time to go back to doing the work yourself. Frameworks have trade-offs. It doesn't mean you should always reinvent the wheel! It’s OK to cut yourself some slack for “wasting time” on avenues that don’t work out, or revisiting decisions that worked well in the past, but don't work so well now. ******************************************************************************* ==== Moving Presentation Logic Back into the Template We're talking about another refactor here; we want to move some functionality out of the form and into the template/views layer.((("templates", "moving presentation logic from form back to")))((("presentation logic, moving from form to template"))) How do we make sure we've got good test coverage? * We currently have some tests for the CSS classes including `is-invalid` in _test_forms.py_. * We have some tests of some form attributes in _test_views.py_—e.g., the asserts on the input's `name`. * And the FTs, ultimately, will tell us if things "really work" or not, including testing the interaction between our HTML, Bootstrap, and the browser (e.g., CSS visibility). What we are learning is that the things we're testing in _test_forms.py_ will need to move. TIP: Lower-level tests are good for exploring an API, but they are tightly coupled to it. Higher-level tests can enable more refactoring. Here's one way to write that kind of test: [role="sourcecode"] .src/lists/tests/test_views.py (ch16l025-1) ==== [source,python] ---- class ListViewTest(TestCase): [...] def test_for_invalid_input_shows_error_on_page(self): [...] def test_for_invalid_input_sets_is_invalid_class(self): response = self.post_invalid_input() parsed = lxml.html.fromstring(response.content) [input] = parsed.cssselect("input[name=text]") self.assertIn("is-invalid", input.get("class")) def test_duplicate_item_validation_errors_end_up_on_lists_page(self): [...] ---- ==== That's green straight away: ---- Ran 34 tests in 0.040s OK ---- As always, it's nice to deliberately break it, to see whether it has a nice failure message, if nothing else. Let's do that in _forms.py_: [role="sourcecode"] .src/lists/forms.py (ch16l025-2) ==== [source,diff] ---- @@ -24,7 +24,7 @@ class ItemForm(forms.models.ModelForm): def is_valid(self): result = super().is_valid() if not result: - self.fields["text"].widget.attrs["class"] += " is-invalid" + self.fields["text"].widget.attrs["class"] += " boo!" return result def save(self, for_list): ---- ==== [role="pagebreak-before"] Reassuringly, both our old test and the new one fail: ---- [...] ====================================================================== FAIL: test_invalid_form_has_bootstrap_is_invalid_css_class (lists.tests.test_fo rms.ItemFormTest.test_invalid_form_has_bootstrap_is_invalid_css_class) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/lists/tests/test_forms.py", line 30, in test_invalid_form_has_bootstrap_is_invalid_css_class self.assertEqual( ~~~~~~~~~~~~~~~~^ field.widget.attrs["class"], ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ "form-control form-control-lg is-invalid", ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ AssertionError: 'form-control form-control-lg boo!' != 'form-control form-control-lg is-invalid' - form-control form-control-lg boo! ? ^^^^ + form-control form-control-lg is-invalid ? ^^^^^^^^^^ ====================================================================== FAIL: test_for_invalid_input_sets_is_invalid_class (lists.tests.test_views.List ViewTest.test_for_invalid_input_sets_is_invalid_class) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/lists/tests/test_views.py", line 129, in test_for_invalid_input_sets_is_invalid_class self.assertIn("is-invalid", input.get("class")) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'is-invalid' not found in 'form-control form-control-lg boo!' --------------------------------------------------------------------- Ran 34 tests in 0.039s FAILED (failures=2) ---- Let's revert that and get back to passing. So, rather than using the `{{ form.text }}` magic in our template, let's bring back our handcrafted HTML. It'll be longer, but at least all of our Bootstrap classes will be in one place, where we expect them, in the template: [role="sourcecode dofirst-ch16l025-3"] .src/lists/templates/base.html (ch16l025-4) ==== [source,diff] ---- @@ -16,10 +16,22 @@ <h1 class="display-1 mb-4">{% block header_text %}{% endblock %}</h1> <form method="POST" action="{% block form_action %}{% endblock %}" > - {{ form.text }} {% csrf_token %} + <input <1> + id="id_text" + name="text" + class="form-control <2> + form-control-lg + {% if form.errors %}is-invalid{% endif %}" + placeholder="Enter a to-do item" + value="{{ form.text.value | default:'' }}" <3> + aria-describedby="id_text_feedback" <4> + required + /> {% if form.errors %} - <div class="invalid-feedback">{{ form.errors.text }}</div> + <div id="id_text_feedback" class="invalid-feedback"> <4> + {{ form.errors.text.0 }} <5> + </div> {% endif %} </form> </div> ---- ==== <1> Here's our artisan `<input>` once again, and the most important custom setting will be its `class` attributes. <2> As you can see, we can use conditionals even for providing additional ++class++-es.footnote:[ We've split the `input` tag across multiple lines so it fits nicely on the screen. If you've not seen that before, it may look a little weird, but I promise it is valid HTML. You don't have to use it if you don't like it though.] <3> The `| default` "filter" is a way to avoid the string "None" from showing up as the value in our input field. <4> We add an `id` to the error message to be able to use `aria-describedby` on the input, as recommended in the Bootstrap docs; it makes the error message more accessible to screen readers. <5> If you just try to use `form.errors.text`, you'll see that Django injects a `<ul>` list, because the forms framework can report multiple errors for each field. We know we've only got one, so we can use use `form.errors.text.0`. // TODO: show a screenshot of this bullet point earlier That passes: ---- Ran 34 tests in 0.034s OK ---- Out of curiosity, let's try a deliberate failure here: [role="sourcecode"] .src/lists/templates/base.html (ch16l025-5) ==== [source,diff] ---- @@ -22,7 +22,7 @@ name="text" class="form-control form-control-lg - {% if form.errors %}is-invalid{% endif %}" + {% if form.errors %}isnt-invalid{% endif %}" placeholder="Enter a to-do item" value="{{ form.text.value | default:'' }}" aria-describedby="id_text_feedback" ---- ==== The failure looks like this: ---- self.assertIn("is-invalid", input.get("class")) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'is-invalid' not found in 'form-control\n form-control-lg\n isnt-invalid' ---- [role="pagebreak-before"] Hmm, that's not ideal actually. Let's tweak our assert: [role="sourcecode"] .src/lists/tests/test_views.py (ch16l025-6) ==== [source,python] ---- def test_for_invalid_input_sets_is_invalid_class(self): response = self.post_invalid_input() parsed = lxml.html.fromstring(response.content) [input] = parsed.cssselect("input[name=text]") self.assertIn("is-invalid", set(input.classes)) # <1> ---- ==== <1> Rather than using `get("class")`, which returns a raw string, `lxml` can give us the classes as a list (well, actually a special object, but one that we can turn into a set). That's more semantically correct, and gives a better error message: ---- self.assertIn("is-invalid", set(input.classes)) ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: 'is-invalid' not found in {'form-control', 'isnt-invalid', 'form-control-lg'} ---- OK, that's good; we can revert the deliberate mistake in _base.html_. Let's do a quick FT run to check we've got it right: [role="dofirst-ch16l025-7"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_list_item_validation*] Found 2 test(s). [...] OK ---- Good! [role="pagebreak-before less_space"] ==== Tidying Up the Forms Now let's start tidying up our forms.((("presentation-layer tests, deleting from ItemFormTest"))) We can start by deleting the three presentation-layer tests from `ItemFormTest`: [role="sourcecode"] .src/lists/tests/test_forms.py (ch16l026) ==== [source,diff] ---- @@ -10,28 +10,11 @@ from lists.models import Item, List class ItemFormTest(TestCase): - def test_form_item_input_has_placeholder_and_css_classes(self): - form = ItemForm() - - rendered = form.as_p() - - self.assertIn('placeholder="Enter a to-do item"', rendered) - self.assertIn('class="form-control form-control-lg"', rendered) - def test_form_validation_for_blank_items(self): form = ItemForm(data={"text": ""}) self.assertFalse(form.is_valid()) self.assertEqual(form.errors["text"], [EMPTY_ITEM_ERROR]) - def test_invalid_form_has_bootstrap_is_invalid_css_class(self): - form = ItemForm(data={"text": ""}) - self.assertFalse(form.is_valid()) - field = form.fields["text"] - self.assertEqual( - field.widget.attrs["class"], - "form-control form-control-lg is-invalid", - ) - def test_form_save_handles_saving_to_a_list(self): mylist = List.objects.create() form = ItemForm(data={"text": "do me"}) @@ -42,11 +25,6 @@ class ItemFormTest(TestCase): class ExistingListItemFormTest(TestCase): - def test_form_renders_item_text_input(self): - list_ = List.objects.create() - form = ExistingListItemForm(for_list=list_) - self.assertIn('placeholder="Enter a to-do item"', form.as_p()) - def test_form_validation_for_blank_items(self): list_ = List.objects.create() form = ExistingListItemForm(for_list=list_, data={"text": ""}) ---- ==== [role="pagebreak-before"] And now((("ItemForm class, removing custom logic from"))) we can remove all that custom logic from the base `ItemForm` class: [role="sourcecode dofirst-ch16l027-1"] .src/lists/forms.py (ch16l027) ==== [source,diff] ---- @@ -11,22 +11,8 @@ class ItemForm(forms.models.ModelForm): class Meta: model = Item fields = ("text",) - widgets = { - "text": forms.widgets.TextInput( - attrs={ - "placeholder": "Enter a to-do item", - "class": "form-control form-control-lg", - } - ), - } error_messages = {"text": {"required": EMPTY_ITEM_ERROR}} - def is_valid(self): - result = super().is_valid() - if not result: - self.fields["text"].widget.attrs["class"] += " is-invalid" - return result - def save(self, for_list): self.instance.list = for_list return super().save() ---- ==== Deleting code, yay! At this point we should be down to 31 passing tests: ---- Ran 31 tests in 0.024s OK ---- ==== Switching Back to Simple Forms Now let's change our forms away from being `ModelForms` and back to regular forms. ((("ModelForms", "switching from to simple forms")))We'll keep the `save()` methods for now, but we'll switch to using the ORM more explicitly, rather than relying on the `ModelForm` magic: [role="sourcecode"] .src/lists/forms.py (ch16l028) ==== [source,diff] ---- @@ -7,27 +7,29 @@ EMPTY_ITEM_ERROR = "You can't have an empty list item" DUPLICATE_ITEM_ERROR = "You've already got this in your list" -class ItemForm(forms.models.ModelForm): - class Meta: - model = Item - fields = ("text",) - error_messages = {"text": {"required": EMPTY_ITEM_ERROR}} +class ItemForm(forms.Form): + text = forms.CharField( + error_messages={"required": EMPTY_ITEM_ERROR}, + required=True, + ) def save(self, for_list): - self.instance.list = for_list - return super().save() + return Item.objects.create( + list=for_list, + text=self.cleaned_data["text"], + ) class ExistingListItemForm(ItemForm): def __init__(self, for_list, *args, **kwargs): super().__init__(*args, **kwargs) - self.instance.list = for_list + self._for_list = for_list def clean_text(self): text = self.cleaned_data["text"] - if self.instance.list.item_set.filter(text=text).exists(): + if self._for_list.item_set.filter(text=text).exists(): raise forms.ValidationError(DUPLICATE_ITEM_ERROR) return text def save(self): - return forms.models.ModelForm.save(self) + return super().save(for_list=self._for_list) ---- ==== We should still have passing tests at this point: ---- Ran 31 tests in 0.026s OK ---- And we're in a better place I think! //// We still have the Liskov violations on the `__init__()` and `save()`, but perhaps we can live with those for now. TODO: start by addressing this in 15, ch14l034, no need to pass the `for_list=` into the save() method. Then the custom constructor //// === Wrapping Up: What We've Learned About Testing Django ((("class-based generic views (CBGVs)", "key tests and assertions"))) ((("Django framework", "class-based generic views"))) We're now at a point where our app looks a lot more like a "standard" Django app, and it implements the three common Django layers: models, forms, and views. We no longer have any "training wheel” tests, and our code looks pretty much like code we'd be happy to see in a real app.((("models, forms, and views (Django layers)"))) We have one unit test file for each of our key source code files. Here's a recap of the biggest (and highest-level) one: _test_views_. [[what-to-test-in-views]] .Wrap-Up: What to Test in Views ****************************************************************************** By way of a recap, let's see an outline of all the test methods and main assertions in our `test_views`. ((("Test-Driven Development (TDD)", "testing in views")))This isn't to say you should copy-paste these exactly—it's more like a list of things you should at least consider testing: [role="sourcecode skipme small-code"] .src/lists/tests/test_views.py, selected test methods and asserts ==== [source,python] ---- class ListViewTest(TestCase): def test_uses_list_template(self): response = self.client.get(f"/lists/{mylist.id}/") # <1> self.assertTemplateUsed(response, "list.html") # <2> def test_renders_input_form(self): parsed = lxml.html.fromstring(response.content) # <3> self.assertIn("text", [input.get("name") for input in inputs]) # <3> def test_displays_only_items_for_that_list(self): self.assertContains(response, "itemey 1") # <4> self.assertContains(response, "itemey 2") # <4> self.assertNotContains(response, "other list item") # <4> def test_can_save_a_POST_request_to_an_existing_list(self): self.assertEqual(new_item.text, "A new item for an existing list") # <5> def test_POST_redirects_to_list_view(self): self.assertRedirects(response, f"/lists/{correct_list.id}/") # <5> def test_for_invalid_input_nothing_saved_to_db(self): self.assertEqual(Item.objects.count(), 0) # <6> def test_for_invalid_input_renders_list_template(self): self.assertEqual(response.status_code, 200) # <6> self.assertTemplateUsed(response, "list.html") # <6> def test_for_invalid_input_shows_error_on_page(self): self.assertContains(response, html.escape(EMPTY_ITEM_ERROR)) # <6> def test_duplicate_item_validation_errors_end_up_on_lists_page(self): self.assertContains(response, expected_error) # <7> self.assertTemplateUsed(response, "list.html") # <7> self.assertEqual(Item.objects.all().count(), 1) # <7> ---- ==== <1> Use the Django test client. <2> Optionally (this is a bit of an implementation detail), check the template used. <3> Check that key parts of your HTML are present. Things that are critical to the integration of frontend and backend are good candidates, like form action and input `name` attributes. Using `lxml` might be overkill, but it does give you less brittle tests. <4> Think about smoke-testing any other template contents, or any logic in the template: any `{% for %}` or `{% if %}` might deserve a check. <5> For POST requests, test the valid case via its database side effects, and the redirect response. <6> For invalid requests, it's worth a basic check that errors make it back to the template. <7> You don't _always_ have to have ultra-granular tests though. // TODO: link // If you'd like to see a worked example of a major refactor, // enabled by these tests, // check out // <<appendix_Django_Class-Based_Views>> ((("", startref="FDVduplicate15")))((("", startref="UIduplicate15"))) ****************************************************************************** Next, we'll try to make our data validation more friendly by using a bit of client-side code. Uh-oh, you know what that means... ================================================ FILE: chapter_17_javascript.asciidoc ================================================ [[chapter_17_javascript]] == A Gentle Excursion into JavaScript [quote, Geoffrey Willans, English author and journalist] ______________________________________________________________ You can never understand one language until you understand at least two. ______________________________________________________________ Our new validation logic is good, but wouldn't it be nice if the duplicate-item error messages disappeared once the user started fixing the problem, just like our nice HTML5 validation errors do? Try it--spin up the site with `./src/manage.py runserver`, start a list, and if you try to submit an empty item, you get the "Please fill out this field" pop-up, and it disappears as soon as you enter some text. By contrast, enter an item twice, you get the "You've already got this in your list" message in red—and even if you edit your submission to something valid, the error stays there until you submit the form (see <<duplicate_item_error>>). [[duplicate_item_error]] .But I've fixed it! image::images/tdd3_1701.png["A screenshot of the 'Please fill out this field' error in red, still shown despite the fact that the input value has been modified to be different from the existing item in the list"] To get that error to disappear dynamically, we'd need a teeny-tiny bit of JavaScript.((("JavaScript"))) Python is a delightful language to program in. JavaScript wasn't always that. But many of the rough edges have been smoothed off, and I think it's fair to say that JavaScript is actually quite nice now. And in the world of web development, using JavaScript is unavoidable. So let's dip our toes in, and see if we can't have a bit of fun. NOTE: I'm going to assume you know the basics of JavaScript syntax. If not, the Mozilla guides on https://oreil.ly/RCAPk[MDN] are always good quality. I've also heard good things about https://eloquentjavascript.net[_Eloquent JavaScript_], if you prefer a real book. ((("JavaScript testing", "additional resources"))) === Starting with an FT ((("JavaScript testing", "functional test"))) ((("functional tests (FTs)", "JavaScript", id="FTjava16"))) Let's add a new functional test (FT) to the `ItemValidationTest` class; that asserts that our error message disappears when we start typing: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch17l001) ==== [source,python] ---- def test_error_messages_are_cleared_on_input(self): # Edith starts a list and causes a validation error: self.browser.get(self.live_server_url) self.get_item_input_box().send_keys("Banter too thick") self.get_item_input_box().send_keys(Keys.ENTER) self.wait_for_row_in_list_table("1: Banter too thick") self.get_item_input_box().send_keys("Banter too thick") self.get_item_input_box().send_keys(Keys.ENTER) self.wait_for( # <1> lambda: self.assertTrue( # <1> self.browser.find_element( By.CSS_SELECTOR, ".invalid-feedback" ).is_displayed() # <2> ) ) # She starts typing in the input box to clear the error self.get_item_input_box().send_keys("a") # She is pleased to see that the error message disappears self.wait_for( lambda: self.assertFalse( self.browser.find_element( By.CSS_SELECTOR, ".invalid-feedback" ).is_displayed() # <2> ) ) ---- ==== [role="pagebreak-before"] <1> We use another of our `wait_for` invocations, this time with `assertTrue`. <2> `is_displayed()` tells you whether an element is visible or not. We can't just rely on checking whether the element is _present_ in the DOM, because we're now going to mark elements as hidden, rather than removing them from the document object model (DOM) altogether. The FT fails appropriately: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_list_item_validation.\ ItemValidationTest.test_error_messages_are_cleared_on_input*] FAIL: test_error_messages_are_cleared_on_input (functional_tests.test_list_item _validation.ItemValidationTest.test_error_messages_are_cleared_on_input) [...] File "...goat-book/src/functional_tests/test_list_item_validation.py", line 89, in <lambda> lambda: self.assertFalse( ~~~~~~~~~~~~~~~~^ self.browser.find_element( ^^^^^^^^^^^^^^^^^^^^^^^^^^ By.CSS_SELECTOR, ".invalid-feedback" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ).is_displayed() ^^^^^^^^^^^^^^^^ ) ^ AssertionError: True is not false ---- But, before we move on: three strikes and refactor! We've got several places where we find the error element using CSS. Let's move the logic to a helper function: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch17l002) ==== [source,python] ---- class ItemValidationTest(FunctionalTest): def get_error_element(self): return self.browser.find_element(By.CSS_SELECTOR, ".invalid-feedback") [...] ---- ==== [role="pagebreak-before"] And we then make three replacements in 'test_list_item_validation', like this: [role="sourcecode"] .src/functional_tests/test_list_item_validation.py (ch17l003) ==== [source,python] ---- self.wait_for( lambda: self.assertEqual( self.get_error_element().text, "You've already got this in your list", ) ) [...] self.wait_for( lambda: self.assertTrue(self.get_error_element().is_displayed()), ) [...] self.wait_for( lambda: self.assertFalse(self.get_error_element().is_displayed()), ) ---- ==== We still have our expected failure: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_list_item_validation*] [...] lambda: self.assertFalse(self.get_error_element().is_displayed()), ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AssertionError: True is not false ---- TIP: I like to keep helper methods in the FT class that's using them, and only promote them to the base class when they're actually needed elsewhere.((("helper methods"))) It stops the base class from getting too cluttered. You ain’t gonna need it (YAGNI)! [[js-spike]] === A Quick Spike ((("spike"))) ((("exploratory coding", see="also spiking and de-spiking"))) ((("spiking and de-spiking", "defined"))) ((("prototyping", see="spiking and de-spiking"))) This will be our first bit of JavaScript. We're also interacting with the Bootstrap CSS framework, which we maybe don't know very well. In <<chapter_15_simple_form>>, we saw that you can use a unit test as a way of exploring a new API or tool. Sometimes though, you just want to hack something together without any tests at all, just to see if it works, to learn it or get a feel for it. That's absolutely fine! When learning a new tool or exploring a new possible solution, it's often appropriate to leave the rigorous TDD process to one side, and build a little prototype without tests, or perhaps with very few tests. The Goat doesn't mind looking the other way for a bit. TIP: It's actually _fine_ to code without tests sometimes, when you want to explore a new tool or build a throwaway proof-of-concept—as long as you geniunely do throw that hacky code away, and start again with TDD for the real thing. The code _always_ comes out much nicer the second time around. This kind of prototyping activity is often called a "spike", for https://oreil.ly/Ey27H[reasons that aren't entirely clear], but it's a nice memorable name.footnote:[ This chapter shows a very small spike. We'll come back and look at the spiking process again, with a weightier Python/Django example, in <<chapter_19_spiking_custom_auth>> .] Before we start, let's commit our FT. When embarking on a spike, you want to be able to get back to a clean slate: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *git diff* # new method in src/tests/functional_tests/test_list_item_validation.py $ *git commit -am"FT that validation errors disapper on type" ---- TIP: Always do a commit before embarking on a spike. ==== A Simple Inline Script I hacked around for a bit, and here's more or less the first thing I came up with.((("inline scripts (JavaScript)"))) I'm adding the JavaScript inline, in a `<script>` tag at the bottom of our _base.html_ template: [role="sourcecode"] .src/lists/templates/base.html (ch17l004) ==== [source,html] ---- [...] </div> <script> const textInput = document.querySelector("#id_text"); //<1> textInput.oninput = () => { //<2><3> const errorMsg = document.querySelector(".invalid-feedback"); errorMsg.style.display = "none"; //<4> } </script> </body> </html> ---- ==== <1> `document.querySelector` is a way of finding an element in the DOM, using CSS selector syntax, very much like the Selenium `find_element(By.CSS_SELECTOR)` method from our FTs. Grizzled readers may remember having to use jQuery's `$` function for this. <2> `oninput` is how you attach an event listener "callback" function, which will be called whenever the user inputs something into the text box. <3> Arrow functions, `() => {...}`, are the new way of writing anonymous functions in JavaScript, a bit like Python's `lambda` syntax. I think they're cute! Arguments go in the round brackets and the function body goes in the curly brackets. This is a function that takes no arguments—or I should say, ignores any arguments you try to give it. So, what does it do? <4> It finds the error message element, and then hides it by setting its `style.display` to "none". That's actually good enough to get our FT passing: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test functional_tests.test_list_item_validation.\ ItemValidationTest.test_error_messages_are_cleared_on_input* Found 1 test(s). [...] . --------------------------------------------------------------------- Ran 1 test in 3.284s OK ---- TIP: It's good practice to put your script loads at the end of your body HTML, as it means the user doesn't have to wait for all your JavaScript to load before they can see something on the page.((("HTML", "script loads at end of body"))) It also helps to make sure most of the DOM has loaded before any scripts run. See also <<columbo-onload>> later in this chapter. [role="pagebreak-before less_space"] ==== Using the Browser DevTools The test might be happy, but our solution is a little unsatisfactory.((("browsers", "editing HTML using DevTools")))((("DevTools (developer tools)", "editing HTML in"))) If you actually try it in your browser, you'll see that although the error message is gone, the input is still red and invalid-looking (see <<input-still-red>>). [[input-still-red]] .The error message is gone but the input box is still red image::images/tdd3_1702.png["Screenshot of our page where the error `div` is gone but the input is still red."] You're probably imagining that this has something to do with Bootstrap. We might have been able to hide the error message, but we also need to tell Bootstrap that this input no longer has invalid contents.((("Bootstrap", "is-invalid CSS class"))) This is where I'd normally open up DevTools. If level one of hacking is spiking code directly into an inline `<script>` tag, level two is hacking things directly in the browser, where it's not even saved to a file! In <<editing-html-in-devtools>>, you can see me directly editing the HTML of the page, and finding out that removing the `is-invalid` class from the input element seems to do the trick. It not only removes the error message, but also the red border around the input box. [[editing-html-in-devtools]] .Editing the HTML in the browser DevTools image::images/tdd3_1703.png["Screenshot of the browser devtools with us editing the classes for the input element"] We have a reasonable solution now; let's write it down: [role="scratchpad"] ***** * Remove is-invalid Bootstrap CSS class to hide error message and red border. ***** Time to de-spike! [role="pagebreak-before less_space"] .Do We Really Need to Write Unit Tests for This? ******************************************************************************* Do we really need to write unit tests for this?((("unit tests"))) By this point in the book, you probably know I'm going to say "yes", but let's talk about it anyway. Our FT definitely covers the functionality that our JavaScript is delivering, and we could extend it if we wanted to, to check on the colour of the input box or to look at the input element's CSS classes. And if I was really sure that this was the _only_ bit of JavaScript we were ever going to write, I probably would be tempted to leave it at that. But I want to press on for two reasons. Firstly, because any book on web development has to talk about JavaScript and, in a TDD book, I have to show a bit of TDD in JavaScript. More importantly though, as always, we have the boiled frog problem.footnote:[For a reminder, read back on this problem in <<trivial_tests_trivial_functions>>.] We might not have enough JavaScript _yet_ to justify a full test suite, but what about when we come along later and add a tiny bit more? And a tiny bit more again? It's always a judgement call. On the one hand YAGNI, but on the other hand, I think it's best to put the scaffolding in place early so that going test-first is the easy choice later. I can already think of several extra things I'd want to do in the frontend! What about resetting the input to being invalid if someone types in the exact duplicate text again? ******************************************************************************* === Choosing a Basic JavaScript Test Runner ((("test running libraries"))) ((("JavaScript testing", "test running libraries", id="JStestrunner16"))) ((("pytest"))) Choosing your testing tools in the Python world is fairly straightforward. The standard library `unittest` package is perfectly adequate, and the Django test runner also makes a good default choice. More and more though, people will choose http://pytest.org[pytest] for its `assert`-based assertions, and its fixture management. We don't need to get into the pros and cons now!((("assertions", "pytest"))) The point is that there's a "good enough" default, and there's one main popular alternative. The JavaScript world has more of a proliferation! Mocha, Karma, Jester, Chai, AVA, and Tape are just a few of the options I came across when researching for the third edition. I chose Jasmine, because it's still popular despite being around for nearly a decade, and because it offers a "stand-alone" test runner that you can use without needing to dive into the whole Node.js/NPM ecosystem. ((("Node.js")))((("Jasmine")))((("unittest module", "how testing works with"))) === An Overview of Jasmine By now, we're used to the way that testing works with Python's `unittest` library: 1. We have a tests file, separate from the code we're actually testing. 2. We have a way of grouping blocks of code into a test: it's a method, whose name starts with `test_`, on a class that inherits from `unittest.TestCase`. 3. We have a way of making assertions in the test (the special `assert` methods, e.g., `self.assertEqual()`). 4. We have a way of grouping related tests together (putting them in the same class). 5. We can specify shared setup and cleanup code that runs before and after all the tests in a given group, the `setUp()` and `tearDown()` methods. 6. We have some additional helpers that set up our app in a way that simulates what happens “in real life”—whether that's Selenium and the `LiveServerTestCase`, or the Django test client. This is sometimes called the "test harness". There are going to be fairly straightforward equivalents for the first five of these concepts((("Jasmine", "unittest and"))) in Jasmine: 1. There is a tests file (_Spec.js_). 2. Tests go into an anonymous function inside an `it()` block. 3. Assertions use a special function called `expect()`, with a syntax based on method chaining for asserting equality. 4. Blocks of related tests go into a function in a `describe()` block. 5. `setUp()` and `tearDown()` are called `beforeEach()` and `afterEach()`, respectively. There are some differences for sure, but you'll see over the course of the chapter that they're fundamentally the same. What _is_ substantially different is the "test harness" part—the way that Jasmine creates an environment for us to work against. Because we're using the browser runner, what we're actually going to do is define an HTML file (_SpecRunner.html_), and the engine for running our code is going to be an actual browser (with JavaScript running inside it). That HTML will be the entry point for our tests, so it will be in charge of importing our framework, our tests file, and the code under test. It's essentially a parallel, standalone web page that isn't actually part of our app, but it _does_ import the same JavaScript source code that our app uses. === Setting Up Our JavaScript Test Environment // TODO: go all in and use jasmine-browser-runner instead, // it will let me use ES6 modules. Let's download((("Jasmine", "installing"))) Jasmine now: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *wget -O jasmine.zip \ https://github.com/jasmine/jasmine/releases/download/v4.6.1/jasmine-standalone-4.6.1.zip* $ *unzip jasmine.zip -d src/lists/static/tests* $ *rm jasmine.zip* # if you're on Windows you may not have wget or unzip, # but i'm sure you can manage to manually download and unzip the jasmine release # move the example tests "Spec" file to a more central location $ *mv src/lists/static/tests/spec/PlayerSpec.js src/lists/static/tests/Spec.js* # delete all the other stuff we don't need $ *rm -rf src/lists/static/tests/src* $ *rm -rf src/lists/static/tests/spec* ---- //005-1 That leaves us with a directory structure like this: [subs="specialcharacters,quotes"] ---- $ *tree src/lists/static/tests* src/lists/static/tests ├── MIT.LICENSE ├── Spec.js ├── SpecRunner.html └── lib └── jasmine-4.6.1 ├── boot0.js ├── boot1.js ├── jasmine-html.js ├── jasmine.css ├── jasmine.js └── jasmine_favicon.png 3 directories, 9 files ---- _SpecRunner.html_ is the file that ties the proverbial room together. So, we need to go edit it to make sure it's pointing at the right places, to take into account the things we've moved around: [role="sourcecode"] .src/lists/static/tests/SpecRunner.html (ch17l006) [source,diff] ---- @@ -14,12 +14,10 @@ <script src="lib/jasmine-4.6.1/boot1.js"></script> <!-- include source files here... --> - <script src="src/Player.js"></script> - <script src="src/Song.js"></script> + <script src="../lists.js"></script> <!-- include spec files here... --> - <script src="spec/SpecHelper.js"></script> - <script src="spec/PlayerSpec.js"></script> + <script src="Spec.js"></script> </head> ---- We change the source files to point at a (for-now imaginary) _lists.js_ file that we'll put into the _static_ folder, and we change the spec files to point at the single _Spec.js_ file, in the _static/tests_ folder. === Our First Smoke Test: Describe, It, Expect Now, let's open up that _Spec.js_ file and strip it down ((("JavaScript testing", "first smoke test, describe, it, and expect")))to a single minimal smoke test: [role="sourcecode"] .src/lists/static/tests/Spec.js (ch17l007) ==== [source,javascript] ---- describe("Superlists JavaScript", () => { //<1> it("should have working maths", () => { //<2> expect(1 + 1).toEqual(2); //<3> }); }); ---- ==== <1> The `describe` block is a way of grouping tests together, a bit like we use classes in our Python tests. It starts with a string name, and then an arrow function for its body. <2> The `it` block is a single test, a bit like a method in a Python test class. Similarly to the `describe` block, we have a name and then a function to contain the test code. As you can see, the convention is for the descriptive name to complete the sentence started by `it`, in the context of the `describe()` block earlier; so, they often start with "should". <3> Now we have our assertion. This is a little different from assertions in unittest; it's using what's sometimes called "expect" style, often also seen in the Ruby world. We wrap our "actual" value in the `expect()` function, and then our assertions are methods on the resulting expect object, where `.toEqual` is the equivalent of `assertEqual` in Python. ==== Running the Tests via the Browser Let's see how that looks.((("browsers", "running Jasmine spec runner test")))((("web browsers", "running Jasmine spec runner test"))) Open up _SpecRunner.html_ in your browser; you can do this from the command line with: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *firefox src/lists/static/tests/SpecRunner.html* # or, on a mac: $ *open src/lists/static/tests/SpecRunner.html* ---- Or, you can navigate to it in the address bar, using the `file://` protocol—something like this: _file://home/your-username/path/to/superlists/src/lists/static/tests/SpecRunner.html_. Either way you get there, you should see something like <<jasmine-specrunner-green>>. [[jasmine-specrunner-green]] .The Jasmine spec runner in action image::images/tdd3_1704.png["Jasmine browser-based spec runner showing one passing test."] Let's try adding a deliberate failure to see what that looks like: [role="sourcecode"] .src/lists/static/tests/Spec.js (ch17l008) ==== [source,javascript] ---- it("should have working maths", () => { expect(1 + 1).toEqual(3); }); ---- ==== Now if we refresh our browser, we'll see red (<<jasmine-specrunner-red>>). [[jasmine-specrunner-red]] .Our Jasmine tests are now red image::images/tdd3_1705.png["Jasmine browser-based spec runner showing one failing test, with lots of red."] .Is the Jasmine Standalone Browser Test Runner Unconventional? ******************************************************************************* Is the Jasmine standalone browser test runner unconventional? I think it probably is, to be honest.((("Jasmine", "standalone browser test runner")))((("browsers", "Jasmine standalone browser test runner"))) Although, the JavaScript world moves so fast, so I could be wrong by the time you read this. What I do know is that, along with moving very fast, JavaScript things can very quickly become very complicated. A lot of people are working with frameworks these days (React being the main one), and that comes with TypeScript, transpilers, Node.js, NPM, the massive _node_modules_ folder—and a very steep learning curve.((("Node.js")))((("frameworks", "JavaScript"))) In this chapter, my aim is to stick with the basics. The standalone/browser-based test runner lets us write tests without needing to install Node.js or anything else, and it lets us test interactions with the DOM. That's enough to give us a basic environment in which to do TDD in JavaScript. If you decide to go further in the world of frontend, you probably will eventually get into the complexity of frameworks and TypeScript and transpilers, but the basics we work with here will still be a good foundation. We will actually take things a small step further in this book, including dipping our toes into NPM and Node.js in <<chapter_25_CI>>, where we _will_ get CLI-based JavaScript tests working. So, look out for that!((("", startref="JStestrunner16"))) ((("", startref="qunit16"))) ******************************************************************************* === Testing with Some DOM Content What do we _actually_ want to test?((("JavaScript testing", "testing with DOM content", id="ix_JStstDOM"))) We want some JavaScript that will hide the `.invalid-feedback` error div when the user starts typing into the input box. In other words, our code is going to interact with the `input` element on the page and with the `div.invalid-feedback`. Let's look at how to set up some copies of these elements in our JavaScript test environment, for our tests and our code to interact with: [role="sourcecode small-code dofirstch17l009"] .src/lists/static/tests/Spec.js (ch17l010) ==== [source,javascript] ---- describe("Superlists JavaScript", () => { let testDiv; //<4> beforeEach(() => { //<1> testDiv = document.createElement("div"); //<2> testDiv.innerHTML = ` //<3> <form> <input id="id_text" name="text" class="form-control form-control-lg is-invalid" placeholder="Enter a to-do item" value="Value as submitted" aria-describedby="id_text_feedback" required /> <div id="id_text_feedback" class="invalid-feedback">An error message</div> </form> `; document.body.appendChild(testDiv); }); afterEach(() => { //<1> testDiv.remove(); }); ---- ==== <1> The `beforeEach` and `afterEach` functions are Jasmine's equivalent of `setUp` and `tearDown`. <2> The `document` global is a built-in browser variable that represents the current HTML page. So, in our case, it's a reference to the _SpecRunner.html_ page. <3> We create a new `div` element and populate it with some HTML that matches the elements we care about from our Django template. Notice the use of backticks (+`+) to enable us to write multiline strings. Depending on your text editor, it may even nicely syntax-highlight the HTML for you. <4> A little quirk of JavaScript here, because we want the same `testDiv` variable to be available inside both the `beforeEach` and `afterEach` functions: we declare the variable with `let` in the containing scope outside of both functions. In theory, we could have just added the HTML to the _SpecRunner.html_ file, but by using `beforeEach` and `afterEach`, I'm making sure that each test gets a completely fresh copy of the HTML elements involved, so that one test can't affect another. TIP: To ensure isolation between browser-based JavaScript tests, use `beforeEach()` and `afterEach()` to create and tidy up any DOM elements that your code needs to interact with. Let's now play with our testing framework to see if we can find DOM elements and make assertions on whether they are visible. We'll also try the same `style.display=none` hiding technique that we originally used in our spiked code: [role="sourcecode"] .src/lists/static/tests/Spec.js (ch17l011) ==== [source,javascript] ---- it("should have a useful html fixture", () => { const errorMsg = document.querySelector(".invalid-feedback"); expect(errorMsg.checkVisibility()).toBe(true); //<1> }); it("can hide things manually and check visibility in tests", () => { const errorMsg = document.querySelector(".invalid-feedback"); errorMsg.style.display = "none"; //<2> expect(errorMsg.checkVisibility()).toBe(false); //<3> }); ---- ==== <1> We retrieve our error `div` with `querySelector` again, and then use another fairly new API in JavaScript-Land called `checkVisibility()` to check if it's displayed or hidden.footnote:[ Read up on the `checkVisibility()` method in the https://oreil.ly/hk6qg[MDN documentation].] <2> We _manually_ hide the element in the test, by setting its `style.display` to "none". (Again, our objective here is to smoke-test, both our ability to hide things and our ability to test that they are hidden.) <3> And we check it worked, with `checkVisibility()` again. Notice that I'm being really good about splitting things out into multiple tests, with one assertion each. Jasmine encourages that by deprecating the ability to pass failure messages into individual `expect/toBe` expressions, for example. [role="pagebreak-before"] If you refresh the browser, you should see that all passes: [[first-jasmine-output]] ==== [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 2 specs, 0 failures, randomized with seed 12345 finished in 0.005s Superlists JavaScript * can hide things manually and check visibility in tests * should have a useful html fixture ---- ==== (From now on, I'll show the Jasmine outputs as text, like this, to avoid filling the chapter with screenshots.) === Building a JavaScript Unit Test for Our Desired Functionality ((("JavaScript testing", "testing with DOM content", startref="ix_JStstDOM")))((("JavaScript testing", "unit test"))) ((("unit tests", "JavaScript"))) Now that we're acquainted with our JavaScript testing tools, we can start to write the real thing: [role="sourcecode small-code"] .src/lists/static/tests/Spec.js (ch17l012) ==== [source,javascript] ---- it("should have a useful html fixture", () => { // <1> const errorMsg = document.querySelector(".invalid-feedback"); expect(errorMsg.checkVisibility()).toBe(true); }); it("should hide error message on input", () => { //<2> const textInput = document.querySelector("#id_text"); //<3> const errorMsg = document.querySelector(".invalid-feedback"); textInput.dispatchEvent(new InputEvent("input")); //<4> expect(errorMsg.checkVisibility()).toBe(false); //<5> }); ---- ==== <1> As it's not doing any harm, let's keep the first smoke test. <2> Let's change the second one, and give it a name that describes what we want to happen; our objective is that, when the user starts typing into the input box, we should hide the error message. <3> We retrieve the `<input>` element from the DOM, in a similar way to how we found the error message `div`. <4> Here's how we simulate a user typing into the input box. <5> And here's our real assertion: the error `div` should be hidden after the input box sees an input event. That gives us our expected failure: [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 2 specs, 1 failure, randomized with seed 12345 finished in 0.005s Spec List | Failures Superlists JavaScript > should hide error message on input Expected true to be false. <Jasmine> @file:///...goat-book/src/lists/static/tests/Spec.js:38:40 <Jasmine> ---- Now let's try reintroducing the code we hacked together in our spike, into _lists.js_: [role="sourcecode"] .src/lists/static/lists.js (ch17l014) ==== [source,javascript] ---- const textInput = document.querySelector("#id_text"); textInput.oninput = () => { const errorMsg = document.querySelector(".invalid-feedback"); errorMsg.style.display = "none"; }; ---- ==== That doesn't work! We get an unexpected error: [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 2 specs, 2 failures, randomized with seed 12345 finished in 0.005s Error during loading: TypeError: can't access property "oninput", textInput is null in file:///...goat-book/src/lists/static/lists.js line 2 Spec List | Failures Superlists JavaScript > should hide error message on input Expected true to be false. <Jasmine> @file:///...goat-book/src/lists/static/tests/Spec.js:38:40 <Jasmine> ---- [role="pagebreak-before"] If your Jasmine output shows `Script error` instead of `textInput is null`, open up the DevTools console, and you'll see the actual error printed in there, as in <<typeerror-in-devools>>.footnote:[ Some users have also reported that Google Chrome will show a different error, to do with the browser preventing loading local files.((("web browsers", "textInput is null errors and"))) If you really can't use Firefox, you might be able to find some solutions on https://oreil.ly/EkwdH[Stack Overflow].] [[typeerror-in-devools]] .`textInput` is null, one way or another image::images/tdd3_1706.png["Screenshot of devtools console showing the textInput is null TypeError"] `textInput is null`, it says. Let's see if we can figure out why. === Fixtures, Execution Order, and Global State: Key Challenges of JavaScript Testing ((("JavaScript testing", "managing global state"))) ((("global state"))) ((("JavaScript testing", "key challenges of", id="JSTkey16"))) ((("HTML fixtures"))) One of the difficulties with JavaScript in general, and testing in particular, is understanding the order of execution of our code (i.e., what happens when). When does our code in _lists.js_ run, and when do each of our tests run? How do they all interact with global state—that is, the DOM of our web page and the fixtures that we've already seen are supposed to be cleaned up after each test? [role="pagebreak-before less_space"] ==== console.log for Debug Printing ((("print", "debugging with"))) ((("debugging", "print-based"))) ((("console.log"))) Let's add a couple of debug prints, or "console.logs": [role="sourcecode"] .src/lists/static/tests/Spec.js (ch17l015) ==== [source,javascript] ---- console.log("Spec.js loading"); describe("Superlists JavaScript", () => { let testDiv; beforeEach(() => { console.log("beforeEach"); testDiv = document.createElement("div"); [...] it("should have a useful html fixture", () => { console.log("in test 1"); const errorMsg = document.querySelector(".invalid-feedback"); [...] it("should hide error message on input", () => { console.log("in test 2"); const textInput = document.querySelector("#id_text"); [...] ---- ==== And the same in our actual JavaScript code: [role="sourcecode"] .src/lists/static/lists.js (ch17l016) ==== [source,javascript] ---- console.log("lists.js loading"); const textInput = document.querySelector("#id_text"); textInput.oninput = () => { const errorMsg = document.querySelector(".invalid-feedback"); errorMsg.style.display = "none"; }; ---- ==== [role="pagebreak-before"] Rerun the tests, opening up the browser debug console (Ctrl+Shift+I or Cmd+Alt+I) and you should see something like <<jasmine-with-js-console>>. [[jasmine-with-js-console]] .Jasmine tests with `console.log` debug outputs image::images/tdd3_1707.png["Jasmine tests with console.log debug outputs"] What do we see? . First, _lists.js_ loads. . Then, we see the error saying `textInput is null`. . Next, we see our tests loading in _Spec.js_. . Then, we see a `beforeEach`, which is when our test fixture actually gets added to the DOM. . Finally, we see the first test run. This explains the problem: when _lists.js_ loads, the input node doesn't exist yet. [role="pagebreak-before less_space"] === Using an Initialize Function for More Control Over Execution Time We need more control over the order of execution of our JavaScript.((("JavaScript testing", "using initialize function to control execution time")))((("initialize function in JavaScript testing"))) Rather than just relying on the code in _lists.js_ running whenever it is loaded by a `<script>` tag, we can use a common pattern: define an "initialize" function and call that when we want to in our tests (and later in real life).footnote:[Have you been enjoying the British English spelling in the book so far and are shocked to see the _z_ in “initialize”? By convention, even us Brits often use American spelling in code, because it makes it easier for international colleagues to read, and to make it correspond better with code samples on the internet.] Here's what that function could look like: [role="sourcecode"] .src/lists/static/lists.js (ch17l017) ==== [source,javascript] ---- console.log("lists.js loading"); const initialize = () => { console.log("initialize called"); const textInput = document.querySelector("#id_text"); textInput.oninput = () => { const errorMsg = document.querySelector(".invalid-feedback"); errorMsg.style.display = "none"; }; }; ---- ==== And in our tests file, we call `initialize()` in our key test: [role="sourcecode"] .src/lists/static/tests/Spec.js (ch17l018) ==== [source,javascript] ---- it("should have a useful html fixture", () => { console.log("in test 1"); const errorMsg = document.querySelector(".invalid-feedback"); expect(errorMsg.checkVisibility()).toBe(true); }); it("should hide error message on input", () => { console.log("in test 2"); const textInput = document.querySelector("#id_text"); const errorMsg = document.querySelector(".invalid-feedback"); initialize(); //<1> textInput.dispatchEvent(new InputEvent("input")); expect(errorMsg.checkVisibility()).toBe(false); }); }); ---- ==== <1> This is where we call `initialize()`. We don't need to call it in our fixture sense-check. And that will actually get our tests passing! [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 2 specs, 0 failures, randomized with seed 12345 finished in 0.005s Superlists JavaScript * should hide error message on input * should have a useful html fixture ---- And now the `console.log` outputs should be in a more sensible order: [role="skipme"] ---- lists.js loading lists.js:1:9 Spec.js loading Spec.js:1:9 beforeEach Spec.js:7:13 in test 1 Spec.js:31:13 beforeEach Spec.js:7:13 in test 2 Spec.js:37:13 initialize called lists.js:3:11 ---- === Deliberately Breaking Our Code to Force Ourselves to Write More Tests I'm always nervous when I see green tests. We've copy-pasted five lines of code from our spike with just one test. That was a little too easy, even if we did have to go through that little `initialize()` dance. So, let's change our `initialize()` function to deliberately break it. What if we just immediately hide errors? [role="sourcecode"] .src/lists/static/lists.js (ch17l019) ==== [source,javascript] ---- const initialize = () => { // const textInput = document.querySelector("#id_text"); // textInput.oninput = () => { const errorMsg = document.querySelector(".invalid-feedback"); errorMsg.style.display = "none"; // }; }; ---- ==== Oh dear, as I feared—the tests just pass: [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 2 specs, 0 failures, randomized with seed 12345 finished in 0.005s Superlists JavaScript * should hide error message on input * should have a useful html fixture ---- We need an extra test, to check that our `initialize()` function isn't overzealous: [role="sourcecode"] .src/lists/static/tests/Spec.js (ch17l020) ==== [source,javascript] ---- it("should hide error message on input", () => { [...] }); it("should not hide error message before event is fired", () => { const errorMsg = document.querySelector(".invalid-feedback"); initialize(); expect(errorMsg.checkVisibility()).toBe(true); //<1> }); ---- ==== <1> In this test, we don't fire the input event with `dispatchEvent`, so we expect the error message to still be visible. That gives us our expected failure: [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 3 specs, 1 failure, randomized with seed 12345 finished in 0.005s Spec List | Failures Superlists JavaScript > should not hide error message before event is fired Expected false to be true. <Jasmine> @file:///...goat-book/src/lists/static/tests/Spec.js:48:40 <Jasmine> ---- This justifies us to restore the `textInput.oninput()`: [role="sourcecode"] .src/lists/static/lists.js (ch17l021) ==== [source,javascript] ---- const initialize = () => { const textInput = document.querySelector("#id_text"); textInput.oninput = () => { const errorMsg = document.querySelector(".invalid-feedback"); errorMsg.style.display = "none"; }; }; ---- ==== === Red/Green/Refactor: Removing Hardcoded Selectors The `#id_text` and `.invalid-feedback` selectors are "magic constants" at the moment. It would be better to pass them into `initialize()`, both in the tests and in _base.html_, so that they're defined in the same file that actually has the HTML elements. And while we're at it, our tests could do with a bit of refactoring too, to remove some duplication.((("JavaScript testing", "red/green/refactor, removing hardcoded selectors")))((("CSS (Cascading Style Sheets)", "removing hardcoded selectors")))((("Red/Green/Refactor", "removing hardcoded selectors")))((("selectors (CSS)", "removing hardcoded selectors"))) We'll start with that, by defining a few more variables in the top-level scope, and populate them in the `beforeEach`: [role="sourcecode small-code"] .src/lists/static/tests/Spec.js (ch17l022) ==== [source,javascript] ---- describe("Superlists JavaScript", () => { const inputId = "id_text"; //<1> const errorClass = "invalid-feedback"; //<1> const inputSelector = `#${inputId}`; //<2> const errorSelector = `.${errorClass}`; //<2> let testDiv; let textInput; //<3> let errorMsg; //<3> beforeEach(() => { console.log("beforeEach"); testDiv = document.createElement("div"); testDiv.innerHTML = ` <form> <input id="${inputId}" //<4> name="text" class="form-control form-control-lg is-invalid" placeholder="Enter a to-do item" value="Value as submitted" aria-describedby="id_text_feedback" required /> <div id="id_text_feedback" class="${errorClass}">An error message</div> //<4> </form> `; document.body.appendChild(testDiv); textInput = document.querySelector(inputSelector); //<5> errorMsg = document.querySelector(errorSelector); //<5> }); ---- ==== <1> Let's define some constants to represent the selectors for our input element and our error message `div`. <2> We can use JavaScript's string interpolation (the equivalent of f-strings) to then define the CSS selectors for the same elements. <3> We'll also set up some variables to hold the elements we're always referring to in our tests (these can't be constants, as we'll see shortly). <4> We use a bit more interpolation to reuse the constants in our HTML template. A first bit of de-duplication! <5> Here's why `textInput` and `errorMsg` can't be constants: we're re-creating the DOM fixture in every `beforeEach`, so we need to re-fetch the elements each time. Now we ((("Don't Repeat Yourself (DRY)")))can apply some DRY ("don't repeat yourself") to strip down our tests: [role="sourcecode"] .src/lists/static/tests/Spec.js (ch17l023) ==== [source,javascript] ---- it("should have a useful html fixture", () => { expect(errorMsg.checkVisibility()).toBe(true); }); it("should hide error message on input", () => { initialize(); textInput.dispatchEvent(new InputEvent("input")); expect(errorMsg.checkVisibility()).toBe(false); }); it("should not hide error message before event is fired", () => { initialize(); expect(errorMsg.checkVisibility()).toBe(true); }); ---- ==== You can definitely overdo DRY in test, but I think this is working out very nicely. Each test is between one and three lines long, meaning it's very easy to see what each one is doing, and what it's doing differently from the others. We've only refactored the tests so far, so let's check that they still pass: [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 3 specs, 0 failures, randomized with seed 12345 finished in 0.005s Superlists JavaScript * should hide error message on input * should have a useful html fixture * should not hide error message before event is fired ---- [role="pagebreak-before"] The next refactor is wanting to pass the selectors to `initialize()`. Let's see what happens if we just do that straight away, in the tests: [role="sourcecode"] .src/lists/static/tests/Spec.js (ch17l024) ==== [source,diff] ---- @@ -40,14 +40,14 @@ describe("Superlists JavaScript", () => { }); it("should hide error message on input", () => { - initialize(); + initialize(inputSelector, errorSelector); textInput.dispatchEvent(new InputEvent("input")); expect(errorMsg.checkVisibility()).toBe(false); }); it("should not hide error message before event is fired", () => { - initialize(); + initialize(inputSelector, errorSelector); expect(errorMsg.checkVisibility()).toBe(true); }); }); ---- ==== Now we look at the tests: [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 3 specs, 0 failures, randomized with seed 12345 finished in 0.005s Superlists JavaScript * should hide error message on input * should have a useful html fixture * should not hide error message before event is fired ---- They still pass! You might have been expecting a failure to do with the fact that `initialize()` was defined as taking no arguments—but we passed two! That's because JavaScript is too chill for that.((("JavaScript", "calling functions with too few or too many arguments"))) You can call a function with too many or too few arguments, and JavaScript will just _deal with it_. Let's fish those arguments out in `initialize()`: [role="sourcecode"] .src/lists/static/lists.js (ch17l025) ==== [source,javascript] ---- const initialize = (inputSelector, errorSelector) => { const textInput = document.querySelector(inputSelector); textInput.oninput = () => { const errorMsg = document.querySelector(errorSelector); errorMsg.style.display = "none"; }; }; ---- ==== And the tests still pass: [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 3 specs, 0 failures, randomized with seed 12345 finished in 0.005s ---- Let's deliberately use the arguments the wrong way round, just to check we get a failure: [role="sourcecode"] .src/lists/static/lists.js (ch17l026) ==== [source,javascript] ---- const initialize = (errorSelector, inputSelector) => { ---- ==== Phew, that does indeed fail: [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 3 specs, 1 failure, randomized with seed 12345 finished in 0.005s Spec List | Failures Superlists JavaScript > should hide error message on input Expected true to be false. <Jasmine> @file:///...goat-book/src/lists/static/tests/Spec.js:46:40 <Jasmine> ---- OK, back to the right way around: [role="sourcecode"] .src/lists/static/lists.js (ch17l027) ==== [source,javascript] ---- const initialize = (inputSelector, errorSelector) => { ---- ==== === Does it Work? And for the moment of truth, we'll pull in our script and invoke our initialize function on our real pages.((("JavaScript testing", "inline script calling initialize with right selectors"))) Let's use another `<script>` tag to include our _lists.js_, and strip down the the inline JavaScript to just calling `initialize()` with the right selectors: [role="sourcecode"] .src/lists/templates/base.html (ch17l028) ==== [source,html] ---- </div> <script src="/static/lists.js"></script> <script> initialize("#id_text", ".invalid-feedback"); </script> </body> </html> ---- ==== [role="pagebreak-before"] Aaaand we run our FT: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test functional_tests.test_list_item_validation.\ ItemValidationTest.test_error_messages_are_cleared_on_input* [...] Ran 1 test in 3.023s OK ---- Hooray! That's a commit! ((("", startref="JSTkey16"))) [subs="specialcharacters,quotes"] ---- $ *git add src/lists* $ *git commit -m"Despike our js, add jasmine tests"* ---- NOTE: We're using a `<script>` tag to import our code, but modern JavaScript lets you use `import` and `export` to explicitly import particular parts of your code.((("JavaScript", "import and export in to import code"))) However, that involves specifying the scripts as modules, which is fiddly to get working with the single-file test runner we're using. So, I decided to use the "simple" old-fashioned way. By all means, investigate modules in your own projects! === Testing Integration with CSS and Bootstrap As the tests flashed past, you may have noticed an unsatisfactory bit of red, still left around our input box. Wait a minute! We forgot one of the key things we learned in our spike!((("JavaScript testing", "testing integration with CSS and Bootstrap")))((("Bootstrap", "JavaScript test's integration with"))) [role="scratchpad"] ***** * Remove is-invalid Bootstrap CSS class to hide error message and red border. ***** We don't need to manually hack `style.display=none`; we can work _with_ the Bootstrap framework and just remove the `.is-invalid` class. [role="pagebreak-before"] OK, let's try it in our implementation: [role="sourcecode"] .src/lists/static/lists.js (ch17l029) ==== [source,javascript] ---- const initialize = (inputSelector, errorSelector) => { const textInput = document.querySelector(inputSelector); textInput.oninput = () => { textInput.classList.remove("is-invalid"); }; }; ---- ==== Oh dear; it seems like that doesn't quite work: [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 3 specs, 1 failure, randomized with seed 12345 finished in 0.005s Spec List | Failures Superlists JavaScript > should hide error message on input Expected true to be false. <Jasmine> @file:///...goat-book/src/lists/static/tests/Spec.js:46:40 <Jasmine> ---- What's happening here? Well, as hinted in the section title, we're now relying on the integration with Bootstrap's CSS, but our test runner doesn't know about Bootstrap yet. We can include it in a reasonably familiar way, which is by including it in the `<head>` of our _SpecRunner.html_ file: [role="sourcecode"] .src/lists/static/tests/SpecRunner.html (ch17l030) ==== [source,html] ---- <link rel="stylesheet" href="lib/jasmine-4.6.1/jasmine.css"> <!-- Bootstrap CSS --> <link href="../bootstrap/css/bootstrap.min.css" rel="stylesheet"> <script src="lib/jasmine-4.6.1/jasmine.js"></script> ---- ==== That gets us back to passing tests: [role="jasmine-output"] [subs="specialcharacters,quotes"] ---- 3 specs, 0 failures, randomized with seed 12345 finished in 0.005s Superlists JavaScript * should hide error message on input * should have a useful html fixture * should not hide error message before event is fired ---- Let's do a little more refactoring. If your editor is set up to do some JavaScript linting, you might have seen a warning saying: [role="skipme"] ---- 'errorSelector' is declared but its value is never read. ---- Great! Looks like we can get away with just one argument to our `initialize()` function: [role="sourcecode"] .src/lists/static/lists.js (ch17l031) ==== [source,javascript] ---- const initialize = (inputSelector) => { const textInput = document.querySelector(inputSelector); textInput.oninput = () => { textInput.classList.remove("is-invalid"); }; }; ---- ==== Are you enjoying the way the tests keep passing even though we're giving the function too many arguments? JavaScript is so chill, man. Let's strip them down anyway: [role="sourcecode"] .src/lists/static/tests/Spec.js (ch17l032) ==== [source,diff] ---- @@ -40,14 +40,14 @@ describe("Superlists JavaScript", () => { }); it("should hide error message on input", () => { - initialize(inputSelector, errorSelector); + initialize(inputSelector); textInput.dispatchEvent(new InputEvent("input")); expect(errorMsg.checkVisibility()).toBe(false); }); it("should not hide error message before event is fired", () => { - initialize(inputSelector, errorSelector); + initialize(inputSelector); expect(errorMsg.checkVisibility()).toBe(true); }); }); ---- ==== And the base template, yay. Nothing more satisfying than _deleting code_: [role="sourcecode"] .src/lists/templates/base.html (ch17l033) ==== [source,html] ---- <script> initialize("#id_text"); </script> ---- ==== And we can run the FT one more time, just for safety: ---- OK ---- [role="pagebreak-before less_space"] .Trade-offs in JavaScript Unit Testing Versus Selenium ******************************************************************************* Similarly to the way our Selenium tests and our Django unit tests interact, we have an overlap between the functionality covered by our JavaScript unit tests and our Selenium FTs.((("unit tests", "trade-offs in JavaScript unit testing versus Selenium")))((("Selenium", "trade-offs in JavaScript unit testing and")))((("JavaScript testing", "trade-offs in unit testing versus Selenium"))) As always, the downside of the FTs is that they are slow, and they can't always point you towards exactly what went wrong. But they _do_ give us the best reassurance that all our components--in this case, browser, CSS framework, and JavaScript--are all working together. On the other hand, by using the jasmine-browser-runner, we are _also_ testing the integration between our browser, our JavaScript, and Bootstrap. This comes at the expense of having a slightly clunky testing setup. If you wanted to switch to faster, more focused unit tests, you could try the following: * Stop using the browser runner. * Switch to a node-based CLI test runner. * Change from asserting using `checkVisibility()` (which won't work without a real DOM) to asserting what the JavaScript code is actually doing—removing the `.is-invalid` CSS class. It might look something like this: [role="sourcecode skipme"] .src/lists/static/tests/Spec.js ==== [source,javascript] ---- it("should hide error message on input", () => { initialize(inputSelector); textInput.dispatchEvent(new InputEvent("input")); expect(errorMsg.classList).not.toContain("is-invalid"); }); ---- ==== The trade-off here is that you get faster, more focused unit tests, but you need to lean more heavily on Selenium to test the integration with Bootstrap. That could be worth it, but probably only if you start to have a lot more JavaScript code. ******************************************************************************* [[columbo-onload]] === Columbo Says: Wait for Onload [quote, Columbo (fictional trench-coat-wearing American detective known for his persistence)] ______________________________________________________________ Wait, there's just one more thing... ______________________________________________________________ As always, there's one final thing.((("JavaScript testing", "JavaScript interacting with the DOM, wrapping in onload boilerplate"))) Whenever you have some JavaScript that interacts with the DOM, it's good to wrap it in some "onload" boilerplate to make sure that the page has fully loaded before it tries to do anything. Currently it works anyway, because we've placed the `<script>` tag right at the bottom of the page, but we shouldn't rely on that. https://oreil.ly/buBe8[The MDN documentation] on this is good, as usual. The modern JavaScript onload boilerplate is minimal: [role="sourcecode"] .src/lists/templates/base.html (ch17l034) ==== [source,javascript] ---- <script> window.onload = () => { initialize("#id_text"); }; </script> ---- ==== That's a commit folks! [subs="specialcharacters,quotes"] ---- $ *git status* $ *git add src/lists/static* # all our js and tests $ *git add src/lists/templates* # changes to the base template $ *git commit -m"Javascript to hide error messages on input"* ---- === JavaScript Testing in the TDD Cycle ((("JavaScript testing", "in the TDD cycle", secondary-sortas="TDD cycle"))) ((("Test-Driven Development (TDD)", "JavaScript testing in double loop TDD cycle"))) You may be wondering how these JavaScript tests fit in with our "double loop" TDD cycle (see <<double-loop-tdd-ch17>>). [[double-loop-tdd-ch17]] .Double-loop TDD reminder image::images/tdd3_aa01.png["Diagram showing an inner loop of red/green/refactor, and an outer loop of red-(inner loop)-green."] The answer is that the JavaScript unit-test/code cycle plays exactly the same role as the Python unit one: [role="pagebreak-before"] 1. Write an FT and see it fail. 2. Figure out what kind of code you need next: Python or JavaScript? 3. Write a unit test in either language, and see it fail. 4. Write some code in either language, and make the test pass. 5. Rinse and repeat. Phew. Well, hopefully some sense of closure there. The next step is to deploy our new code to our servers. There is more JavaScript fun in this book too! Have a look at the https://www.obeythetestinggoat.com/book/appendix_rest_api.html[Online Appendix: Building a REST API]), when you're ready for it. ((("", startref="FTjava16"))) NOTE: Want a little more practice with JavaScript? See if you can get our error messages to be hidden when the user clicks inside the input element, as well as just when they type in it. You should be able to FT it too, if you want a bit of extra Selenium practice. .JavaScript Testing Notes ******************************************************************************* Selenium as the outer loop:: One of the great advantages of Selenium is that it enables you to test that your JavaScript really works, just as it tests your Python code. But, as always, FTs are a very blunt tool, so it's often worth pairing them with some lower-level tests.((("Selenium", "and JavaScript", secondary-sortas="JavaScript"))) Choosing your testing framework:: There are many JavaScript test-running libraries out there. Jasmine has been around for a while, but the others are also worth investigating. ((("JavaScript testing", "test running libraries"))) Idiosyncrasies of the browser:: No matter which testing library you use, if you're working with Vanilla JavaScript (i.e., not a framework like React), you'll need to work around the key "gotchas" of JavaScript: + * The DOM and HTML fixtures * Global state * Understanding and controlling execution order((("JavaScript testing", "managing global state"))) ((("global state"))) Frontend frameworks:: An awful lot of frontend work these days is done in frameworks, React being the 1,000-pound gorilla. There are lots of resources on React testing out there, so I'll let you go out and find them if you need them. ******************************************************************************* //IDEA: take the opportunity to use {% static %} tag in templates? ================================================ FILE: chapter_18_second_deploy.asciidoc ================================================ [[chapter_18_second_deploy]] == Deploying Our New Code ((("deployment", "procedure for", id="Dpro17"))) It's time to deploy our brilliant new validation code to our live servers. This will be a chance to see our automated deploy scripts in action for the second time. Let's take the opportunity to make a little deployment checklist. NOTE: At this point I always want to say a huge thanks to Andrew Godwin and the whole Django team. In the first edition, I used to have a whole long section, entirely devoted to migrations. Since Django 1.7, migrations now "just work", so I was able to drop it altogether. I mean yes this all happened nearly ten years ago, but still--open source software is a gift. We get such amazing things, entirely for free. It's worth taking a moment to be grateful, now and again. === The Deployment Checklist Let's make a little checklist of pre-deployment tasks: 1. We run all our unit tests and functional tests (FTs) in the regular way—just in case! 2. We rebuild our Docker image and run our tests against Docker on our local machine. 3. We deploy to staging, and run our FTs against staging. 4. Now we can deploy to prod. TIP: A deployment checklist like this should be a temporary measure. Once you've worked through it manually a few times, you should be looking to take the next step in automation: continuous deployment straight to production using a CI/CD (continuous integration/continuous development) pipeline. We'll touch on this in <<chapter_25_CI>>. === A Full Test Run Locally Of course, under the watchful eye of the Testing Goat, we're running the tests all the time! But, just in case: [subs="specialcharacters,quotes"] ---- $ *cd src && python manage.py test* [...] Ran 37 tests in 15.222s OK ---- === Quick Test Run Against Docker The next step towards production is running things in Docker.((("Docker", "test run against")))((("containers", "rebuilding Docker image and local container"))) This was one of the main reasons we went to the trouble of containerising our app: to reproduce the production environment as faithfully as possible on our own machine. [role="pagebreak-before"] So let's rebuild our Docker image and spin up a local Docker container: [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 \ -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -e DJANGO_DB_PATH=/home/nonroot/db.sqlite3 \ -it superlists => [internal] load build definition from Dockerfile 0.0s => => transferring dockerfile: 371B 0.0s => [internal] load metadata for docker.io/library/python:3.14-slim 1.4s [...] => => naming to docker.io/library/superlists 0.0s + docker run -p 8888:8888 --mount type=bind,source="$PWD/src/db.sqlite3",target=/src/db.sqlite3 -e DJANGO_SECRET_KEY=sekrit -e DJANGO_ALLOWED_HOST=localhost -e EMAIL_PASSWORD -it superlists [2025-01-27 21:29:37 +0000] [7] [INFO] Starting gunicorn 22.0.0 [2025-01-27 21:29:37 +0000] [7] [INFO] Listening at: http://0.0.0.0:8888 (7) [2025-01-27 21:29:37 +0000] [7] [INFO] Using worker: sync [2025-01-27 21:29:37 +0000] [8] [INFO] Booting worker with pid: 8 ---- And now, in a separate terminal, we can run our FT suite against the Docker: [subs="specialcharacters,quotes"] ---- $ *TEST_SERVER=localhost:8888 python src/manage.py test functional_tests* [...] ...... --------------------------------------------------------------------- Ran 6 tests in 17.047s OK ---- Looking good! Let's move on to staging.((("staging server", "deployment to and test run")))((("Ansible", "deployment to staging, playbook for"))) [role="pagebreak-before less_space"] === Staging Deploy and Test Run Here's our `ansible-playbook` command to deploy to staging: [role="against-server small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*ansible-playbook --user=elspeth -i staging.ottg.co.uk, infra/deploy-playbook.yaml -vv*] [...] PLAY [all] ********************************************************************* TASK [Gathering Facts] ********************************************************* [...] ok: [staging.ottg.co.uk] TASK [Install docker] ********************************************************** ok: [staging.ottg.co.uk] => {"cache_update_time": [...] TASK [Add our user to the docker group, so we don't need sudo/become] ********** ok: [staging.ottg.co.uk] => {"append": true, "changed": false, [...] TASK [Reset ssh connection to allow the user/group change to take effect] ****** TASK [Build container image locally] ******************************************* changed: [staging.ottg.co.uk -> 127.0.0.1] => {"actions": ["Built image [...] TASK [Export container image locally] ****************************************** changed: [staging.ottg.co.uk -> 127.0.0.1] => {"actions": ["Archived image [...] TASK [Upload image to server] ************************************************** changed: [staging.ottg.co.uk] => {"changed": true, "checksum": [...] TASK [Import container image on server] **************************************** changed: [staging.ottg.co.uk] => {"actions": ["Loaded image superlists:latest [...] TASK [Ensure .secret-key file exists] ****************************************** ok: [staging.ottg.co.uk] => {"changed": false, "dest": [...] TASK [Read secret key back from file] ****************************************** ok: [staging.ottg.co.uk] => {"changed": false, "content": [...] TASK [Ensure db.sqlite3 file exists outside container] ************************* changed: [staging.ottg.co.uk] => {"changed": true, "dest": [...] TASK [Run container] *********************************************************** changed: [staging.ottg.co.uk] => {"changed": true, "container": [...] TASK [Run migration inside container] ****************************************** changed: [staging.ottg.co.uk] => {"changed": true, "rc": 0, "stderr": "", [...] PLAY RECAP ********************************************************************* staging.ottg.co.uk : ok=12 changed=7 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 ---- NOTE: If your server is offline because you ran out of free credits with your provider, you'll have to create a new one. Skip back to <<chapter_11_server_prep>> if you need. And now we run the FTs against staging: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=staging.ottg.co.uk python src/manage.py test functional_tests*] OK ---- Hooray! === Production Deploy As all is looking well, we can deploy to prod!((("production-ready deployment"))) [role="against-server small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*ansible-playbook --user=elspeth -i www.ottg.co.uk, infra/deploy-playbook.yaml -vv*] ---- === What to Do If You See a Database Error Because our migrations introduce a new integrity constraint, you may find that it fails to apply because some existing data violates that constraint.((("databases", "deployed database, integrity error"))) For example, here's what you might see if any of the lists on the server already contain duplicate items: [role="skipme"] ---- sqlite3.IntegrityError: columns list_id, text are not unique ---- At this point, you have two choices: 1. Delete the database on the server and try again—after all, it's only a toy project! 2. Create a data migration. You can find out more in the https://docs.djangoproject.com/en/5.2/topics/migrations/#data-migrations[Django migrations docs]. ==== How to Delete the Database on the Staging Server Here's how you((("databases", "deleting database on staging server")))((("staging server", "deleting database on"))) might do option 1: [role="skipme"] ---- ssh elspeth@staging.ottg.co.uk rm db.sqlite3 ---- The `ssh` command takes an arbitrary shell command to run as its last argument, so we pass in `rm db.sqlite3`. We don't need a full path because we keep the SQLite database in our home folder. WARNING: Try not to accidentally delete your production database. === Wrap-Up: git tag the New Release The last thing to do is to tag the release((("Git", "tagging releases"))) in our version control system (VCS)—it's important that we're always able to keep track of what's live: [subs="specialcharacters,quotes"] ---- $ *git tag -f LIVE* # needs the -f because we are replacing the old tag $ *export TAG=`date +DEPLOYED-%F/%H%M`* $ *git tag $TAG* $ *git push -f origin LIVE $TAG* ---- NOTE: Some people don't like to use `push -f` and update an existing tag, and will instead use some kind of version number to tag their releases. Use whatever works for you. And on that note, we can wrap up the last of the concepts we discussed in <<part3>>, and move on to the more exciting topics that comprise <<part4>>. Can't wait! .Deployment Procedure Review ******************************************************************************* We've done a couple of deploys now, so this is a good time for a little recap: * Deploy to staging first. * Run our FTs against staging. * Deploy to live. * Tag the release. Deployment procedures evolve and get more complex as projects grow, and it's an area that can become hard to maintain—full of manual checks and procedures—if you're not careful to keep things automated. There's lots more to learn about this, but it's out of scope for this book. Dave Farley's https://oreil.ly/X2O_T[video on continuous delivery] is a good place to start. ((("", startref="Dpro17"))) ******************************************************************************* ================================================ FILE: chapter_19_spiking_custom_auth.asciidoc ================================================ [[chapter_19_spiking_custom_auth]] == User Authentication, Spiking, [keep-together]#and De-Spiking# ((("authentication", id="AuthSpike18"))) Our beautiful lists site has been live for a few days, and our users are starting to come back to us with feedback. "We love the site", they say, "but we keep losing our lists. Manually remembering URLs is hard. It'd be great if it could remember what lists we'd started." Remember Henry Ford and faster horses. Whenever you hear a user requirement, it's important to dig a little deeper and think--what is the real requirement here? And how can I make it involve a cool new technology I've been wanting to try out? Clearly the requirement here is that people want to have some kind of user account on the site. So, without further ado, let's dive into authentication. ((("passwords"))) Naturally we're not going to mess about with remembering passwords ourselves--besides being _so_ '90s, secure storage of user passwords is a security nightmare we'd rather leave to someone else. We'll use something fun called "passwordless authentication" instead.footnote:[If you _insist_ on storing your own passwords, Django's default authentication module is ready and waiting for you. It's nice and straightforward, and I'll leave it to you to discover on your own.] === Passwordless Auth with "Magic Links" ((("authentication", "passwordless"))) ((("magic links"))) ((("OAuth"))) ((("Openid"))) What authentication system could we use to avoid storing passwords ourselves? OAuth? OpenID? "Sign in with Facebook"? Ugh.((("passwords", "passwordless authentication with magic links"))) For me, those all have unacceptable creepy overtones; why should Google or Facebook know what sites you're logging in to and when? Instead, for the second edition,footnote:[ In the first edition, I used an experimental project called "Persona", cooked up by some of the wonderful techno-hippie-idealists at Mozilla, but sadly that project was abandoned.] I found a fun approach to authentication that now goes by the name of "Magic Links", but you might call it "just use email". The system was invented (or at least popularised) back in 2014((("emails", "using to verify identity"))) by someone annoyed at having to create new passwords for so many websites. They found themselves just using random, throwaway passwords, not even trying to remember them, and using the "forgot my password" feature whenever they needed to log in again. You can https://oreil.ly/je14i[read all about it on Medium]. The concept is: just use email to verify someone's identity. If you're going to have a "forgot my password" feature, then you're trusting email anyway, so why not just go the whole hog? Whenever someone wants to log in, we generate a unique URL for them to use, email it to them, and they then click through that to get into the site. It's by no means a perfect system, and in fact there are lots of subtleties to be thought through before it would really make a good login solution for a production website, but this is just a fun toy project so let's give it a go. === A Somewhat Larger Spike NOTE: Reminder: a spike is a phase of exploratory coding, where we can code without tests, in order to explore a new tool or experiment with a new idea. We will come back and redo the code "properly" with TDD later.((("spiking and de-spiking", "spiking magic links authentication", id="ix_spkauth"))) ((("django-allauth"))) ((("python-social-auth"))) To get this Magic Links project set up, the first thing I did was take a look at existing Python and Django authentication packages, like https://docs.allauth.org[django-allauth], but both of them looked overcomplicated for this stage (and besides, it'll be more fun to code our own!). So instead, I dived in and hacked about, and after a few dead ends and wrong turns, I had something that just about works. I'll take you on a tour, and then we'll go through and "de-spike" the implementation--that is, replace the prototype with tested, production-ready code. You should go ahead and add this code to your own site too, and then you can have a play with it. Try logging in with your own email address, and convince yourself that it really does work. ==== Starting a Branch for the Spike ((("spiking and de-spiking", "branching your VCS"))) ((("Git", "creating branches"))) This spike is going to be a bit more involved than the last one, so we'll be a little more rigorous with our version control. Before embarking on a spike it's a good idea to start a new branch, so you can still use your VCS without worrying about your spike commits getting mixed up with your production code: [subs="specialcharacters,quotes"] ---- $ *git switch -c passwordless-spike* ---- Let's keep track of some of the things we're hoping to learn from the spike: [role="scratchpad"] ***** * _How to send emails_ * _Generating and recognising unique tokens_ * _How to authenticate someone in Django_ ***** ==== Frontend Login UI ((("authentication", "frontend login UI"))) Let's start with the frontend by adding in an actual form to enter your email address into the navbar, along with a logout link for users who are already authenticated: [role="sourcecode"] .src/lists/templates/base.html (ch19l001) ==== [source,html] ---- <body> <div class="container"> <div class="navbar"> {% if user.is_authenticated %} <p>Logged in as {{ user.email }}</p> <form method="POST" action="/accounts/logout"> {% csrf_token %} <button id="id_logout" type="submit">Log out</button> </form> {% else %} <form method="POST" action ="accounts/send_login_email"> Enter email to log in: <input name="email" type="text" /> {% csrf_token %} </form> {% endif %} </div> <div class="row justify-content-center p-5 bg-body-tertiary rounded-3"> [...] ---- ==== ==== Sending Emails from Django ((("authentication", "sending emails from Django", id="SDemail18"))) ((("Django framework", "sending emails", id="DFemail18"))) ((("send_mail function", id="sendmail18"))) ((("emails", "sending from Django", id="ix_emlDj"))) The login will be something like <<magic-links-diagram>>. [[magic-links-diagram]] .Overview of the Magic Links login process image::images/tdd3_1901.png["A diagram showing the interaction between user and server. First the user fills in a login form and gives their email. the server creates a token associated with that email, and then sends them an email containg a magic link, which includes the token. the user waits for the email. they click the link in the email, which goes to another endpoint on the server. it then checks if the token is valid, and if so, logs the user in."] 1. When someone wants to log in, we generate a unique secret token for them, link it to their email, store it in the database, and send it to them. 2. The user then checks their email, which will have a link for a URL that includes that token. 3. When they click that link, we check whether the token exists in the database and, if so, they are logged in as the associated user. // https://docs.djangoproject.com/en/5.2/topics/auth/customizing/ First, let's prep an app for our accounts stuff: [subs="specialcharacters,quotes"] ---- $ *cd src && python manage.py startapp accounts && cd ..* $ *ls src/accounts* __init__.py admin.py apps.py migrations models.py tests.py views.py ---- // DAVID: Worth discussing why you chose to make this an app? And we'll wire up _urls.py_ with at least one URL. In the top-level _superlists/urls.py_... [role="sourcecode"] .src/superlists/urls.py (ch19l003) ==== [source,python] ---- from django.urls import include, path from lists import views as list_views urlpatterns = [ path("", list_views.home_page, name="home"), path("lists/", include("lists.urls")), path("accounts/", include("accounts.urls")), ] ---- ==== [role="pagebreak-before"] And we give the accounts module its own _urls.py_: [role="sourcecode"] .src/accounts/urls.py (ch19l004) ==== [source,python] ---- from django.urls import path from accounts import views urlpatterns = [ path("send_login_email", views.send_login_email, name="send_login_email"), ] ---- ==== Here's the view that's in charge of((("tokens", "creating, view for"))) creating a token associated with the email address that the user puts in our login form: [role="sourcecode"] .src/accounts/views.py (ch19l005) ==== [source,python] ---- import sys import uuid from django.core.mail import send_mail from django.shortcuts import render from accounts.models import Token def send_login_email(request): email = request.POST["email"] uid = str(uuid.uuid4()) Token.objects.create(email=email, uid=uid) print("saving uid", uid, "for email", email, file=sys.stderr) url = request.build_absolute_uri(f"/accounts/login?uid={uid}") send_mail( "Your login link for Superlists", f"Use this link to log in:\n\n{url}", "noreply@superlists", [email], ) return render(request, "login_email_sent.html") ---- ==== For that to work, we'll need((("templates", "messaging confirming login email sent"))) a template with a placeholder message confirming the email was sent: [role="sourcecode"] .src/accounts/templates/login_email_sent.html (ch19l006) ==== [source,html] ---- <html> <h1>Email sent</h1> <p>Check your email, you'll find a message with a link that will log you into the site.</p> </html> ---- ==== (You can see how hacky this code is--we'd want to integrate this template with our 'base.html' in the real version.) ==== Email Server Config for Django The https://docs.djangoproject.com/en/5.2/topics/email[django docs on email] explain how `send_mail()` works, as well as how you configure it by telling Django what email server to use, and how to authenticate with it. Here, I'm just using my Gmailfootnote:[ Didn't I just spend a whole intro banging on about the privacy implications of using Google for login, only to go on and use Gmail? Yes, it's a contradiction (honest, I will move off Gmail one day!).((("SMTP (Simple Mail Transfer Protocol)"))) But in this case I'm just using it for testing, and the important thing is that I'm not forcing Google on my users.] account for now—but you can use any email provider you like, as long as they support SMTP (Simple Mail Transfer Protocol): [role="sourcecode"] .src/superlists/settings.py (ch19l007) ==== [source,python] ---- EMAIL_HOST = "smtp.gmail.com" EMAIL_HOST_USER = "obeythetestinggoat@gmail.com" EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_PASSWORD") EMAIL_PORT = 587 EMAIL_USE_TLS = True ---- ==== TIP: If you want to use Gmail as well, you'll probably have to visit your Google account security settings page. If you're using two-factor authentication, you'll want to set up an https://myaccount.google.com/apppasswords[app-specific password]. If you're not, you will probably still need to https://www.google.com/settings/security/lesssecureapps[allow access for less secure apps]. You might want to consider creating a new Google account for this purpose, rather than using one containing sensitive data. ((("Gmail"))) ((("emails", "sending from Django", startref="ix_emlDj"))) ((("", startref="sendmail18"))) ((("", startref="DFemail18"))) ((("", startref="SDemail18"))) ==== Another Secret, Another Environment Variable ((("authentication", "avoiding secrets in source code"))) ((("environment variables")))((("secrets", "storing in environment variables"))) Once again, we have a "secret" that we want to avoid keeping directly in our source code or on GitHub, so another environment variable is used in the `os.environ.get`. To get this to work, we need to set it in the shell that's running my dev server: [subs="specialcharacters,quotes"] ---- $ *export EMAIL_PASSWORD="ur-email-server-password-here"* ---- Later, we'll see about adding that to the env file on the staging server as well. [role="pagebreak-before less_space"] ==== Storing Tokens in the Database // CSANAD (transcribed) you should probably hash the tokens ((("authentication", "storing tokens in databases"))) ((("tokens", "storing in the database"))) How are we doing? Let's review where we're at in the process: [role="scratchpad"] ***** * _[strikethrough line-through]#How to send emails#_ * _Generating and recognising unique tokens_ * _How to authenticate someone in Django_ ***** // DAVID: In practice would we really cross something off the list like this before giving it a try? // Might be better to gradually build things up, e.g. write a function to send an email (and check it works). // Could even use as an excuse to introduce manage.py shell and do it from there? // Equally with the user interface stuff, maybe starting up the application and having a look at what it looks like? // Or maybe start with the model and then layer things on top of that. We'll need a model to store our tokens in the database--they link an email address with a unique ID. It's pretty simple: [role="sourcecode"] .src/accounts/models.py (ch19l008) ==== [source,python] ---- from django.db import models class Token(models.Model): email = models.EmailField() uid = models.CharField(max_length=255) <1> ---- ==== <1> Django does have a specific UID (universally unique identifier) fields type for many databases, but I just want to keep things simple for now. The point of this spike is about authentication and emails, not optimising database storage. We've got enough things we need to learn as it is! Let's switch on our new accounts app in _settings.py_: [role="sourcecode"] .src/superlists/settings.py (ch19l008-1) ==== [source,python] ---- INSTALLED_APPS = [ # "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", "lists", "accounts", ] ---- ==== [role="pagebreak-before"] We can then do a quick ((("database migrations", "adding token model to database")))migrations dance to add the token model to the database: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py makemigrations*] Migrations for 'accounts': src/accounts/migrations/0001_initial.py + Create model Token $ pass:quotes[*python src/manage.py migrate*] Operations to perform: Apply all migrations: accounts, auth, contenttypes, lists, sessions Running migrations: Applying accounts.0001_initial... OK ---- //ch19l008-2 And at this point, if you actually try the email form in your browser, you'll see it really does send an actual real email—to your real email address hopefully (best not spam someone else now!). See Figures <<spike-email-sent, 19-2>> and <<spike-email-received, 19-3>>. [[spike-email-sent]] .Looks like we might have sent an email image::images/tdd3_1902.png["The email sent confirmation page, indicating the server at least thinks it sent an email successfully"] [[spike-email-received]] .Yep, looks like we received it image::images/tdd3_1903.png["Screenshot of my email client showing the email from the server, saying 'your login link for superlist' and including a token url"] [role="pagebreak-before less_space"] ==== Custom Authentication Models ((("authentication", "custom authentication models"))) OK, so we've done the first half of "Generating and recognising unique tokens": [role="scratchpad"] ***** * '[strikethrough line-through]#How to send emails#' * '[strikethrough line-through]#Generating# and recognising unique tokens' * 'How to authenticate someone in Django' ***** But, before we can move on to recognising them and making the login work end-to-end though, we need to explore Django's authentication system.((("Django framework", "authentication system")))((("user models", "Django authentication user model"))) The first thing we'll need is a user model. I took a dive into the https://docs.djangoproject.com/en/5.2/topics/auth/customizing[Django auth documentation] and tried to hack in the simplest possible one: [role="sourcecode"] .src/accounts/models.py (ch19l009) ==== [source,python] ---- from django.contrib.auth.models import ( AbstractBaseUser, BaseUserManager, ) [...] class ListUser(AbstractBaseUser): email = models.EmailField(primary_key=True) USERNAME_FIELD = "email" # REQUIRED_FIELDS = ['email', 'height'] objects = ListUserManager() @property def is_staff(self): return self.email == "harry.percival@example.com" @property def is_active(self): return True ---- ==== // DAVID: Maybe better include the ListUserManager() here too? Or leave it out until we create it? [role="pagebreak-before"] That's what I call a minimal user model! One field, none of this first name/last name/username nonsense, and—pointedly—no password! That's somebody else's problem! But, again, you can see that this code isn't ready for production—from the commented-out lines to the hardcoded Harry email address. We'll neaten this up quite a lot when we de-spike. To get it to work, I needed to add a model manager for the user, for some reason: [role="sourcecode small-code"] .src/accounts/models.py (ch19l010) ==== [source,python] ---- [...] class ListUserManager(BaseUserManager): def create_user(self, email): ListUser.objects.create(email=email) def create_superuser(self, email, password): self.create_user(email) ---- ==== // CSANAD: ListUserManager has to be defined before ListUser, since its // reference to ListUser isn't evaluated until `create_user` is called. This is // not the case the other way around, ListUser's reference to ListUserManager // is instantiated in the class definition. Maybe we could leave a note about // this? No need to worry about what a model manager is at this stage; for now, we just need it because we need it, and it works. When we de-spike, we'll examine each bit of code that actually ends up in production and make sure we understand it fully. We'll need to run `makemigrations` and `migrate` again to make the user model real: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py makemigrations*] Migrations for 'accounts': src/accounts/migrations/0002_listuser.py + Create model ListUser $ pass:quotes[*python src/manage.py migrate*] [...] Running migrations: Applying accounts.0002_listuser... OK ---- //ch19l009-1 ==== Finishing the Custom Django Auth Let's review our scratchpad: [role="scratchpad"] ***** * _[strikethrough line-through]#How to send emails#_ * _[strikethrough line-through]#Generating# and recognising unique tokens_ * _How to authenticate someone in Django_ ***** [role="pagebreak-before"] ((("Django framework", "custom authentication system", id="ix_Djcusauth")))((("authentication", "custom Django authentication", id="SDcustom18"))) Hmm, we can't quite cross off anything yet. Turns out the steps we _thought_ we'd go through aren't quite the same as the steps we're _actually_ going through (this is not uncommon, as I'm sure you know). // CSANAD: I find it vague like this. Maybe it would be helpful to clarify what // it is that "we are actually going through" and what it was that // "we thought we'd go through". Still, we're almost there--our last step will combine recognising the token and then actually logging the user in. Once we've done this, we'll be able to pretty much strike off all the items on our scratchpad. So here's the view that actually handles the click-through from the link in the email: [role="sourcecode small-code"] .src/accounts/views.py (ch19l011) ==== [source,python] ---- import sys import uuid from django.contrib.auth import authenticate from django.contrib.auth import login as auth_login from django.core.mail import send_mail from django.shortcuts import redirect, render from accounts.models import Token def send_login_email(request): [...] def login(request): print("login view", file=sys.stderr) uid = request.GET.get("uid") user = authenticate(request, uid=uid) if user is not None: auth_login(request, user) return redirect("/") ---- ==== The `authenticate()` function invokes Django's authentication framework, which we configure using a custom "authentication backend," whose job it is to validate the UID (unique identifier) and return a user with the right email. [role="pagebreak-before"] We could have done this stuff directly in the view, but we may as well structure things the way Django expects. It makes for a reasonably neat separation of concerns: [role="sourcecode"] .src/accounts/authentication.py (ch19l012) ==== [source,python] ---- import sys from accounts.models import ListUser, Token from django.contrib.auth.backends import BaseBackend class PasswordlessAuthenticationBackend(BaseBackend): def authenticate(self, request, uid): print("uid", uid, file=sys.stderr) if not Token.objects.filter(uid=uid).exists(): print("no token found", file=sys.stderr) return None token = Token.objects.get(uid=uid) print("got token", file=sys.stderr) try: user = ListUser.objects.get(email=token.email) print("got user", file=sys.stderr) return user except ListUser.DoesNotExist: print("new user", file=sys.stderr) return ListUser.objects.create(email=token.email) def get_user(self, email): return ListUser.objects.get(email=email) ---- ==== Again, lots of debug prints in there, and some duplicated code—not something we'd want in production, but it works...as long as we add it to _settings.py_ (it doesn't matter where): [role="sourcecode"] .src/superlists/settings.py (ch19l012-1) ==== [source,python] ---- AUTH_USER_MODEL = "accounts.ListUser" AUTHENTICATION_BACKENDS = [ "accounts.authentication.PasswordlessAuthenticationBackend", ] ---- ==== [role="pagebreak-before"] And finally, a logout view: [role="sourcecode"] .src/accounts/views.py (ch19l013) ==== [source,python] ---- from django.contrib.auth import authenticate from django.contrib.auth import login as auth_login from django.contrib.auth import logout as auth_logout [...] def logout(request): auth_logout(request) return redirect("/") ---- ==== Add login and logout to our _urls.py_... [role="sourcecode"] .src/accounts/urls.py (ch19l014) ==== [source,python] ---- urlpatterns = [ path("send_login_email", views.send_login_email, name="send_login_email"), path("login", views.login, name="login"), path("logout", views.logout, name="logout"), ] ---- ==== And we should be all done! Spin up a dev server with `runserver` and try it--believe it or not, it _actually_ works (see <<spike-login-worked>>). [[spike-login-worked]] .It works! It works! image::images/tdd3_1904.png["screenshot of several windows including gmail and terminals but in the foreground our site showing us as being logged in."] TIP: If you get an `SMTPSenderRefused` error message, don't forget to set the `EMAIL_PASSWORD` environment variable in the shell that's running `runserver`. ((("SMTPSenderRefused error message")))Also, if you see a message saying "Application-specific password required", that's a Gmail security policy. Follow the link in the error message. That's pretty much it! Along the way, I had to fight pretty hard, including clicking around the Gmail account security UI for a while, stumbling over several missing attributes on my custom user model (because I didn't read the docs properly), and even at one point switching to the dev version of Django to overcome a bug, which thankfully turned out to be a red herring. ((("Django framework", "custom authentication system", startref="ix_Djcusauth")))((("", startref="SDcustom18"))) But we now have a working solution! Let's commit it on our spike branch: [subs="specialcharacters,quotes"] ---- $ *git status* $ *git add src/accounts* $ *git commit -am "spiked in custom passwordless auth backend"* ---- [role="scratchpad"] ***** * _[strikethrough line-through]#How to send emails#_ * _[strikethrough line-through]#Generating and recognising unique tokens#_ * _[strikethrough line-through]#_How to authenticate someone in Django#_ ***** Time to de-spike! [role="pagebreak-before less_space"] === De-spiking ((("spiking and de-spiking", "spiking magic links authentication", startref="ix_spkauth")))((("spiking and de-spiking", "de-spiking authentication code", id="SDde18")))((("authentication", "de-spiking authentication code", id="ix_authdes"))) De-spiking means rewriting your prototype code using TDD. In this section, we’ll work through how to do that in a safe and methodical way. We’ll take the knowledge we’ve acquired during the spiking process—whether that’s in our heads, in our notes, or in our branch in Git—and apply it as we re-implement gradually in a test-first way. And the hope is that our code will turn out a bit nicer the second time around! ==== Making a Plan While it's fresh in our minds, let's make a few notes based on what we've learned about what we know we're probably going to need to build during our de-spike: [role="scratchpad"] ***** * _Token model with email and UID_ * _View to create token and send login email incl. url w/ token UID_ * _Custom user model with USERNAME_FIELD=email_ * _Authentication backend with authenticate() and get_user() functions_ * _Registering auth backend in settings.py_ * _Login view calls authenticate() and login() from django.contrib.auth_ * _Logout view calls django.contrib.auth.logout_ ***** ==== Wring an FT Against the Spiked Code We now have enough information to "do it properly". So, what's the first step? An FT, of course! We'll stay on the spike branch for now to see our FT pass against our spiked code. Then we'll go back to our main branch and commit just the FT. [role="pagebreak-before"] Here's a first, simple version of the FT: [role="sourcecode small-code"] .src/functional_tests/test_login.py (ch19l018) ==== [source,python] ---- import re from django.core import mail from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from .base import FunctionalTest TEST_EMAIL = "edith@example.com" # <1> SUBJECT = "Your login link for Superlists" class LoginTest(FunctionalTest): def test_login_using_magic_link(self): # Edith goes to the awesome superlists site # and notices a "Log in" section in the navbar for the first time # It's telling her to enter her email address, so she does self.browser.get(self.live_server_url) self.browser.find_element(By.CSS_SELECTOR, "input[name=email]").send_keys( TEST_EMAIL, Keys.ENTER ) # A message appears telling her an email has been sent self.wait_for( lambda: self.assertIn( "Check your email", self.browser.find_element(By.CSS_SELECTOR, "body").text, ) ) # She checks her email and finds a message email = mail.outbox.pop() # <2> self.assertIn(TEST_EMAIL, email.to) self.assertEqual(email.subject, SUBJECT) # It has a URL link in it self.assertIn("Use this link to log in", email.body) url_search = re.search(r"http://.+/.+$", email.body) if not url_search: self.fail(f"Could not find url in email body:\n{email.body}") url = url_search.group(0) self.assertIn(self.live_server_url, url) # she clicks it self.browser.get(url) # she is logged in! self.wait_for( lambda: self.browser.find_element(By.CSS_SELECTOR, "#id_logout"), ) navbar = self.browser.find_element(By.CSS_SELECTOR, ".navbar") self.assertIn(TEST_EMAIL, navbar.text) ---- ==== <1> Whenever you're testing against something that can send real emails, you don't want to use a real address. It's best practice to use a special domain like `@example.com`, which has been reserved for exactly this sort of thing, to avoid accidentally spamming anyone! <2> Were you worried about how we were going to handle retrieving emails in our tests? Thankfully, we can cheat for now! When running tests, Django gives us access to any emails that the server tries to send via the `mail.outbox` attribute. We'll discuss checking "real" emails in <<chapter_23_debugging_prod>>. And if we run the FT, it works! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_login*] [...] Not Found: /favicon.ico saving uid [...] login view uid [...] got token new user . --------------------------------------------------------------------- Ran 1 test in 2.729s OK ---- You can even see some of the debug output I left in my spiked view implementations. Now it's time to revert all of our temporary changes, and reintroduce them one by one in a test-driven way. ==== Reverting Our Spiked Code We can revert our spike using our version ((("Git", "reverting spiked code")))control system: [subs="specialcharacters,quotes"] ---- $ *git switch main* # switch back to main branch $ *rm -rf src/accounts* # remove any trace of spiked code $ *git add src/functional_tests/test_login.py* $ *git commit -m "FT for login via email"* ---- Now we rerun the FT and let it be the main driver of our development, referring back to our scratchpad from time to time when we need to: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_login*] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: input[name=email]; [...] [...] ---- TIP: If you see an exception saying "No module named accounts", you may have missed a step in the de-spiking process—maybe a commit or the change of branch. The first thing it wants us to do is add an email input element. Bootstrap has some built-in classes for navigation bars, so we'll use them, and include a form for the login email:footnote:[ We are now introducing a conceptual dependency from the base template to the `accounts` app because its URL is in the form. I didn't want to spend time on this in the book, but this might be a good time to consider moving the template out of _lists/templates_ and into _superlists/templates_. By convention, that's the place for templates whose scope is wider than a single app.] [role="sourcecode"] .src/lists/templates/base.html (ch19l020) ==== [source,html] ---- <body> <div class="container"> <nav class="navbar"> <div class="container-fluid"> <a class="navbar-brand" href="/">Superlists</a> <form method="POST" action="/accounts/send_login_email"> <div class="input-group"> <label class="navbar-text me-2" for="id_email_input"> Enter your email to log in </label> <input id="id_email_input" name="email" class="form-control" placeholder="your@email.com" /> {% csrf_token %} </div> </form> </div> </nav> <div class="row justify-content-center p-5 bg-body-tertiary rounded-3"> <div class="col-lg-6 text-center"> <h1 class="display-1 mb-4">{% block header_text %}{% endblock %}</h1> [...] ---- ==== [role="pagebreak-before"] At this point, you'll find that the unit tests start to fail: ---- ERROR: test_renders_input_form [...] [form] = parsed.cssselect("form[method=POST]") ^^^^^^ ValueError: too many values to unpack (expected 1, got 2) ERROR: test_renders_input_form [form] = parsed.cssselect("form[method=POST]") ^^^^^^ ValueError: too many values to unpack (expected 1, got 2) ---- It's because these unit tests had a hard assumption that there's only one POST form on the page. Let's change them to be more resilient. Here's how you might change the first one: [role="sourcecode small-code"] .src/lists/tests/test_views.py (ch19l020-1) ==== [source,python] ---- def test_renders_input_form(self): response = self.client.get("/") parsed = lxml.html.fromstring(response.content) forms = parsed.cssselect("form[method=POST]") # <1> self.assertIn("/lists/new", [form.get("action") for form in forms]) # <2> [form] = [form for form in forms if form.get("action") == "/lists/new"] # <3> inputs = form.cssselect("input") # <4> self.assertIn("text", [input.get("name") for input in inputs]) # <4> ---- ==== <1> We get all forms, rather than using the clever `[form] =` syntax. <2> We check that at least _one_ of the forms has the right `action=` URL. I'm using `assertIn()`, so we get a nice error message. If we can't find the right URL, we'll see the list of URLs that _do_ exist on the page. <3> Now we can feel free to go back to unpacking, and get the right form, based on its `action` attribute. <4> The rest of the test is as before. [role="pagebreak-before"] Here's a similar set of changes in the second test: [role="sourcecode"] .src/lists/tests/test_views.py (ch19l020-2) ==== [source,diff] ---- @@ -65,10 +65,12 @@ class ListViewTest(TestCase): def test_renders_input_form(self): mylist = List.objects.create() - response = self.client.get(f"/lists/{mylist.id}/") + url = f"/lists/{mylist.id}/" + response = self.client.get(url) parsed = lxml.html.fromstring(response.content) - [form] = parsed.cssselect("form[method=POST]") - self.assertEqual(form.get("action"), f"/lists/{mylist.id}/") + forms = parsed.cssselect("form[method=POST]") + self.assertIn(url, [form.get("action") for form in forms]) + [form] = [form for form in forms if form.get("action") == url] inputs = form.cssselect("input") self.assertIn("text", [input.get("name") for input in inputs]) ---- ==== It's pretty much the same edit, except this time I decided to have a `url` variable, to remove the duplication of using `/lists/{mylist.id}/` three times. That gets our unit tests passing again: ---- OK ---- If we try our FT again, we'll see it fails because the login form doesn't send us to a real URL yet--you'll see the `Not found:` message in the server output, as well as the assertion reporting the content of the default 404 page: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_login*] [...] Not Found: /accounts/send_login_email [...] AssertionError: 'Check your email' not found in 'Not Found\nThe requested resource was not found on this server.' ---- Time to start writing some Django code. We begin, like in the spike, by creating an app called `accounts` to hold all the files related to login: [subs="specialcharacters,quotes"] ---- $ *cd src && python manage.py startapp accounts && cd ..* $ *ls src/accounts* __init__.py admin.py apps.py migrations models.py tests.py views.py ---- //ch19l021 You could even do a commit just for that, to be able to distinguish the placeholder app files from our modifications. ((("authentication", "de-spiking authentication code", startref="ix_authdes")))((("spiking and de-spiking", "de-spiking authentication code", startref="ix_spikdeauth"))) === A Minimal Custom User Model // IDEA: consider starting with a test for the login view instead. ((("user models", "minimum custom user model for authentication", id="ix_usrmdcus")))((("authentication", "minimal custom user model", id="SDminimal18"))) Let's turn to the models layer:footnote:[ In this chapter, we're building things in a "bottom-up" way, starting with the models, and then building the layers on top—the views and templates that depend on them. This is a common approach, but it's not the only one! In <<chapter_24_outside_in>> we'll explore building software from the outside in, which has all sorts of advantages too.] [role="scratchpad"] ***** * _Token model with email and UID_ * _View to create token and send login email incl. url w/ token UID_ * _Custom user model with USERNAME_FIELD=email_ * _Authentication backend with authenticate() and get_user() functions_ * _Registering auth backend in settings.py_ * _Login view calls authenticate() and login() from django.contrib.auth_ * _Logout view calls django.contrib.auth.logout_ ***** We know we have to build a token model and a custom user model, and the user model was the messiest part in our spike. So, let's have a go at redoing that test-first, to see if it comes out nicer. Django's built-in user model makes all sorts of assumptions about what information you want to track about users—from explicitly requiring a first name and last namefootnote:[ This is a decision that even some prominent Django maintainers have said they now regret—not everyone has a first and last name.] to forcing you to use a username. I'm a great believer in not storing information about users unless you absolutely must, so a user model that records an email address and nothing else sounds good to me! Let's start straight away with a tests folder instead of _tests.py_ in this app: [subs=""] ---- $ <strong>rm src/accounts/tests.py</strong> $ <strong>mkdir src/accounts/tests</strong> $ <strong>touch src/accounts/tests/__init__.py</strong> ---- And now, let's add a _test_models.py_ to say: [role="sourcecode"] .src/accounts/tests/test_models.py (ch19l023) ==== [source,python] ---- from django.test import TestCase from accounts.models import User class UserModelTest(TestCase): def test_user_is_valid_with_email_only(self): user = User(email="a@b.com") user.full_clean() # should not raise ---- ==== That gives us the expected failure: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] ImportError: cannot import name 'User' from 'accounts.models' (...goat-book/src/accounts/models.py) ---- OK, let's try the absolute minimum then: [role="sourcecode"] .src/accounts/models.py (ch19l024) ==== [source,python] ---- from django.db import models class User(models.Model): email = models.EmailField() ---- ==== That gives us an error because Django won't recognise models unless they're in `INSTALLED_APPS`: [subs="specialcharacters,macros"] ---- RuntimeError: Model class accounts.models.User doesn't declare an explicit app_label and isn't in an application in INSTALLED_APPS. ---- So, let's add it to _settings.py_: [role="sourcecode"] .src/superlists/settings.py (ch19l025) ==== [source,python] ---- INSTALLED_APPS = [ # "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", "accounts", "lists", ] ---- ==== And that gets our tests passing! ---- OK ---- Now let's see if we've built a user model that Django can actually work with. There's a built-in function in `django.contrib.auth` called `get_user_model()`—which retrieves the currently active user model and, as we'll see, also performs some checks on it. Let's use it in our tests: [role="sourcecode"] .src/accounts/tests/test_models.py (ch19l026-1) ==== [source,python] ---- from django.contrib import auth from django.test import TestCase from accounts.models import User class UserModelTest(TestCase): def test_model_is_configured_for_django_auth(self): self.assertEqual(auth.get_user_model(), User) def test_user_is_valid_with_email_only(self): [...] ---- ==== That gives: ---- AssertionError: <class 'django.contrib.auth.models.User'> != <class 'accounts.models.User'> ---- OK, so let's try wiring up our model inside _settings.py_, in a variable called `AUTH_USER_MODEL`: [role="sourcecode"] .src/superlists/settings.py (ch19l026-2) ==== [source,python] ---- AUTH_USER_MODEL = "accounts.User" ---- ==== Now when we run our tests, Django complains that our custom user model is missing a couple of bits of metadata. In fact, it's so unhappy that it won't even run the tests: [role="ignore-errors"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] Traceback (most recent call last): [...] File ".../django/contrib/auth/checks.py", line 46, in check_user_model if not isinstance(cls.REQUIRED_FIELDS, (list, tuple)): ^^^^^^^^^^^^^^^^^^^ AttributeError: type object 'User' has no attribute 'REQUIRED_FIELDS' ---- Sigh. Come on, Django; it's only got one field, so you should be able to figure out the answers to these questions for yourself. [role="pagebreak-before"] Here you go: [role="sourcecode"] .src/accounts/models.py (ch19l027) ==== [source,python] ---- class User(models.Model): email = models.EmailField() REQUIRED_FIELDS = [] ---- ==== Next silly question?footnote:[ You might ask, if I think Django is so silly, why don't I submit a pull request to fix it? It should be quite a simple fix. Well, I promise I will, as soon as I've finished updating the book. For now, snarky comments will have to suffice.] [subs="specialcharacters,macros"] ---- AttributeError: type object 'User' has no attribute 'USERNAME_FIELD' ---- We'll go through a few more of these, until we get to: [role="sourcecode"] .src/accounts/models.py (ch19l029) ==== [source,python] ---- class User(models.Model): email = models.EmailField() REQUIRED_FIELDS = [] USERNAME_FIELD = "email" is_anonymous = False is_authenticated = True ---- ==== And now we get a slightly different error: [role="ignore-errors"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] SystemCheckError: System check identified some issues: ERRORS: accounts.User: (auth.E003) 'User.email' must be unique because it is named as the 'USERNAME_FIELD'. ---- Well, the simple way to fix that would be like this: [role="sourcecode"] .src/accounts/models.py (ch19l030) ==== [source,python] ---- email = models.EmailField(unique=True) ---- ==== And now we get a different error again, slightly more familiar this time! Django is a bit happier with the structure of our custom user model, but it's unhappy about the database: ---- django.db.utils.OperationalError: no such table: accounts_user ---- [role="pagebreak-before"] In other words, we need to create a migration: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py makemigrations*] Migrations for 'accounts': src/accounts/migrations/0001_initial.py + Create model User ---- //ch19l031 And our tests pass: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test accounts* [...] Ran 2 tests in 0.001s OK ---- But our model isn't quite as simple as it could be. It has the email field, and also an autogenerated "ID" field as its primary key. We could make it even simpler! // DAVID: Maybe spell this out more clearly to the reader that there are actually two fields, // they might not realise this. ==== Tests as Documentation ((("tests", "as documentation", secondary-sortas="documentation"))) ((("documentation, tests as"))) Let's go all the way and make the email field the primary key,footnote:[ Emails may not be the perfect primary key in real life. One reader—clearly deeply scarred—wrote me an emotional email about how much they've suffered for over a decade from trying to deal with the consquences of using email as a primary key, particularly how it makes multiuser account management nearly impossible. So, as ever, YMMV.] and thus implicitly remove the autogenerated `id` column. Although we could just _do it_ and our test would still pass, and conceivably claim it was "just a refactor", it would be better to have a specific test: [role="sourcecode"] .src/accounts/tests/test_models.py (ch19l032) ==== [source,python] ---- class UserModelTest(TestCase): def test_model_is_configured_for_django_auth(self): [...] def test_user_is_valid_with_email_only(self): [...] def test_email_is_primary_key(self): user = User(email="a@b.com") self.assertEqual(user.pk, "a@b.com") ---- ==== It'll help us remember if we ever come back and look at the code again in future: ---- self.assertEqual(user.pk, "a@b.com") AssertionError: None != 'a@b.com' ---- TIP: Your tests can be a form of documentation for your code--they express the requirements for a particular class or function. Sometimes, if you forget why you've done something a particular way, going back and looking at the tests will give you the answer. That's why it's important to make your tests readable, including giving them explicit, verbose method names. Here's the implementation (`primary_key` makes the `unique=True` obsolete): [role="sourcecode"] .src/accounts/models.py (ch19l033) ==== [source,python] ---- email = models.EmailField(primary_key=True) ---- ==== And we mustn't forget to adjust our migrations: [subs="specialcharacters,macros"] ---- $ pass:quotes[*rm src/accounts/migrations/0001_initial.py*] $ pass:quotes[*python src/manage.py makemigrations*] Migrations for 'accounts': src/accounts/migrations/0001_initial.py + Create model User ---- //ch19l034 // DAVID: Deleting migrations can get readers in a pickle if they have already run migrations locally. // Might be worth saying we're only doing this because we've just created it, and advise them to delete // their database if they happen to have run the migration they've just deleted? (Or you can get them // to run `migrate accounts zero` I think.) ((("user models", "minimum custom user model for authentication", startref="ix_usrmdcus")))((("", startref="SDminimal18"))) Now both our tests pass: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] Ran 3 tests in 0.001s OK ---- It's probably a good time for a commit, too: [subs="specialcharacters,quotes"] ---- $ *git add src/accounts* $ *git commit -m "custom user model with email as primary key"* ---- And we can cross off one item from our de-spiking list. Hooray! [role="scratchpad"] ***** * _Token model with email and UID_ * _View to create token and send login email incl. url w/ token UID_ * _[strikethrough line-through]#Custom user model with USERNAME_FIELD=email#_ * _Authentication backend with authenticate() and get_user() functions_ * _Registering auth backend in settings.py_ * _Login view calls authenticate() and login() from django.contrib.auth_ * _Logout view calls django.contrib.auth.logout_ ***** === A Token Model to Link Emails with a Unique ID ((("emails", "token model linking with unique ID", id="ix_emltkn")))((("authentication", "token model to link emails", id="SDtoken18")))((("tokens", "token model linking emails and UID", id="ix_tknmod"))) Next let's build a token model. Here's a short unit test that captures the essence--you should be able to link an email to a unique ID, and that ID shouldn't be the same twice in a row: [role="sourcecode"] .src/accounts/tests/test_models.py (ch19l035) ==== [source,python] ---- from accounts.models import Token, User [...] class TokenModelTest(TestCase): def test_links_user_with_auto_generated_uid(self): token1 = Token.objects.create(email="a@b.com") token2 = Token.objects.create(email="a@b.com") self.assertNotEqual(token1.uid, token2.uid) ---- ==== I won't show every single listing for creating the token class in _models.py_; I'll let you do that yourself instead. Driving Django models with basic TDD involves jumping through a few hoops because of the migration, so you'll see a few iterations like this--minimal code change, make migrations, get new error, delete migrations, re-create new migrations, another code change, and so on... [role="dofirst-ch19l036"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] TypeError: Token() got unexpected keyword arguments: 'email' ---- I'll trust you to go through these conscientiously--remember, I may not be able to see you, but the Testing Goat can! You might go through a hoop like this one, for example, where you find yourself needing to create and then delete a migration for an incomplete solution: [role="dofirst-ch19l037"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py makemigrations*] Migrations for 'accounts': src/accounts/migrations/0002_token.py + Create model Token $ pass:quotes[*python src/manage.py test accounts*] AttributeError: 'Token' object has no attribute 'uid'. Did you mean: 'id'? $ pass:quotes[*rm src/accounts/migrations/0002_token.py*] ---- Eventually, you should get to this code... [role="sourcecode dofirst-ch19l038-0"] .src/accounts/models.py (ch19l038) ==== [source,python] ---- class Token(models.Model): email = models.EmailField() uid = models.CharField(max_length=40) ---- ==== // DAVID: could it confuse people that the max_length is 40 here but 255 in the spike? And this error: [role="dofirst-ch19l039"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] self.assertNotEqual(token1.uid, token2.uid) AssertionError: '' == '' ---- And here we have to decide how to generate our random unique ID field. We could use the `random` module, but Python actually comes with another module specifically designed for generating unique IDs called "UUID" (for "universally unique ID"). We can use it like this: // DAVID: It feels like a strange time to introduce it, seeing as we've already used it in the spike earlier. [role="sourcecode"] .src/accounts/models.py (ch19l040) ==== [source,python] ---- import uuid [...] class Token(models.Model): email = models.EmailField() uid = models.CharField(default=uuid.uuid4, max_length=40) # <1> ---- ==== <1> The `default=` argument for a field can be either a static value or a callable that returns a value at the time the model is created. In our case, using a callable means we'll get a different unique ID for every model. [role="pagebreak-before"] And, perhaps with a bit more wrangling of `makemigrations`... [subs="specialcharacters,macros"] ---- $ pass:quotes[*rm src/accounts/migrations/0002_token.py*] $ pass:quotes[*python src/manage.py makemigrations*] Migrations for 'accounts': src/accounts/migrations/0002_token.py + Create model Token ---- ...that should get us to passing tests: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test accounts* [...] Ran 4 tests in 0.015s OK ---- So, we are well on our way! [role="scratchpad"] ***** * _[strikethrough line-through]#Token model with email and UID#_ * _View to create token and send login email incl. url w/ token UID_ * _[strikethrough line-through]#Custom user model with USERNAME_FIELD=email#_ * _Authentication backend with authenticate() and get_user() functions_ * _Registering auth backend in settings.py_ * _Login view calls authenticate() and login() from django.contrib.auth_ * _Logout view calls django.contrib.auth.logout_ ***** The models layer is done, at least. In the next chapter, we'll get into mocking—a key technique for testing external dependencies like email.((("tokens", "token model linking emails and UID", startref="ix_tknmod")))((("emails", "token model linking with unique ID", startref="ix_emltkn")))((("", startref="SDtoken18"))) [role="pagebreak-before"] .Exploratory Coding, Spiking, and De-spiking ******************************************************************************* Spiking:: Spiking is exploratory coding to find out about a new API, or to explore the feasibility of a new solution. Spiking can be done without tests. It's a good idea to do your spike on a new branch, and go back to your main branch when de-spiking. ((("spiking and de-spiking", "defined"))) De-spiking:: De-spiking means taking the work from a spike and making it part of the production codebase. The idea is to throw away the old spike code altogether, and start again from scratch, using TDD once again. De-spiked code can often come out looking quite different from the original spike, and usually much nicer. Writing your FT against spiked code:: Whether or not this is a good idea depends on your circumstances. The reason it can be useful is because it can help you write the FT correctly--figuring out how to test your spike can be just as challenging as the spike itself. On the other hand, it might constrain you to reimplementing a solution very similar to your spiked one; something to watch out for. ((("functional tests (FTs)", "spiked code and"))) ((("", startref="AuthSpike18"))) ******************************************************************************* ================================================ FILE: chapter_20_mocking_1.asciidoc ================================================ [[chapter_20_mocking_1]] == Using Mocks to Test External Dependencies ((("Django framework", "sending emails"))) ((("emails", "sending from Django"))) ((("mail.outbox attribute"))) In this chapter, we'll start testing the parts of our code that send emails—i.e., the second item on our scratchpad: [role="scratchpad"] ***** * _[strikethrough line-through]#Token model with email and UID#_ * _View to create token and send login email incl. url w/ token UID_ * _[strikethrough line-through]#Custom user model with USERNAME_FIELD=email#_ * _Authentication backend with authenticate() and get_user() functions_ * _Registering auth backend in settings.py_ * _Login view calls authenticate() and login() from django.contrib.auth_ * _Logout view calls django.contrib.auth.logout_ ***** In the functional test (FT), you saw that Django gives us a way of retrieving any emails it sends by using the `mail.outbox` attribute. But in this chapter, I want to demonstrate a widespread testing technique called _mocking_. So, for the purpose of these unit tests, we'll pretend that this nice Django shortcut doesn't exist. ((("mocks", "benefits and drawbacks of"))) Am I telling you _not_ to use Django's `mail.outbox`? No—use it; it's a neat helper. But I want to teach you about mocks because they're a useful general-purpose tool for unit testing external dependencies. You may not always be using Django! And even if you are, you may not be sending email--any interaction with a third-party API is a place you might find yourself wanting to test with mocks. ((("mocks", "deciding whether to use")))((("external dependencies"))) .To Mock or Not to Mock? ******************************************************************************* I once gave a talk called https://oreil.ly/XJPbT["Stop Using Mocks"]; it's entirely possible to find ways to write tests for external dependencies without using mocks at all. I'm covering mocking in this book because it's such a common technique, but it does come with some downsides, as we'll see. Other techniques—including dependency injection and the use of custom fake objects—are well worth exploring, but they're more advanced. My second book https://www.cosmicpython.com[_Architecture Patterns with Python_] goes into some detail on these alternatives. ******************************************************************************* === Before We Start: Getting the Basic Plumbing In ((("mocks", "preparing for"))) Let's just get a basic view and URL set up first. We can do so with a simple test to ensure that our new URL for sending the login email should eventually redirect back to the home page: [role="sourcecode dofirst-ch20l002"] .src/accounts/tests/test_views.py (ch20l001) ==== [source,python] ---- from django.test import TestCase class SendLoginEmailViewTest(TestCase): def test_redirects_to_home_page(self): response = self.client.post( "/accounts/send_login_email", data={"email": "edith@example.com"} ) self.assertRedirects(response, "/") ---- ==== [role="pagebreak-before"] Wire up the `include` in _superlists/urls.py_, plus the `url` in _accounts/urls.py_, and get the test passing with something a bit like this: [role="sourcecode dofirst-ch20l003"] .src/accounts/views.py (ch20l004) ==== [source,python] ---- from django.core.mail import send_mail # <1> from django.shortcuts import redirect def send_login_email(request): return redirect("/") ---- ==== <1> I've added the import of the `send_mail` function as a placeholder for now. If you've got the plumbing right, the tests should pass at this point: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test accounts* [...] Ran 5 tests in 0.015s OK ---- OK, now we have a starting point—so let's get mocking! === Mocking Manually—aka Monkeypatching ((("mocks", "manual", id="Mmanual19"))) ((("monkeypatching", id="monkey19")))((("send_mail function", "mocking", id="ix_sndml"))) When we call `send_mail` in real life, we expect Django to be making a connection to our email provider, and sending an actual email across the public internet. That's not something we want to happen in our tests. It's a similar problem whenever you have code that has external side effects—calling an API, sending out an SMS, integrating with a payment provider, whatever it may be. When running our unit tests, we don't want to be sending out real payments or making API calls across the internet. But we would still like a way of testing that our code is correct. Mocksfootnote:[I'm using the generic term "mock", but testing enthusiasts like to distinguish other types of a general class of test tools called "test doubles", including spies, fakes, and stubs. The differences don't really matter for this book, but if you want to get into the nitty-gritty, check out the https://github.com/testdouble/contributing-tests/wiki/Test-Double[amazing wiki by Justin Searls]. Warning: absolutely chock full of great testing content.] give us one way to do that. Actually, one of the great things about Python is that its dynamic nature makes it very easy to do things like mocking—or what's sometimes called https://oreil.ly/vXHWY[monkeypatching]. Let's suppose that, as a first step, we want to get to some code that invokes `send_mail` with the right subject line, "from" address, and "to" address. That would look something like this: [role="sourcecode skipme"] .src/accounts/views.py (expected future contents) ==== [source,python] ---- def send_login_email(request): email = request.POST["email"] # expected future code: send_mail( "Your login link for Superlists", "some kind of body text tbc", "noreply@superlists", [email], ) return redirect("/") ---- ==== How can we test this without calling the _real_ `send_mail` function? The answer is that our test can ask Python to swap out the `send_mail` function for a fake version, at runtime, just before we invoke the `send_login_email` view. Check this out: [role="sourcecode"] .src/accounts/tests/test_views.py (ch20l005) ==== [source,python] ---- from django.test import TestCase import accounts.views # <2> class SendLoginEmailViewTest(TestCase): def test_redirects_to_home_page(self): [...] def test_sends_mail_to_address_from_post(self): self.send_mail_called = False def fake_send_mail(subject, body, from_email, to_list): # <1> self.send_mail_called = True self.subject = subject self.body = body self.from_email = from_email self.to_list = to_list accounts.views.send_mail = fake_send_mail # <2> self.client.post( "/accounts/send_login_email", data={"email": "edith@example.com"} ) self.assertTrue(self.send_mail_called) self.assertEqual(self.subject, "Your login link for Superlists") self.assertEqual(self.from_email, "noreply@superlists") self.assertEqual(self.to_list, ["edith@example.com"]) ---- ==== <1> We define a `fake_send_mail` function, which looks like the real `send_mail` function, but all it does is save some information about how it was called, using some variables on `self`. <2> Then, before we execute the code under test by doing the `self.client.post`, we swap out the real `accounts.views.send_mail` with our fake version—it's as simple as just assigning it. // DAVID: Might be better to get everything else working, and the test passing, without send_mail at all. // Then we introduce it, run the test and see it fail because it has some dependencies? Then we can just concentrate on // the mock bit. It's important to realise that there isn't really anything magical going on here; we're just taking advantage of Python's dynamic nature and scoping rules. Up until we actually invoke a function, we can modify the variables it has access to, as long as we get into the right namespace. That's why we import the top-level `accounts` module: to be able to get down to the `accounts.views` module, which is the scope in which the `accounts.views.send_login_email` function will run. This isn't even something that only works inside unit tests—you can do this kind of monkeypatching in any Python code! That may take a little time to sink in. See if you can convince yourself that it's not all totally crazy—and then consider a couple of extra details that are worth knowing: * Why do we use `self` as a way of passing information around? It's just a convenient variable that's available both inside the scope of the `fake_send_mail` function and outside of it. We could use any mutable object, like a list or a dictionary, as long as we are making in-place changes to an existing variable that exists outside our fake function.((("self variable"))) (Feel free to have a play around with different ways of doing this, if you're curious, and see what works and doesn't.) * The "before" is critical! I can't tell you how many times I've sat there, wondering why a mock isn't working, only to realise that I didn't mock _before_ I called the code under test. Let's see if our hand-rolled mock object will let us test-drive some code: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test accounts* [...] self.assertTrue(self.send_mail_called) AssertionError: False is not true ---- [role="pagebreak-before"] So let's call `send_mail`, naively: [role="sourcecode"] .src/accounts/views.py (ch20l006-1) ==== [source,python] ---- from django.core.mail import send_mail # <1> [...] def send_login_email(request): send_mail() # <2> return redirect("/") ---- ==== <1> This import should still be in the file from earlier, but in case an overenthusiastic IDE has removed it, I'm re-listing it for you here. <2> Here's our new call to `send_mail()`. That gives: [subs="specialcharacters,macros"] ---- TypeError: SendLoginEmailViewTest.test_sends_mail_to_address_from_post.<locals> .fake_send_mail() missing 4 required positional arguments: 'subject', 'body', 'from_email', and 'to_list' ---- It looks like our monkeypatch is working! We've called `send_mail`, and it's gone into our `fake_send_mail` function, which wants more arguments. Let's try this: [role="sourcecode"] .src/accounts/views.py (ch20l006-2) ==== [source,python] ---- def send_login_email(request): send_mail("subject", "body", "from_email", ["to email"]) return redirect("/") ---- ==== That gives: ---- self.assertEqual(self.subject, "Your login link for Superlists") AssertionError: 'subject' != 'Your login link for Superlists' ---- That's working pretty well! Now we can work step-by-step, all the way through to something like this: [role="sourcecode"] .src/accounts/views.py (ch20l006) ==== [source,python] ---- def send_login_email(request): email = request.POST["email"] send_mail( "Your login link for Superlists", "body text tbc", "noreply@superlists", [email], ) return redirect("/") ---- ==== And we have passing tests! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] Ran 6 tests in 0.016s OK ---- Brilliant! We've managed to write tests for some code, which would ordinarily go out and try to send real emails across the internet, and by "mocking out" the `send_email` function, we're able to write the tests and code all the same.footnote:[Again, we're acting as if Django's `mail.outbox` didn't exist, for the sake of learning. After all, what if you were using Flask? Or what if this was an API call, not an email?] But our hand-rolled mock has a couple of problems: * It involved a fair bit of boilerplate code, populating all those `self.xyz` variables to let us assert on them. * More importantly, although we didn't see this, the monkeypatching will persist from one test to the next, breaking isolation between tests. This can cause serious confusion. ((("send_mail function", "mocking", startref="ix_sndml")))((("", startref="monkey19")))((("", startref="Mmanual19"))) // TODO: illustrate this explicitly === The Python Mock Library ((("mocks", "Python Mock library", id="Mpythong19"))) ((("Python 3", "Mock library", id="Pmock19"))) The `mock` package was added to the standard library as part of Python 3.3. It provides a magical object called a `Mock`; try this out in a Python shell: [role='skipme'] [source,python] ---- >>> from unittest.mock import Mock >>> m = Mock() >>> m.any_attribute <Mock name='mock.any_attribute' id='140716305179152'> >>> type(m.any_attribute) <class 'unittest.mock.Mock'> >>> m.any_method() <Mock name='mock.any_method()' id='140716331211856'> >>> m.foo() <Mock name='mock.foo()' id='140716331251600'> >>> m.called False >>> m.foo.called True >>> m.bar.return_value = 1 >>> m.bar(42, var='thing') 1 >>> m.bar.call_args call(42, var='thing') ---- [role="pagebreak-before"] A mock is a magical object for a few reasons: * It responds to any request for an attribute or method call with other mocks. * You can configure it in turn to return specific values when called. * It enables you to inspect what it was called with. Sounds like a useful thing to be able to use in our unit tests! ==== Using unittest.patch ((("unittest module", "mock module and")))((("patch function in unittest and mock modules"))) And as if that weren't enough, the `mock` module also provides a helper function called `patch`, which we can use to do the monkeypatching we did by hand earlier. I'll explain how it all works shortly, but let's see it in action first: [role="sourcecode"] .src/accounts/tests/test_views.py (ch20l007) ==== [source,python] ---- from unittest import mock from django.test import TestCase [...] class SendLoginEmailViewTest(TestCase): def test_redirects_to_home_page(self): [...] @mock.patch("accounts.views.send_mail") # <1> def test_sends_mail_to_address_from_post(self, mock_send_mail): # <2> self.client.post( "/accounts/send_login_email", data={"email": "edith@example.com"} ) self.assertEqual(mock_send_mail.called, True) (subject, body, from_email, to_list), kwargs = mock_send_mail.call_args self.assertEqual(subject, "Your login link for Superlists") self.assertEqual(from_email, "noreply@superlists") self.assertEqual(to_list, ["edith@example.com"]) ---- ==== <1> Here's the decorator--we'll go into detail about how it works shortly. <2> Here's the extra argument we add to the test method. Again, detailed explanation to come, but as you'll see, it's going to do most of the work that `fake_send_mail` was doing before. [role="pagebreak-before"] If you rerun the tests, you'll see they still pass. And because we're always suspicious of any test that still passes after a big change, let's deliberately break it just to see: [role="sourcecode"] .src/accounts/tests/test_views.py (ch20l008) ==== [source,python] ---- self.assertEqual(to_list, ["schmedith@example.com"]) ---- ==== And let's add a little debug print to our view as well, to see the effects of the `mock.patch`: [role="sourcecode"] .src/accounts/views.py (ch20l009) ==== [source,python] ---- def send_login_email(request): email = request.POST["email"] print(type(send_mail)) send_mail( [...] ---- ==== Let's run the tests again: [subs="macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...]pass:specialcharacters[ ....<class 'function'> .<class 'unittest.mock.MagicMock'> ][...]pass:[ AssertionError: Lists differ: ['edith@example.com'\] != ['schmedith@example.com'\] ][...] Ran 6 tests in 0.024s FAILED (failures=1) ---- Sure enough, the tests fail. And we can see, just before the failure message, that when we print the `type` of the `send_mail` function, in the first unit test it's a normal function, but in the second unit test we're seeing a mock object. Let's remove the deliberate mistake and dive into exactly what's going on: [role="sourcecode dofirst-ch20l010"] .src/accounts/tests/test_views.py (ch20l011) ==== [source,python] ---- @mock.patch("accounts.views.send_mail") # <1> def test_sends_mail_to_address_from_post(self, mock_send_mail): # <2> self.client.post( # <3> "/accounts/send_login_email", data={"email": "edith@example.com"} ) self.assertEqual(mock_send_mail.called, True) # <4> (subject, body, from_email, to_list), kwargs = mock_send_mail.call_args # <5> self.assertEqual(subject, "Your login link for Superlists") self.assertEqual(from_email, "noreply@superlists") self.assertEqual(to_list, ["edith@example.com"]) ---- ==== <1> The `mock.patch()` decorator takes a dot-notation name of an object to monkeypatch. That's the equivalent of manually replacing the `send_mail` in `accounts.views`. The advantage of the decorator is that, firstly, it automatically replaces the target with a mock. And secondly, it automatically puts the original object back at the end! (Otherwise, the object stays monkeypatched for the rest of the test run, which might cause problems in other tests.) <2> `patch` then injects the mocked object into the test as an argument to the test method. We can choose whatever name we want for it, but I usually use a convention of `mock_` plus the original name of the object. <3> We call our view under test as usual, but everything inside this test method has our mock applied to it, so the view won't call the real `send_mail` object; it'll be seeing `mock_send_mail` instead. <4> And we can now make assertions about what happened to that mock object during the test. We can see it was called... <5> ...and we can also unpack its various positional and keyword call arguments, to examine what it was called with. (See <<mock-call-args-sidebar>> in the next chapter for a longer explanation of `.call_args`.) All crystal clear? No? Don't worry; we'll do a couple more tests with mocks to see if they start to make more sense as we use them more. ==== Getting the FT a Little Further Along First let's get back to our FT and see where it's failing: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_login*] [...] AssertionError: 'Check your email' not found in 'Superlists\nEnter your email to log in\nStart a new To-Do list' ---- Submitting the email address currently has no effect. Hmmm. Currently our form is hardcoded to send to _/accounts/send_login_email_. Let's switch to using the `{% url %}` syntax just to make sure it's the right URL: [role="sourcecode small-code"] .src/lists/templates/base.html (ch20l012) ==== [source,html] ---- <form method="POST" action="{% url 'send_login_email' %}"> ---- ==== Does that help? Nope, same error. Why? Ah, nothing to do with the URL actually; it's because we're not displaying a success message after we send the user an email. Let's add a test for that. ==== Testing the Django Messages Framework ((("messages framework (Django), testing", id="ix_msgfrm")))((("Django framework", "messages framework, testing", id="ix_Djmsg"))) We'll use Django's "messages framework", which is often used to display ephemeral "success" or "warning" messages to show the results of an action, something like what's shown in <<success-message>>. [[success-message]] .A green success message image::images/tdd3_2001.png["Screenshot of success message saying check your email, as it will look at the end of the de-spike."] Have a look at the https://docs.djangoproject.com/en/5.2/ref/contrib/messages[Django messages docs] if you haven't come across it already. Testing Django messages is a bit contorted: [role="sourcecode"] .src/accounts/tests/test_views.py (ch20l013) ==== [source,python] ---- def test_adds_success_message(self): response = self.client.post( "/accounts/send_login_email", data={"email": "edith@example.com"}, follow=True, # <1> ) message = list(response.context["messages"])[0] # <2> self.assertEqual( message.message, "Check your email, we've sent you a link you can use to log in.", ) self.assertEqual(message.tags, "success") ---- ==== [role="pagebreak-before"] <1> We have to pass `follow=True` to the test client to tell it to get the page _after_ the 302-redirect. <2> Then we examine the response context for a `messages` iterable, which we have to listify before it'll play nicely. (We'll use these later in a template with `{% for message in messages %}`.) That gives: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] message = list(response.context["messages"])[0] IndexError: list index out of range ---- And we can get it passing with: [role="sourcecode"] .src/accounts/views.py (ch20l014) ==== [source,python] ---- from django.contrib import messages [...] def send_login_email(request): [...] messages.success( request, "Check your email, we've sent you a link you can use to log in.", ) return redirect("/") ---- ==== [role="pagebreak-before less_space"] [[mocks-tightly-coupled-sidebar]] .Mocks Can Leave You Tightly Coupled to the Implementation ******************************************************************************* TIP: This sidebar is an intermediate-level testing tip. If it goes over your head the first time around, come back and take another look when you've finished this chapter. I said testing messages is a bit contorted; it took me several goes to get it right. In fact, at a previous employer, we gave up on testing them like this and decided to just use mocks. Let's see what that would look like in this case: [role="sourcecode small-code"] .src/accounts/tests/test_views.py (ch20l014-2) ==== [source,python] ---- @mock.patch("accounts.views.messages") def test_adds_success_message_with_mocks(self, mock_messages): response = self.client.post( "/accounts/send_login_email", data={"email": "edith@example.com"} ) expected = "Check your email, we've sent you a link you can use to log in." self.assertEqual( mock_messages.success.call_args, mock.call(response.wsgi_request, expected), ) ---- ==== We mock out the `messages` module, and check that `messages.success` was called with the right arguments: the original request and the message we want. And you could get it passing by using the exact same code as earlier. Here's the problem though: the messages framework gives you more than one way to achieve the same result. I could write the code like this: [role="sourcecode"] .src/accounts/views.py (ch20l014-3) ==== [source,python] ---- messages.add_message( request, messages.SUCCESS, "Check your email, we've sent you a link you can use to log in.", ) ---- ==== And the original, non-mocky test would still pass. But our mocky test will fail, because we're no longer calling `messages.success`; we're calling `messages.add_message`. Even though the end result is the same and our code is "correct", the test is broken.((("mocks", "use of, tight coupling with implementation"))) This is what it means to say that using mocks leave you "tightly coupled with the implementation". We usually say it's better to test behaviour, not implementation details; test what happens, not how you do it. Mocks often end up erring too much on the side of the "how" rather than the "what". TIP: Test should be about behaviour, not implementation. If your tests tie you to specific implementation details, they will prevent you from refactoring as freely. ******************************************************************************* ==== Adding Messages to Our HTML What happens next in the functional test?((("Django framework", "messages framework, testing", startref="ix_Djmsg")))((("messages framework (Django), testing", startref="ix_msgfrm")))((("messages", "adding to HTML for page"))) Ah. Still nothing. We need to actually add the messages to the page. Something like this: [role="sourcecode dofirst-ch20l014-4"] .src/lists/templates/base.html (ch20l015) ==== [source,html] ---- [...] </nav> {% if messages %} <div class="row"> <div class="col-md-12"> {% for message in messages %} {% if message.level_tag == 'success' %} <div class="alert alert-success">{{ message }}</div> {% else %} <div class="alert alert-warning">{{ message }}</div> {% endif %} {% endfor %} </div> </div> {% endif %} ---- ==== // TODO: feed thru change Now do we get a little further? Yes! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] Ran 7 tests in 0.023s OK $ pass:quotes[*python src/manage.py test functional_tests.test_login*] [...] AssertionError: 'Use this link to log in' not found in 'body text tbc' ---- We need to fill out the body text of the email, with a link that the user can use to log in. Let's just cheat for now though, by changing the value in the view: [role="sourcecode"] .src/accounts/views.py (ch20l016) ==== [source,python] ---- send_mail( "Your login link for Superlists", "Use this link to log in", "noreply@superlists", [email], ) ---- ==== That gets the FT a little further: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_login*] [...] AssertionError: Could not find url in email body: Use this link to log in ---- OK, I think we can call the `send_login_email` view done for now: [role="scratchpad"] ***** * _[strikethrough line-through]#Token model with email and UID#_ * _[strikethrough line-through]#View to create token and send login email incl. url w/ token UID#_ * _[strikethrough line-through]#Custom user model with USERNAME_FIELD=email#_ * _Authentication backend with authenticate() and get_user() functions_ * _Registering auth backend in settings.py_ * _Login view calls authenticate() and login() from django.contrib.auth_ * _Logout view calls django.contrib.auth.logout_ ***** ==== Starting on the Login URL We're going to have to build some kind of URL!((("URLs", "starting login URL")))((("tokens", "passing in GET pararameter to login URL"))) Let's build the minimal thing, just a placeholder really: [role="sourcecode"] .src/accounts/tests/test_views.py (ch20l017) ==== [source,python] ---- class LoginViewTest(TestCase): def test_redirects_to_home_page(self): response = self.client.get("/accounts/login?token=abcd123") self.assertRedirects(response, "/") ---- ==== We're imagining we'll pass the token in as a GET parameter, after the `?`. It doesn't need to do anything for now. I'm sure you can find your way through to getting the boilerplate in for a basic URL and view, via errors like these: [role="simplelist"] * No URL: + [role="small-code"] ---- AssertionError: 404 != 302 : Response didn't redirect as expected: Response code was 404 (expected 302) ---- * No view: + [role="dofirst-ch20l018 small-code"] ---- AttributeError: module 'accounts.views' has no attribute 'login' ---- * Broken view: + [role="dofirst-ch20l019 small-code"] ---- ValueError: The view accounts.views.login didn't return an HttpResponse object. It returned None instead. ---- * OK! + [role="dofirst-ch20l020 small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] Ran 8 tests in 0.029s OK ---- And now we can give people a link to use. It still won't do much though, because we still don't have a token to give to the user. ==== Checking That We Send the User a Link with a Token Back in our `send_login_email` view, we've tested the email subject, and the "from", and "to" fields. The body is the part that will have to include a token or URL they can use to log in.((("emails", "checking sending of link with a token")))((("emails", "checking sending of link with token"))) Let's spec out two tests for that: [role="sourcecode"] .src/accounts/tests/test_views.py (ch20l021) ==== [source,python] ---- from accounts.models import Token [...] class SendLoginEmailViewTest(TestCase): def test_redirects_to_home_page(self): [...] def test_adds_success_message(self): [...] @mock.patch("accounts.views.send_mail") def test_sends_mail_to_address_from_post(self, mock_send_mail): [...] def test_creates_token_associated_with_email(self): # <1> self.client.post( "/accounts/send_login_email", data={"email": "edith@example.com"} ) token = Token.objects.get() self.assertEqual(token.email, "edith@example.com") @mock.patch("accounts.views.send_mail") def test_sends_link_to_login_using_token_uid(self, mock_send_mail): # <2> self.client.post( "/accounts/send_login_email", data={"email": "edith@example.com"} ) token = Token.objects.get() expected_url = f"http://testserver/accounts/login?token={token.uid}" (subject, body, from_email, to_list), kwargs = mock_send_mail.call_args self.assertIn(expected_url, body) ---- ==== <1> The first test is fairly straightforward; it checks that the token we create in the database is associated with the email address from the POST request. <2> The second one is our second test using mocks. We mock out the `send_mail` function again using the `patch` decorator, but this time we're interested in the `body` argument from the call arguments. Running them now will fail because we're not creating any kind of token: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] accounts.models.Token.DoesNotExist: Token matching query does not exist. [...] accounts.models.Token.DoesNotExist: Token matching query does not exist. ---- We can get the first one to pass by creating a token: [role="sourcecode"] .src/accounts/views.py (ch20l022) ==== [source,python] ---- from accounts.models import Token [...] def send_login_email(request): email = request.POST["email"] token = Token.objects.create(email=email) send_mail( [...] ---- ==== And now the second test prompts us to actually use the token in the body of our email: [subs=""] ---- [...] AssertionError: 'http://testserver/accounts/login?token=[...] not found in 'Use this link to log in' FAILED (failures=1) ---- So, we can insert the token into our email like this: [role="sourcecode"] .src/accounts/views.py (ch20l023) ==== [source,python] ---- from django.urls import reverse [...] def send_login_email(request): email = request.POST["email"] token = Token.objects.create(email=email) url = request.build_absolute_uri( # <1> reverse("login") + "?token=" + str(token.uid), ) message_body = f"Use this link to log in:\n\n{url}" send_mail( "Your login link for Superlists", message_body, "noreply@superlists", [email], ) [...] ---- ==== <1> `request.build_absolute_uri` deserves a mention--it's one way to build a "full" URL, including the domain name and the HTTP(S) part, in Django. There are other ways, but they usually involve getting into the "sites" framework, which gets complicated pretty quickly. You can find lots more discussion on this if you're curious by doing a bit of googling. // IDEA: investigate kwargs for reverse() call // reverse("login", token=str(token.uid)) And the tests pass: ---- OK ---- I think _that's_ our `send_login_email` view done: [role="scratchpad"] ***** * _[strikethrough line-through]#Token model with email and UID#_ * _[strikethrough line-through]#_View to create token and send login email incl. url w/ token UID#_ * _[strikethrough line-through]#Custom user model with USERNAME_FIELD=email#_ * _Authentication backend with authenticate() and get_user() functions_ * _Registering auth backend in settings.py_ * _Login view calls authenticate() and login() from django.contrib.auth_ * _Logout view calls django.contrib.auth.logout_ ***** The next piece in the puzzle is the authentication backend, whose job it will be to examine tokens for validity and then return the corresponding users. Then, we need to get our login view to actually log users in, if they can authenticate. ((("", startref="Mpythong19")))((("", startref="Pmock19"))) === De-spiking Our Custom Authentication Backend ((("mocks", "de-spiking custom authentication", id="ix_mckdespCA"))) ((("spiking and de-spiking", "de-spiking custom authentication", id="ix_spkdesCA"))) Here's how our authentication backend looked in the spike: [[spike-reminder]] [role="skipme small-code"] [source,python] ---- class PasswordlessAuthenticationBackend(BaseBackend): def authenticate(self, request, uid): print("uid", uid, file=sys.stderr) if not Token.objects.filter(uid=uid).exists(): print("no token found", file=sys.stderr) return None token = Token.objects.get(uid=uid) print("got token", file=sys.stderr) try: user = ListUser.objects.get(email=token.email) print("got user", file=sys.stderr) return user except ListUser.DoesNotExist: print("new user", file=sys.stderr) return ListUser.objects.create(email=token.email) def get_user(self, email): return ListUser.objects.get(email=email) ---- [role="pagebreak-before"] Decoding this: * We take a UID and check if it exists in the database. * We return `None` if it doesn't. * If it does exist, we extract an email address, and either find an existing user with that address or create a new one. // CSANAD: shouldn't we use the numbered annotations instead? ==== One if = One More Test A rule of thumb for these sorts of tests: any `if` means an extra test, and any `try/except` means an extra test. So, this should be about three tests. How about something like this? [role="sourcecode"] .src/accounts/tests/test_authentication.py (ch20l024) ==== [source,python] ---- from django.http import HttpRequest from django.test import TestCase from accounts.authentication import PasswordlessAuthenticationBackend from accounts.models import Token, User class AuthenticateTest(TestCase): def test_returns_None_if_no_such_token(self): result = PasswordlessAuthenticationBackend().authenticate( HttpRequest(), "no-such-token" ) self.assertIsNone(result) def test_returns_new_user_with_correct_email_if_token_exists(self): email = "edith@example.com" token = Token.objects.create(email=email) user = PasswordlessAuthenticationBackend().authenticate( HttpRequest(), token.uid ) new_user = User.objects.get(email=email) self.assertEqual(user, new_user) def test_returns_existing_user_with_correct_email_if_token_exists(self): email = "edith@example.com" existing_user = User.objects.create(email=email) token = Token.objects.create(email=email) user = PasswordlessAuthenticationBackend().authenticate( HttpRequest(), token.uid ) self.assertEqual(user, existing_user) ---- ==== [role="pagebreak-before"] In _authenticate.py_, we'll just have a little placeholder: [role="sourcecode"] .src/accounts/authentication.py (ch20l025) ==== [source,python] ---- class PasswordlessAuthenticationBackend: def authenticate(self, request, uid): pass ---- ==== How do we get on? [subs="macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] .FE.......... ====================================================================== ERROR: test_returns_new_user_with_correct_email_if_token_exists (accounts.tests .test_authentication.AuthenticateTest.test_returns_new_user_with_correct_email_ if_token_exists) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/accounts/tests/test_authentication.py", line 21, in test_returns_new_user_with_correct_email_if_token_exists new_user = User.objects.get(email=email) [...] accounts.models.User.DoesNotExist: User matching query does not exist. ====================================================================== FAIL: test_returns_existing_user_with_correct_email_if_token_exists (accounts.t ests.test_authentication.AuthenticateTest.test_returns_existing_user_with_corre ct_email_if_token_exists) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/accounts/tests/test_authentication.py", line 31, in test_returns_existing_user_with_correct_email_if_token_exists self.assertEqual(user, existing_user) ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^ AssertionError: None != pass:specialcharacters[<User: User object (edith@example.com)>] --------------------------------------------------------------------- Ran 13 tests in 0.038s FAILED (failures=1, errors=1) ---- //TODO: do we need that inline pass:specialcharacters? [role="pagebreak-before"] Here's a first cut: [role="sourcecode"] .src/accounts/authentication.py (ch20l026) ==== [source,python] ---- from accounts.models import Token, User class PasswordlessAuthenticationBackend: def authenticate(self, request, uid): token = Token.objects.get(uid=uid) return User.objects.get(email=token.email) ---- ==== Now, instead of one `FAIL` and one `ERROR`, we get two ++ERROR++s: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] ERROR: test_returns_None_if_no_such_token (accounts.tests.test_authentication.A uthenticateTest.test_returns_None_if_no_such_token) [...] accounts.models.Token.DoesNotExist: Token matching query does not exist. ERROR: test_returns_new_user_with_correct_email_if_token_exists (accounts.tests .test_authentication.AuthenticateTest.test_returns_new_user_with_correct_email_ if_token_exists) [...] accounts.models.User.DoesNotExist: User matching query does not exist. ---- Notice that our third test, +test_returns_existing_user_with_c⁠o⁠r⁠r⁠e⁠c⁠t⁠_⁠e⁠m⁠a⁠i⁠l⁠_​i⁠f⁠_⁠t⁠o⁠k⁠e⁠n⁠_exists+, is actually passing. Our code _does_ currently handle the "happy path", where both the token and the user already exist in the database. Let's fix each of the remaining ones in turn. Notice how the test names are telling us exactly what we need to do. First, `test_returns_None_if_no_such_token`, which is telling us what to do if the token doesn't exist: [role="sourcecode"] .src/accounts/authentication.py (ch20l027) ==== [source,python] ---- def authenticate(self, request, uid): try: token = Token.objects.get(uid=uid) return User.objects.get(email=token.email) except Token.DoesNotExist: return None ---- ==== That gets us down to one failure: [subs="specialcharacters,macros"] ---- ERROR: test_returns_new_user_with_correct_email_if_token_exists (accounts.tests .test_authentication.AuthenticateTest.test_returns_new_user_with_correct_email_ if_token_exists) [...] accounts.models.User.DoesNotExist: User matching query does not exist. FAILED (errors=1) ---- OK, so we need to return a `new_user_with_correct_email` `if_token_exists`? We can do that! [role="sourcecode"] .src/accounts/authentication.py (ch20l028) ==== [source,python] ---- def authenticate(self, request, uid): try: token = Token.objects.get(uid=uid) return User.objects.get(email=token.email) except User.DoesNotExist: return User.objects.create(email=token.email) except Token.DoesNotExist: return None ---- ==== That's turned out neater than our <<spike-reminder,spike>>! ==== The get_user Method ((("get_user method"))) We've handled the `authenticate` function, which Django will use to log new users in. The second part of the protocol we have to implement is the `get_user` method, whose job is to retrieve a user based on their unique identifier (the email address), or to return `None` if it can't find one. (Have another look at <<spike-reminder,the spiked code>> if you need a reminder.) Here are a couple of tests for those two requirements: [role="sourcecode small-code"] .src/accounts/tests/test_authentication.py (ch20l030) ==== [source,python] ---- class GetUserTest(TestCase): def test_gets_user_by_email(self): User.objects.create(email="another@example.com") desired_user = User.objects.create(email="edith@example.com") found_user = PasswordlessAuthenticationBackend().get_user("edith@example.com") self.assertEqual(found_user, desired_user) def test_returns_None_if_no_user_with_that_email(self): self.assertIsNone( PasswordlessAuthenticationBackend().get_user("edith@example.com") ) ---- ==== And our first failure: ---- AttributeError: 'PasswordlessAuthenticationBackend' object has no attribute 'get_user' ---- [role="pagebreak-before"] Let's create a placeholder one then: [role="sourcecode"] .src/accounts/authentication.py (ch20l031) ==== [source,python] ---- class PasswordlessAuthenticationBackend: def authenticate(self, request, uid): [...] def get_user(self, email): pass ---- ==== Now we get: [subs="macros"] ---- self.assertEqual(found_user, desired_user) AssertionError: None != pass:specialcharacters[<User: User object (edith@example.com)>] ---- And (step by step, just to see if our test fails the way we think it will): [role="sourcecode"] .src/accounts/authentication.py (ch20l033) ==== [source,python] ---- def get_user(self, email): return User.objects.first() ---- ==== That gets us past the first assertion, and onto: [subs="macros"] ---- self.assertEqual(found_user, desired_user) AssertionError: pass:specialcharacters[<User: User object (another@example.com)>] != pass:specialcharacters[<User: User object (edith@example.com)>] ---- And so, we call `get` with the email as an argument: [role="sourcecode"] .src/accounts/authentication.py (ch20l034) ==== [source,python] ---- def get_user(self, email): return User.objects.get(email=email) ---- ==== Now our test for the `None` case fails: ---- ERROR: test_returns_None_if_no_user_with_that_email (accounts.tests.test_authen tication.GetUserTest.test_returns_None_if_no_user_with_that_email) [...] accounts.models.User.DoesNotExist: User matching query does not exist. ---- That prompts us to finish the method like this: [role="sourcecode"] .src/accounts/authentication.py (ch20l035) ==== [source,python] ---- def get_user(self, email): try: return User.objects.get(email=email) except User.DoesNotExist: return None # <1> ---- ==== <1> You could just use `pass` here, and the function would return `None` by default. However, because we specifically need the function to return `None`, the "explicit is better than implicit" rule applies here. That gets us to passing tests: ---- OK ---- And we have a working authentication backend! [role="scratchpad"] ***** * _[strikethrough line-through]#Token model with email and UID#_ * _[strikethrough line-through]#_View to create token and send login email incl. url w/ token UID#_ * _[strikethrough line-through]#Custom user model with USERNAME_FIELD=email#_ * _[strikethrough line-through]#Authentication backend with authenticate() and get_user() functions#_ * _Registering auth backend in settings.py_ * _Login view calls authenticate() and login() from django.contrib.auth_ * _Logout view calls django.contrib.auth.logout_ ***** Let's call that a win and, in the next chapter, we'll work on integrating it into our login view and getting our FT passing.((("spiking and de-spiking", "de-spiking custom authentication", startref="ix_spkdesCA")))((("mocks", "de-spiking custom authentication", startref="ix_mckdespCA"))) [role="pagebreak-before less_space"] [[mocking-py-sidebar-1]] .On Mocking in Python ******************************************************************************* Mocking and external dependencies:: One place to consider using mocking is when we have an external dependency that we don't want to actually use in our tests. A mock can be used to simulate the third-party API. Whilst it is possible to "roll your own" mocks in Python, a mocking framework like the +unittest.mock+ module provides a lot of helpful shortcuts that will make it easier to write (and more importantly, read) your tests. ((("external dependencies"))) The mock library:: The `unittest.mock` module from Python's standard library contains most everything you might need for monkeypatching and mocking in Python.footnote:[This library was originally written as a standalone package by Michael Foord while he was working at the company that later spawned PythonAnywhere, a few years before I joined. It became part of the standard library in Python 3.3. Michael was a friend, and sadly passed away in 2025.] ((("mocks", "Python Mock library"))) ((("Python 3", "Mock library"))) Monkeypatching:: This is the process of replacing an object in a namespace at runtime. We use it in our unit tests to replace a real function that has undesirable side effects with a mock object, using the `mock.patch` decorator. ((("monkeypatching"))) The mock.patch decorator:: `unittest.mock` ((("patch decorator")))provides a function called `patch`, which can be used to "mock out" (monkeypatch) any object from the module you're testing. It's commonly used as a decorator on a test method. Importantly, it "undoes" the mocking at the end of the test for you, to avoid contamination between tests. Mocks can leave you tightly coupled to the implementation:: As discussed in the earlier sidebar, mocks can leave you tightly coupled to your implementation. For that reason, you shouldn't use them unless you have a good reason. ******************************************************************************* ================================================ FILE: chapter_21_mocking_2.asciidoc ================================================ [[chapter_21_mocking_2]] == Using Mocks for Test Isolation In this chapter, we'll finish up our login system. While doing so, we'll explore an alternative use of mocks: to isolate parts of the system from each other. This enables more targeted testing, fights combinatorial explosion, and reduces duplication between tests. NOTE: In this chapter, we start to drift towards what's called "London-school TDD", which is a variant on the "Classical" or "Detroit" style of TDD that I mostly show in the book. We won't get into the details here, but London-school TDD places more emphasis on mocking and isolating parts of the system. As always, there are pros and cons! Read more at https://www.obeythetestinggoat.com/book/appendix_purist_unit_tests.html[Online Appendix: Test Isolation and "Listening to Your Tests"]. Along the way, we'll learn a few more useful features of `unittest.mock`, and we'll also have a discussion about how many tests are "enough". === Using Our Auth Backend in the Login View [role="scratchpad"] ***** * _[strikethrough line-through]#Token model with email and UID#_ * _[strikethrough line-through]#View to create token and send login email incl. url w/ token UID#_ * _[strikethrough line-through]#Custom user model with USERNAME_FIELD=email#_ * _[strikethrough line-through]#Authentication backend with authenticate() and get_user() functions#_ * _Registering auth backend in settings.py_ * _Login view calls authenticate() and login() from django.contrib.auth_ * _Logout view calls django.contrib.auth.logout_ ***** We got our auth backend ready in the last chapter; now we need use the backend in our login view. But first, as our scratchpad says, we need to add it to _settings.py_: [role="sourcecode"] .src/superlists/settings.py (ch21l001) ==== [source,python] ---- AUTH_USER_MODEL = "accounts.User" AUTHENTICATION_BACKENDS = [ "accounts.authentication.PasswordlessAuthenticationBackend", ] [...] ---- ==== That was easy! [role="scratchpad"] ***** * _[strikethrough line-through]#Token model with email and UID#_ * _[strikethrough line-through]#View to create token and send login email incl. url w/ token UID#_ * _[strikethrough line-through]#Custom user model with USERNAME_FIELD=email#_ * _[strikethrough line-through]#Authentication backend with authenticate() and get_user() functions#_ * _[strikethrough line-through]#Registering auth backend in settings.py#_ * _Login view calls authenticate() and login() from django.contrib.auth_ * _Logout view calls django.contrib.auth.logout_ ***** Next, let's write some tests for what should happen in our view. Looking back at the spike again: [role="sourcecode skipme"] .src/accounts/views.py ==== [source,python] ---- def login(request): print("login view", file=sys.stderr) uid = request.GET.get("uid") user = auth.authenticate(uid=uid) if user is not None: auth.login(request, user) return redirect("/") ---- ==== TIP: You can view the contents of files from the spike using, for example, `git show passwordless-spike:src/accounts/views.py`. We call `django.contrib.auth.authenticate` and then, if it returns a user, we call `django.contrib.auth.login`. TIP: This is a good time to check out the https://docs.djangoproject.com/en/5.2/topics/auth/default/#how-to-log-a-user-in[Django docs on authentication] for a little more context. ((("Django framework", "documentation"))) ==== Straightforward Non-Mocky Test for Our View Here's the most obvious test we might want to write, thinking in terms of the _behaviour_ we want: * If someone has a valid token, they should get logged in. * If someone tries to use an invalid token (or does not have one), it should not log them in. Here's how we might add the happy-path test for the user with the valid token: [role="sourcecode"] .src/accounts/tests/test_views.py (ch21l002) ==== [source,python] ---- from django.contrib import auth [...] class LoginViewTest(TestCase): def test_redirects_to_home_page(self): [...] def test_logs_in_if_given_valid_token(self): anon_user = auth.get_user(self.client) # <1> self.assertEqual(anon_user.is_authenticated, False) # <2> token = Token.objects.create(email="edith@example.com") self.client.get(f"/accounts/login?token={token.uid}") user = auth.get_user(self.client) self.assertEqual(user.is_authenticated, True) # <3> self.assertEqual(user.email, "edith@example.com") # <3> ---- ==== <1> We use Django's `auth.get_user()` to extract the current user from the test client. <2> We verify we're not logged in before we start. (This isn't strictly necessary, but it's always nice to know you're on firm ground.) <3> And here's where we check that we've been logged in, with a user with the right email address. And that will fail as expected: ---- self.assertEqual(user.is_authenticated, True) AssertionError: False != True ---- [role="pagebreak-before"] We can get it to pass by "cheating", like this: [role="sourcecode small-code"] .src/accounts/views.py (ch21l003) ==== [source,python] ---- from django.contrib import auth, messages [...] from accounts.models import Token, User def send_login_email(request): [...] def login(request): user = User.objects.create(email="edith@example.com") auth.login(request, user) return redirect("/") ---- ==== ... ---- OK ---- That forces us to write another test: [role="sourcecode"] .src/accounts/tests/test_views.py (ch21l004) ==== [source,python] ---- def test_shows_login_error_if_token_invalid(self): response = self.client.get("/accounts/login?token=invalid-token", follow=True) user = auth.get_user(self.client) self.assertEqual(user.is_authenticated, False) message = list(response.context["messages"])[0] self.assertEqual( message.message, "Invalid login link, please request a new one", ) self.assertEqual(message.tags, "error") ---- ==== And now we get that passing by using the most straightforward implementation... [role="sourcecode small-code"] .src/accounts/views.py (ch21l005) ==== [source,python] ---- def login(request): if Token.objects.filter(uid=request.GET["token"]).exists(): # <1> user = User.objects.create(email="edith@example.com") # <2> <3> auth.login(request, user) else: messages.error(request, "Invalid login link, please request a new one") # <4> return redirect("/") ---- ==== <1> Oh wait; we forgot about our authentication backend and just did the query directly from the token model! Well that's arguably more straightforward, but how do we force ourselves to write the code the way we want to—i.e., using Django's authentication API? <2> Oh dear, and the email address is still hardcoded. We might have to think about writing an extra test to force ourselves to fix that. <3> Oh--also, we're hardcoding the creation of a user every time, but actually, we want to have the get-or-create logic that we implemented in our backend. <4> This bit is OK at least! Is this starting to feel a bit familiar? We've already written all the tests for the various permutations of our authentication logic, and we're considering writing equivalent tests at the views layer. === Combinatorial Explosion <<table-21-1>> recaps the tests we might want to write at each layer in our application.((("combinatorial explosion"))) [[table-21-1]] .What we want to test in each layer |======= |Views layer| Authentication backend | Models layer a| * Valid token means user is logged in * Invalid token means user is not logged in a| * Returns correct existing user for a valid token * Creates a new user for a new email address * Returns `none` for an invalid token a| * Token associates email and UID * User can be retrieved from token UID |======= We already have three tests in the models layer, and five in the authentication layer. We started off writing the tests in the views layer, where—_conceptually_—we only really want two test cases, and we're finding ourselves wondering if we need to write a whole bunch of tests that essentially duplicate the authentication layer tests. This is an example of the _combinatorial explosion_ problem. ==== The Car Factory Example Imagine we're testing a car factory: * First, we choose the car type: normal, station-wagon, or convertible. * Then, we choose the engine type: petrol, diesel, or electric. * Finally, we choose the colour: red, white, or hot pink. [role="pagebreak-before"] Here's how it might look in code: [role="skipme"] [source,python] ---- def build_car(car_type, engine_type, colour): engine = _create_engine(engine_type) naked_car = _assemble_car(engine, car_type) finished_car = _paint_car(naked_car, colour) return finished_car ---- How many tests do we need? Well, the upper bound to test every possible combination is 3 × 3 × 3 = 27 tests. That's a lot! How many tests do we _actually_ need to write? Well, it depends on how we're testing, how the different parts of the factory are integrated, and what we know about the system. Do we need to test every single colour? Maybe! Or, maybe, if we're happy that we can do two different colours, then we're happy that we can do any number—whether it's two, three, or hundreds. Perhaps we need two tests, maybe three. OK, but do we need to test that painting works for all the different engine types? Well, the painting process is probably independent of engine type: if we can paint a diesel in red, we can paint it in pink or white too. But, perhaps it _is_ affected by the car type: painting a convertible with a fabric roof might be a very different technological process to painting a hard-bodied car. So, we'd probably want to test that painting _in general_ works for each car type (three tests), but we don't need to test that painting works for every engine type. What we're analysing here is the level of "coupling" between the different parts of the system. Painting is tightly coupled to car type, but not to engine type. Painting "needs to know" about car types, but it does not "need to know" about engine types. TIP: The more tightly coupled two parts of the system are, the more tests you'll need to write to cover all the combinations of their behaviour. Another way of thinking about it is: what level are we writing tests at? You can choose to write low-level tests that cover only one part of the assembly process, or higher-level ones that test several steps together—or perhaps all of them end-to-end. See <<car-factory-illustration>>. [[car-factory-illustration]] .Analysing how many tests are needed at different levels image::images/tdd3_2101.png["An illustration of the car factory, with boxes for each step in the process (build engine, assemble, paint), and descriptions of testing each step separately vs testing them in combination."] // CSANAD: just a tiny thing: in the diagram, below the "Paint" box, there is // an apostrophe missing in "engine type doesn't matter". // SEBASTIAN: How about splitting this big image into several smaller ones? At the first encounter, I skipped it only to discover I need to jump up and down to have visualizations of paragraphs below. // Not a showstopper, tho. Analysing things in these terms, we think about the inputs and outputs that apply to each type of test, as well as which attributes of the inputs matter, and which don't. Testing the first stage of the process—building the engine—is straightforward. The "engine type" input has three possible values, so we need three tests of the output, which is the engine. If we're testing at the end-to-end level, no matter how many tests we have in total, we know we'll need at least three to be the tests that check if we can produce a car with a working engine of each type. Testing the painting needs a bit more thought. If we test at the low level, the inputs are a naked car and a paint colour. There are theoretically nine types of naked car; do we need to test all of them? No. The engine type doesn't matter; we only need to test one of each body type. Does that mean 3 × 3 = 9 tests? No. The colour and body type are independent. We can just test that all three colours work, and that all three body types work—so that's six tests. What about at the end-to-end level? It depends if we're being rigorous about "closed-box" testing, where we're not supposed to know anything about how the production process works. In that case, maybe we _do_ need 27 tests. But if we allow that we know about the internals, then we can apply similar reasoning to what we used at the lower level. However many tests we end up with, we need three of them to be checking each colour, and three that check that each body type can be painted. Let's see if we can apply this sort of analysis to our authentication system. === Using Mocks to Test Parts of Our System in Isolation To recap, so far we have some minimal tests at the models layer, and we have comprehensive tests of our authentication backend, and we're now wondering how many tests we need at the views layer. Here's the current state of our view: [role="sourcecode currentcontents"] .src/accounts/views.py ==== [source,python] ---- def login(request): if Token.objects.filter(uid=request.GET["token"]).exists(): user = User.objects.create(email="edith@example.com") auth.login(request, user) else: messages.error(request, "Invalid login link, please request a new one") return redirect("/") ---- ==== We know we want to transform it to something like this: [role="sourcecode skipme small-code"] .src/accounts/views.py ==== [source,python] ---- def login(request): if user := auth.authenticate(uid=request.GET.get("token")) # <1> auth.login(request, user) # <2> else: messages.error(request, "Invalid login link, please request a new one") # <3> return redirect("/") ---- ==== <1> We want to refactor our logic to use the `authenticate()` function from our backend. Really good place for a walrus (`:=`) too! <2> We have the happy path where the user gets logged in. <3> We have the unhappy path where the user gets an error message instead. But currently, our tests are letting us "get away" with the wrong implementation. Here are three possible options for getting ourselves to the right state: 1. Add more tests for all possible combinations at the view level (token exists but no user, token exists for an existing user, invalid token, etc.), until we end up duplicating all the logic in the auth backend in our view—and then feel justified in refactoring across to just calling the auth backend. 2. Stick with our current two tests, and decide it's OK to refactor already. 3. Test the view in isolation, using mocks to verify that we call the auth backend. Each option has pros and cons! If I was going for option (1), essentially going all in on test coverage at the views layer, I'd probably think about deleting all the tests at the auth layer afterwards. If you were to ask me what my personal preference or instinctive choice would be, I'd say at this point it might be to go with (2), and say with one happy-path and one unhappy-path test, we're OK to refactor and switch across already. But because this chapter is about mocks, let's investigate option (3) instead. Besides, it'll be an excuse to do fun things with them, like playing with `.return_value`. ((("mocks", "reducing duplication with", id="Mreduce19"))) ((("duplication, eliminating", id="dupel19"))) So far, we've used mocks to test external dependencies, like Django's mail-sending function. The main reason to use a mock we've discussed so far is to isolate ourselves from external side effects—in this case, to avoid sending out actual emails during our tests. In this section, we'll look at a different possible use case for mocks: testing parts of our _own_ code in isolation from each other, as a way of reducing duplication and avoiding combinatorial explosion in our tests. ==== Mocks Can Also Let You Test the Implementation, When It Matters On top of that, the fact that we're using the Django `auth.authenticate` function rather than calling our own code directly is relevant. Django has already introduced an abstraction: to decouple the specifics of authentication backends from the views that use them. This makes it easier for us to add further backends in future. So in this case (in contrast to the example in <<mocks-tightly-coupled-sidebar>>) the implementation _does_ matter, because we've decided to use a particular, specific interface to implement our authentication system. This is something we might want to document and verify in our tests—and mocks are one way to enable that. // SEBASTIAN: I am missing one crucial sentence here - that this Django-provided abstraction IS STABLE, so it's safe to mock it. // This is part of a public Django API, meaning it's not going anywhere soon or without breaking backwards-compatibility. That would of course be not welcomed by Django users :) // HARRY - otoh, "don't mock what you don't own". // some ppl would say, better to write your own wrapper around any third party api, // and then your mock doesn't need to change if the third party api changes. [role="pagebreak-before less_space"] === Starting Again: Test-Driving Our Implementation with Mocks Let's see how things would look if we had decided to test-drive our implementation with mocks in the first place. We'll start by reverting all the authentication stuff, both from our test and from our view. Let's disable the test first (we can re-enable them later to sense-check things): [role="sourcecode small-code"] .src/accounts/tests/test_views.py (ch21l006) ==== [source,python] ---- class LoginViewTest(TestCase): def test_redirects_to_home_page(self): <1> [...] def DONT_test_logs_in_if_given_valid_token(self): <2> [...] def DONT_test_shows_login_error_if_token_invalid(self): <2> [...] ---- ==== <1> We can leave the test for the redirect, as that doesn't involve the auth framework. <2> We change the test name so it no longer starts with `test_`, using a highly noticeable set of capital letters so we don't forget to come back and re-enable them later. I call this "DONTifying" tests. :) Now let's revert the view, and replace our hacky code with some to-dos: [role="sourcecode"] .src/accounts/views.py (ch21l007) ==== [source,python] ---- # from django.contrib import auth, messages # <1> from django.contrib import messages [...] def login(request): # TODO: call authenticate(), # <2> # then auth.login() with the user if we get one, # or messages.error() if we get None. return redirect("/") ---- ==== <1> In order to demonstrate a common error message shortly, I'm also reverting our import of the `contrib.auth` module. <2> And here's where we delete our first implementation and replace it with some to-dos. [role="pagebreak-before"] Let's check that all our tests pass: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] Ran 15 tests in 0.021s OK ---- Now let's start again with mock-based tests. First, we can write a test that makes sure we call `authenticate()` correctly: [role="sourcecode small-code"] .src/accounts/tests/test_views.py (ch21l008) ==== [source,python] ---- class LoginViewTest(TestCase): [...] @mock.patch("accounts.views.auth") # <1> def test_calls_authenticate_with_uid_from_get_request(self, mock_auth): # <2> self.client.get("/accounts/login?token=abcd123") self.assertEqual( mock_auth.authenticate.call_args, # <3> mock.call(uid="abcd123"), # <4> ) ---- ==== <1> We expect to be using the `django.contrib.auth` module in _views.py_, and we mock it out here. Note that this time, we're not mocking out a function; we're mocking out a whole module, and thus implicitly mocking out all the functions (and any other objects) that module contains. <2> As usual, the mocked object is injected into our test method. <3> This time, we've mocked out a module rather than a function. So we examine the `call_args`—not of the `mock_auth` module, but of the `mock_auth.authenticate` function. Because all the attributes of a mock are more mocks, that's a mock too. You can start to see why `Mock` objects are so convenient, compared to trying to build your own. <4> Now, instead of "unpacking" the call args, we use the `call` function for a neater way of saying what it should have been called with--that is, the token from the GET request. (See <<mock-call-args-sidebar>>.) [role="less_space pagebreak-before"] [[mock-call-args-sidebar]] .On Mock call_args ******************************************************************************* ((("call_args property"))) The `.call_args` property on a mock represents the positional and keyword arguments that the mock was called with. It's a special "call" object type, which is essentially a tuple of `(positional_args, keyword_args)`. `positional_args` is itself a tuple, consisting of the set of positional arguments. `keyword_args` is a dictionary. Here they all are in action: [role="small-code skipme"] [source,python] ---- >>> from unittest.mock import Mock, call >>> m = Mock() >>> m(42, 43, 'positional arg 3', key='val', thing=666) <Mock name='mock()' id='139909729163528'> >>> m.call_args call(42, 43, 'positional arg 3', key='val', thing=666) >>> m.call_args == ((42, 43, 'positional arg 3'), {'key': 'val', 'thing': 666}) True >>> m.call_args == call(42, 43, 'positional arg 3', key='val', thing=666) True ---- So in our test, we could have done this instead: [role="sourcecode skipme"] .src/accounts/tests/test_views.py ==== [source,python] ---- self.assertEqual( mock_auth.authenticate.call_args, ((,), {'uid': 'abcd123'}) ) # or this args, kwargs = mock_auth.authenticate.call_args self.assertEqual(args, (,)) self.assertEqual(kwargs, {'uid': 'abcd123'}) ---- ==== But you can see how using the `call` helper is nicer. See also <<avoid-assert-called-with-sidebar>>, for some discussion of `call_args` versus the magic `assert_called_with` methods. ******************************************************************************* What happens when we run the test? The first error is this: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] AttributeError: <module 'accounts.views' from '...goat-book/src/accounts/views.py'> does not have the attribute 'auth' ---- TIP: `module foo does not have the attribute bar` is a common first failure in a test that uses mocks. It's telling you that you're trying to mock out something that doesn't yet exist (or isn't yet imported) in the target module. Once we reimport `django.contrib.auth`, the error changes: [role="sourcecode"] .src/accounts/views.py (ch21l009) ==== [source,python] ---- from django.contrib import auth, messages [...] ---- ==== Now we get: [subs="specialcharacters,macros"] ---- FAIL: test_calls_authenticate_with_uid_from_get_request [...] [...] AssertionError: None != call(uid='abcd123') ---- It's telling us that the view doesn't call the `auth.authenticate` function at all. Let's fix that, but get it deliberately wrong, just to see: [role="sourcecode"] .src/accounts/views.py (ch21l010) ==== [source,python] ---- def login(request): # TODO: call authenticate(), auth.authenticate("bang!") # then auth.login() with the user if we get one, # or messages.error() if we get None. return redirect("/") ---- ==== Bang, indeed! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] AssertionError: call('bang!') != call(uid='abcd123') [...] FAILED (failures=1) ---- Let's give `authenticate` the arguments it expects then: [role="sourcecode"] .src/accounts/views.py (ch21l011) ==== [source,python] ---- def login(request): # TODO: call authenticate(), auth.authenticate(uid=request.GET["token"]) # then auth.login() with the user if we get one, # or messages.error() if we get None. return redirect("/") ---- ==== [role="pagebreak-before"] That gets us to passing tests: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] Ran 16 tests in 0.023s OK ---- ==== Using mock.return_value ((("mocks", "mock.return_value"))) Next, we want to check that if the authenticate function returns a user, we pass that into `auth.login`. Let's see how that test looks: [role="sourcecode"] .src/accounts/tests/test_views.py (ch21l012) ==== [source,python] ---- @mock.patch("accounts.views.auth") # <1> def test_calls_auth_login_with_user_if_there_is_one(self, mock_auth): response = self.client.get("/accounts/login?token=abcd123") self.assertEqual( mock_auth.login.call_args, # <2> mock.call( response.wsgi_request, # <3> mock_auth.authenticate.return_value, # <4> ), ) ---- ==== <1> We mock the `contrib.auth` module again. <2> This time we examine the call args for the `auth.login` function. <3> We check that it's called with the request object that the view sees... <4> ...and we check that the second argument was "whatever the `authenticate()` function returned". Because `authenticate()` is also mocked out, we can use its special `.return_value` attribute. We know that, in real life, that will be a user object. But in this test, it's all mocks. Can you see what I mean about mocky tests being hard to understand sometimes? When you call a mock, you get another mock. But you can also get a copy of that returned mock from the original mock that you called. Boy, it sure is hard to explain this stuff without saying "mock" a lot! Another little console illustration might help: [role="skipme"] [source,python] ---- >>> m = Mock() >>> thing = m() >>> thing <Mock name='mock()' id='140652722034952'> >>> m.return_value <Mock name='mock()' id='140652722034952'> >>> thing == m.return_value True ---- [[avoid-assert-called-with-sidebar]] .Avoid Mock's Magic assert_called...Methods? ******************************************************************************* If you've used `unittest.mock` before, you may have come across its special `assert_called...` https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock.assert_called[methods], and you may be wondering why I didn't use them. For example, instead of doing: [role="skipme"] [source,python] ---- self.assertEqual(a_mock.call_args, call(foo, bar)) ---- You can just do: [role="skipme"] [source,python] ---- a_mock.assert_called_with(foo, bar) ---- And the _mock_ library will raise an `AssertionError` for you if there is a mismatch. Why not use that? For me, the problem with these magic methods is that it's too easy to make a silly typo and end up with a test that always passes: [role="skipme"] [source,python] ---- a_mock.asssert_called_with(foo, bar) # will always pass ---- Unless you get the magic method name exactly right,footnote:[ There was actually an attempt to mitigate this problem in Python 3.5, with the addition of an `unsafe` argument that defaults to `False`, which will cause the mock to raise `AttributeError` for some common misspellings of `assert_`. Just not, for example, the one I'm using here—so I prefer not to rely on that. More info in the https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock[Python docs].] it will just silently return another mock, and you may not realise that you've written a test that tests nothing at all. That's why I prefer to always have an explicit `unittest` method in there.footnote:[ If you're using Pytest, there's an additional benefit to seeing the `assert` keyword rather than a normal method call: it makes the assert pop out.] ******************************************************************************* In any case, what do we get from running the test? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] AssertionError: None != call(<WSGIRequest: GET '/accounts/login?t[...] ---- Sure enough, it's telling us that we're not calling `auth.login()` at all yet. Let's first try doing that deliberately wrong as usual! [role="sourcecode"] .src/accounts/views.py (ch21l013) ==== [source,python] ---- def login(request): # TODO: call authenticate(), auth.authenticate(uid=request.GET["token"]) # then auth.login() with the user if we get one, auth.login("ack!") # or messages.error() if we get None. return redirect("/") ---- ==== Ack, indeed! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test accounts*] [...] ERROR: test_redirects_to_home_page [...] TypeError: login() missing 1 required positional argument: 'user' FAIL: test_calls_auth_login_with_user_if_there_is_one [...] [...] AssertionError: call('ack!') != call(<WSGIRequest: GET '/accounts/login?token=[...] [...] Ran 17 tests in 0.026s FAILED (failures=1, errors=1) ---- That's one expected failure from our mocky test, and one (more) unexpected failure from the non-mocky test. Let's see if we can fix them: [role="sourcecode"] .src/accounts/views.py (ch21l014) ==== [source,python] ---- def login(request): # TODO: call authenticate(), user = auth.authenticate(uid=request.GET["token"]) # then auth.login() with the user if we get one, auth.login(request, user) # or messages.error() if we get None. return redirect("/") ---- ==== Well, that does fix our mocky test, but not the other one; it now has a slightly different complaint: [subs="specialcharacters,macros"] ---- ERROR: test_redirects_to_home_page (accounts.tests.test_views.LoginViewTest.test_redirects_to_home_page) [...] File "...goat-book/src/accounts/views.py", line 33, in login auth.login(request, user) [...] AttributeError: 'AnonymousUser' object has no attribute '_meta' ---- It's because we're still calling `auth.login` indiscriminately on any kind of user, and that's causing problems back in our original test for the redirect, which _isn't_ currently mocking out `auth.login`. [role="pagebreak-before"] We can get back to passing like this: [role="sourcecode"] .src/accounts/views.py (ch21l015) ==== [source,python] ---- def login(request): # TODO: call authenticate(), if user := auth.authenticate(uid=request.GET["token"]): # then auth.login() with the user if we get one, auth.login(request, user) ---- ==== This gets our unit test passing: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test accounts* [...] OK ---- ==== Using .return_value During Test Setup I'm a little nervous that we've introduced an `if` without an _explicit_ test for it. Testing the unhappy path will reassure me. We can use our existing test for the error case to crib from. We want to be able to set up our mocks to say: `auth.authenticate()` should return `None`. We can do that by setting the `.return_value` on the mock: [role="sourcecode"] .src/accounts/tests/test_views.py (ch21l016) ==== [source,python] ---- @mock.patch("accounts.views.auth") def test_adds_error_message_if_auth_user_is_None(self, mock_auth): mock_auth.authenticate.return_value = None # <1> response = self.client.get("/accounts/login?token=abcd123", follow=True) message = list(response.context["messages"])[0] self.assertEqual( # <2> message.message, "Invalid login link, please request a new one", ) self.assertEqual(message.tags, "error") ---- ==== <1> We use `.return_value` on our mock once again. But this time, we assign to it _before_ it's used (in the setup part of the test—aka the "arrange" or "given" phase), rather than reading from it (in the assert/“when” part), as we did earlier. <2> Our asserts are copied across from the existing test for the error case, `DONT_test_shows_login_error_if_token_invalid()`. [role="pagebreak-before"] That gives us this somewhat cryptic, but expected failure: ---- ERROR: test_adds_error_message_if_auth_user_is_None [...] [...] message = list(response.context["messages"])[0] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^ IndexError: list index out of range ---- Essentially, that's saying there are no messages in our response. We can get it passing like this, starting with a deliberate mistake as always: [role="sourcecode"] .src/accounts/views.py (ch21l017) ==== [source,python] ---- def login(request): # TODO: call authenticate(), if user := auth.authenticate(uid=request.GET["token"]): # then auth.login() with the user if we get one, auth.login(request, user) else: # or messages.error() if we get None. messages.error(request, "boo") return redirect("/") ---- ==== Which gives us: ---- AssertionError: 'boo' != 'Invalid login link, please request a new one' ---- And so: [role="sourcecode"] .src/accounts/views.py (ch21l018) ==== [source,python] ---- def login(request): # TODO: call authenticate(), if user := auth.authenticate(uid=request.GET["token"]): # then auth.login() with the user if we get one, auth.login(request, user) else: # or messages.error() if we get None. messages.error(request, "Invalid login link, please request a new one") return redirect("/") ---- ==== Now our tests pass: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test accounts* [...] Ran 18 tests in 0.025s OK ---- [role="pagebreak-before"] And we can do a final refactor to remove those comments: [role="sourcecode"] .src/accounts/views.py (ch21l019) ==== [source,python] ---- from accounts.models import Token # <1> [...] def login(request): # <2> if user := auth.authenticate(uid=request.GET["token"]): auth.login(request, user) else: messages.error(request, "Invalid login link, please request a new one") return redirect("/") ---- ==== <1> We no longer need to explicitly import the user model <2> and our view is down to just five lines. Lovely! What's next? ((("", startref="Mreduce19")))((("", startref="dupel19"))) ==== UnDONTifying Remember we still have the DONTified, non-mocky tests? Let's re-enable now to sense-check that our mocky tests have driven us to the right place: [role="sourcecode small-code"] .src/accounts/tests/test_views.py (ch21l020) ==== [source,diff] ---- @@ -63,7 +63,7 @@ class LoginViewTest(TestCase): response = self.client.get("/accounts/login?token=abcd123") self.assertRedirects(response, "/") - def DONT_test_logs_in_if_given_valid_token(self): + def test_logs_in_if_given_valid_token(self): anon_user = auth.get_user(self.client) self.assertEqual(anon_user.is_authenticated, False) @@ -74,7 +74,7 @@ class LoginViewTest(TestCase): self.assertEqual(user.is_authenticated, True) self.assertEqual(user.email, "edith@example.com") - def DONT_test_shows_login_error_if_token_invalid(self): + def test_shows_login_error_if_token_invalid(self): response = self.client.get("/accounts/login?token=invalid-token", follow=True) ---- ==== Sure enough, they both pass: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test accounts* [...] Ran 20 tests in 0.025s OK ---- === Deciding Which Tests to Keep We now definitely have duplicate tests: [role="sourcecode skipme"] .src/accounts/tests/test_views.py ==== [source,python] ---- class LoginViewTest(TestCase): def test_redirects_to_home_page(self): [...] def test_logs_in_if_given_valid_token(self): [...] def test_shows_login_error_if_token_invalid(self): [...] @mock.patch("accounts.views.auth") def test_calls_authenticate_with_uid_from_get_request(self, mock_auth): [...] @mock.patch("accounts.views.auth") def test_calls_auth_login_with_user_if_there_is_one(self, mock_auth): [...] @mock.patch("accounts.views.auth") def test_adds_error_message_if_auth_user_is_None(self, mock_auth): [...] ---- ==== The redirect test could stay the same whether we're using mocks or not. We then have two non-mocky tests for the happy and unhappy paths, and three mocky tests: . One checks that we are integrated with our auth backend correctly. . One checks that we call the built-in `auth.login` function correctly, which tests the happy path. . And one checks we set an error message in the unhappy path. [role="pagebreak-before"] I think there are lots of ways to justify different choices here, but my instinct tends to be to avoid using mocks if you can. So, I propose we delete the two mocky tests for the happy and unhappy paths, as they are reasonably covered by the non-mocky ones. But I think we can justify keeping the first mocky test, because it adds value by checking that we're doing our authentication the "right" way—i.e., by calling into Django's `auth.authenticate()` function (instead of, for example, instantiating and calling our auth backend ourselves, or even just implementing authentication inline in the view). TIP: "Test behaviour, not implementation" is a GREAT rule of thumb for tests. But sometimes, the fact that you're using one implementation rather than another really is important. In these cases, a mocky test can be useful. So let's delete our last two mocky tests. I'm also going to rename the remaining one to make our intention clear; we want to check we are using the Django auth library: [role="sourcecode"] .src/accounts/tests/test_views.py (ch21l021) ==== [source,python] ---- @mock.patch("accounts.views.auth") def test_calls_django_auth_authenticate(self, mock_auth): [...] ---- ==== // CSANAD: I think the `diff` style snippets are better for renaming things. And we're down to 18 tests: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test accounts* [...] Ran 18 tests in 0.015s OK ---- [role="pagebreak-before less_space"] === The Moment of Truth: Will the FT Pass? ((("mocks", "functional test for"))) ((("functional tests (FTs)", "for mocks", secondary-sortas="mocks"))) We're just about ready to try our functional test! Let's just make sure our base template shows a different navbar for logged-in and non–logged-in users. Our FT relies on being able to see the user's email in the navbar in the logged-in state, and it needs a "Log out" button too: [role="sourcecode small-code"] .src/lists/templates/base.html (ch21l022) ==== [source,html] ---- <nav class="navbar"> <div class="container-fluid"> <a class="navbar-brand" href="/">Superlists</a> {% if user.email %} <1> <span class="navbar-text">Logged in as {{ user.email }}</span> <form method="POST" action="TODO"> {% csrf_token %} <button id="id_logout" class="btn btn-outline-secondary" type="submit"> Log out </button> </form> {% else %} <form method="POST" action="{% url 'send_login_email' %}"> <div class="input-group"> <label class="navbar-text me-2" for="id_email_input"> Enter your email to log in </label> <input id="id_email_input" name="email" class="form-control" placeholder="your@email.com" /> {% csrf_token %} </div> </form> {% endif %} </div> </nav> ---- ==== <1> Here's a new `{% if %}`, and navbar content for logged-in users. [role="pagebreak-before"] OK, there's a to-do in there about the log-out button. We'll get to that, but how does our FT look now? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_login*] [...] . --------------------------------------------------------------------- Ran 1 test in 3.282s OK ---- === It Works in Theory! Does It Work in Practice? ((("mocks", "practical application of"))) Wow! Can you believe it? I scarcely can! Time for a manual look around with `runserver`: [role="skipme"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py runserver*] [...] Internal Server Error: /accounts/send_login_email Traceback (most recent call last): File "...goat-book/accounts/views.py", line 20, in send_login_email ConnectionRefusedError: [Errno 111] Connection refused ---- ==== Using Our New Environment Variable, and Saving It to .env You'll probably get an error, like I did, when you try to run things manually. It's because of two things. Firstly, we need to re-add the email configuration to _settings.py_: // DAVID: Shouldn't we write a failing test first? If not, why not? [role="sourcecode"] .src/superlists/settings.py (ch21l023) ==== [source,python] ---- EMAIL_HOST = "smtp.gmail.com" EMAIL_HOST_USER = "obeythetestinggoat@gmail.com" EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_PASSWORD") EMAIL_PORT = 587 EMAIL_USE_TLS = True ---- ==== Secondly, we (probably) need to reset the `EMAIL_PASSWORD` in our shell: [subs="specialcharacters,quotes"] ---- $ *export EMAIL_PASSWORD="yoursekritpasswordhere"* ---- [role="pagebreak-before less_space"] .Using a Local .env File for Development ******************************************************************************* Until now, we've not needed to "save" any of our local environment variables, because the command-line ones are easy to remember and type, and we've made sure all the other ones that affect config settings have sensible defaults for dev. But there's just no way to get a working login system without this one! Rather than having to go look up this password every time you start a new shell, it's quite common to save these sorts of settings into a local file in your project folder named `.env`. It's a convention that makes it a hidden file, on Unix-like systems at least: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *echo .env >> .gitignore* # we don't want to commit our secrets into git! $ *echo 'EMAIL_PASSWORD="yoursekritpasswordhere"' >> .env* $ *set -a; source .env; set +a;* ---- It does mean you have to remember to do that weird `set -a; source...` dance, every time you start working on the project, as well as remembering to activate your virtualenv. If you search or ask around, you'll find there are some tools and shell plugins that load virtualenvs and _.env_ files automatically, or Django plugins that handle this stuff too. A few options: * Django-specific: https://django-environ.readthedocs.io[django-environ] or https://github.com/jpadilla/django-dotenv[django-dotenv] * More general Python project management: https://docs.pipenv.org[Pipenv] * Or even: https://oreil.ly/F9iV3[roll your own] ******************************************************************************* And now... [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *python src/manage.py runserver* ---- ...you should see something like <<despiked-success-message>>. [[despiked-success-message]] .Check your email... image::images/tdd3_2102.png["De-spiked site with success message"] Woohoo! I've been waiting to do a commit up until this moment, just to make sure everything works. At this point, you could make a series of separate commits--one for the login view, one for the auth backend, one for the user model, one for wiring up the template. Or you could decide that—because they're all interrelated, and none will work without the others—you may as well just have one big commit: [subs="specialcharacters,quotes"] ---- $ *git status* $ *git add .* $ *git diff --staged* $ *git commit -m "Custom passwordless auth backend + custom user model"* ---- [role="scratchpad"] ***** * _[strikethrough line-through]#Token model with email and UID#_ * _[strikethrough line-through]#View to create token and send login email incl. url w/ token UID#_ * _[strikethrough line-through]#Custom user model with USERNAME_FIELD=email#_ * _[strikethrough line-through]#Authentication backend with authenticate() and get_user() functions#_ * _[strikethrough line-through]#Registering auth backend in settings.py#_ * _[strikethrough line-through]#Login view calls authenticate() and login() from django.contrib.auth#_ * _Logout view calls django.contrib.auth.logout_ ***** === Finishing Off Our FT: Testing Logout ((("mocks", "logout link"))) The last thing we need to do before we call it a day is to test the logout button. We extend the FT with a couple more steps: [role="sourcecode small-code"] .src/functional_tests/test_login.py (ch21l024) ==== [source,python] ---- [...] # she is logged in! self.wait_for( lambda: self.browser.find_element(By.CSS_SELECTOR, "#id_logout"), ) navbar = self.browser.find_element(By.CSS_SELECTOR, ".navbar") self.assertIn(TEST_EMAIL, navbar.text) # Now she logs out self.browser.find_element(By.CSS_SELECTOR, "#id_logout").click() # She is logged out self.wait_for( lambda: self.browser.find_element(By.CSS_SELECTOR, "input[name=email]") ) navbar = self.browser.find_element(By.CSS_SELECTOR, ".navbar") self.assertNotIn(TEST_EMAIL, navbar.text) ---- ==== With that, we can see that the test is failing because the logout button doesn't have a valid URL to submit to: [subs=""] ---- $ <strong>python src/manage.py test functional_tests.test_login</strong> [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: input[name=email]; [...] ---- So, let's tell the base template that we want a new URL named "logout": [role="sourcecode small-code"] .src/lists/templates/base.html (ch21l025) ==== [source,html] ---- {% if user.email %} <span class="navbar-text">Logged in as {{ user.email }}</span> <form method="POST" action="{% url 'logout' %}"> {% csrf_token %} <button id="id_logout" class="btn btn-outline-secondary" type="submit"> Log out </button> </form> {% else %} ---- ==== If you try the FTs at this point, you'll see an error saying that the URL doesn't exist yet: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_login*] Internal Server Error: / [...] django.urls.exceptions.NoReverseMatch: Reverse for 'logout' not found. 'logout' is not a valid view function or pattern name. ====================================================================== ERROR: test_login_using_magic_link (functional_tests.test_login.LoginTest.test_login_using_magic_link) [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: #id_logout; [...] ---- Implementing a logout URL is actually very simple: we can use Django's https://docs.djangoproject.com/en/5.2/topics/auth/default/#module-django.contrib.auth.views[built-in logout view], which clears down the user's session and redirects them to a page of our choice: [role="sourcecode small-code"] .src/accounts/urls.py (ch21l026) ==== [source,python] ---- from django.contrib.auth import views as auth_views from django.urls import path from . import views urlpatterns = [ path("send_login_email", views.send_login_email, name="send_login_email"), path("login", views.login, name="login"), path("logout", auth_views.LogoutView.as_view(next_page="/"), name="logout"), ] ---- ==== [role="pagebreak-before"] And that gets us a fully passing FT--indeed, a fully passing test suite: [subs="specialcharacters,quotes"] ---- $ *python src/manage.py test functional_tests.test_login* [...] OK $ *cd src && python manage.py test* [...] Ran 56 tests in 78.124s OK ---- WARNING: We're nowhere near a truly secure or acceptable login system here. As this is just an example app for a book, we'll leave it at that, but in "real life" you'd want to explore a lot more security and usability issues before calling the job done. We're dangerously close to "rolling our own crypto" here, and relying on a more established login system would be much safer. Read more at https://security.stackexchange.com/a/18198. ((("security issues and settings", "login systems"))) // CSANAD: for demonstrating a security issue with our current, custom // authentication, we could mention that after logout, we can log in using any // of the previous login magic links (there is no token invalidation) [role="scratchpad"] ***** * _[strikethrough line-through]#Token model with email and UID#_ * _[strikethrough line-through]#View to create token and send login email incl. url w/ token UID#_ * _[strikethrough line-through]#Custom user model with USERNAME_FIELD=email#_ * _[strikethrough line-through]#Authentication backend with authenticate() and get_user() functions#_ * _[strikethrough line-through]#Registering auth backend in settings.py#_ * _[strikethrough line-through]#Login view calls authenticate() and login() from django.contrib.auth#_ * _[strikethrough line-through]#Logout view calls django.contrib.auth.logout#_ ***** In the next chapter, we'll start trying to put our login system to good use. In the meantime, do a commit and enjoy this recap. [role="pagebreak-before less_space"] [[mocking-py-sidebar]] .On Mocking in Python ******************************************************************************* Using mock.return_value:: The `.return_value` attribute on a mock can be used in two ways. You can _read_ it, to access the return value of a mocked-out function, and thus check on how it gets used later in your code; this usually happens in the "assert" or "then" part of your test. Alternatively, you can _assign_ to it, usually up-front in the "arrange" or "given" part of your test, as a way of saying "I want this mocked-out function to return a particular value". Mocks can ensure test isolation and reduce duplication:: You can use mocks to isolate different parts of your code from each other, and thus test them independently. This can help you to avoid duplication, because you're only testing a single layer at a time, rather than having to think about combinations of interactions of different layers. Used extensively, this approach leads to "London-style" TDD, but that's quite different from the style I mostly follow and show in this book. ((("mocks", "reducing duplication with"))) ((("duplication, eliminating"))) Mocks can enable you to verify implementation details:: Most tests should test behaviour, not implementation. At some point though, we decided using a particular implementation _was_ important. And so, we used a mock as a way to verify that, and to document it for our future selves. There are alternatives to mocks, but they require rethinking how your code is structured:: In a way, mocks make it "too easy". In programming languages that lack Python's dynamic ability to monkeypatch things at runtime, developers have had to work on alternative ways to test code with dependencies. While these techniques can be more complex, they do force you to think about how your code is structured—to cleanly identify your dependencies and build clean abstractions and interfaces around them. Further discussion is beyond the scope of this book, but check out http://cosmicpython.com[Cosmic Python]. ******************************************************************************* ================================================ FILE: chapter_22_fixtures_and_wait_decorator.asciidoc ================================================ [[chapter_22_fixtures_and_wait_decorator]] == Test Fixtures and a Decorator [keep-together]#for Explicit Waits# ((("authentication", "skipping in FTs"))) Now that we have a functional authentication system, we want to use it to identify users, and to show them all the lists they have created. To do that, we're going to have to write FTs that have a logged-in user. Rather than making each test go through the (time-consuming) login email dance, we want to be able to skip that part. This is about separation of concerns.((("functional tests (FTs)", "versus unit tests", secondary-sortas="unit")))((("unit tests", "versus functional tests", secondary-sortas="functional"))) Functional tests aren't like unit tests, in that they don't usually have a single assertion. But, conceptually, they should be testing a single thing. There's no need for every single FT to test the login/logout mechanisms. If we can figure out a way to "cheat" and skip that part, we'll spend less time waiting for tests to repeat these duplicated setup steps. TIP: Don't overdo de-duplication in FTs. One of the benefits of an FT is that it can catch strange and unpredictable interactions between different parts of your application. In this short chapter, we'll start writing our new FT, and use that as an opportunity to talk about de-duplication using test fixtures for FTs. We'll also refactor out a nice helper for explicit waits, using Python's lovely decorator syntax. === Skipping the Login Process by Pre-creating a Session ((("sessions", "pre-creating", id="ix_sesspre"))) ((("login process, skipping", seealso="authentication"))) ((("cookies"))) It's quite common for a user to return to a site and still have a cookie, which means they are "pre-authenticated", so this isn't an unrealistic cheat at all. Here's how you can set it up: [role="sourcecode"] .src/functional_tests/test_my_lists.py (ch22l001) ==== [source,python] ---- from django.conf import settings from django.contrib.auth import BACKEND_SESSION_KEY, SESSION_KEY, get_user_model from django.contrib.sessions.backends.db import SessionStore from .base import FunctionalTest User = get_user_model() class MyListsTest(FunctionalTest): def create_pre_authenticated_session(self, email): user = User.objects.create(email=email) session = SessionStore() session[SESSION_KEY] = user.pk # <1> session[BACKEND_SESSION_KEY] = settings.AUTHENTICATION_BACKENDS[0] session.save() ## to set a cookie we need to first visit the domain. ## 404 pages load the quickest! self.browser.get(self.live_server_url + "/404_no_such_url/") self.browser.add_cookie( dict( name=settings.SESSION_COOKIE_NAME, value=session.session_key, # <2> path="/", ) ) ---- ==== <1> We create a session object in the database. The session key is the primary key of the user object (which is actually the user's email address). // CSANAD: there is a different suggested way of importing SessionStore, using // the SESSION_ENGINE from the `settings`: // https://docs.djangoproject.com/en/5.2/topics/http/sessions/#using-sessions-out-of-views <2> We then add a cookie to the browser that matches the session on the server--on our next visit to the site, the server should recognise us as a logged-in user. Note that, as it is, this will only work because we're using `LiveServerTestCase`, so the `User` and `Session` objects we create will end up in the same database as the test server. At some point, we'll need to think about how this will work against Docker or staging.((("LiveServerTestCase"))) [role="pagebreak-before less_space"] .Django Sessions: How a User's Cookies Tell the Server They Are Authenticated ********************************************************************** This is an attempt to explain sessions, cookies, and authentication in Django. ((("authentication", "cookies and"))) HTTP is a "stateless" protocol, meaning that the protocol itself doesn't keep track of any state from one request to the next, and each request is independent of the next. There's no built-in way to tell that a series of requests come from the same client. For this reason, servers need a way of recognising different clients with _every single request_. The usual solution is to give each client a unique session ID, which the browser will store in a text file called a "cookie" and send with every request.((("cookies", "session"))) The server will store that ID somewhere (by default, in the database), and then it can recognise each request that comes in as being from a particular client. If you log in to the site using the dev server, you can actually take a look at your session ID by hand if you like. It's stored under the key `sessionid` by default. See <<session-cookie-screenshot>>. [[session-cookie-screenshot]] .Examining the session cookie in the DevTools UI image::images/tdd3_2201.png["A browser with the devtools open, showing a session cookie called sessionid for localhost:800"] These session cookies are set for all visitors to a Django site, whether they're logged in or not. When we want to recognise a client as being a logged-in and authenticated user, again, rather than asking the client to send their username and password with every single request, the server can actually just mark that client's session as authenticated, and associate it with a user ID in its database.((("user IDs (UIDs)", "for Django sessions", secondary-sortas="Django")))((("Django framework", "sessions"))) A Django session is a dictionary-like data structure, and the user ID is stored under the key given by `django.contrib.auth.SESSION_KEY`. You can check this out in a [keep-together]#`./manage.py`# `shell` if you like: ++++ <pre translate="no" data-type="programlisting" class="skipme small-code">$ <strong>python src/manage.py shell</strong> [...] In [1]: from django.contrib.sessions.models import Session # substitute your session id from your browser cookie here In [2]: session = Session.objects.get( session_key="8u0pygdy9blo696g3n4o078ygt6l8y0y" ) In [3]: print(session.get_decoded()) {'_auth_user_id': 'obeythetestinggoat@gmail.com', '_auth_user_backend': 'accounts.authentication.PasswordlessAuthenticationBackend'}</pre> ++++ You can also store any other information you like on a user's session, as a way of temporarily keeping track of some state. This works for non–logged-in users too.((("sessions", "pre-creating", startref="ix_sesspre"))) Just use `request.session` inside any view, and it works as a dictionary. There's more information in the https://docs.djangoproject.com/en/5.2/topics/http/sessions[Django docs on sessions]. ********************************************************************** ==== Checking That It Works To check that the `create_pre_authenticated_session()` system works, it would be good to reuse some of the code from our previous test.((("sessions", "testing pre-creation of sessions"))) Let's make a couple of functions: `wait_to_be_logged_in` and `wait_to_be_logged_out`. To access them from a different test, we'll need to pull them up into `FunctionalTest`. We'll also tweak them slightly so that they can take an arbitrary email address as a parameter: [role="sourcecode small-code"] .src/functional_tests/base.py (ch22l002) ==== [source,python] ---- class FunctionalTest(StaticLiveServerTestCase): [...] def wait_to_be_logged_in(self, email): self.wait_for( lambda: self.browser.find_element(By.CSS_SELECTOR, "#id_logout"), ) navbar = self.browser.find_element(By.CSS_SELECTOR, ".navbar") self.assertIn(email, navbar.text) def wait_to_be_logged_out(self, email): self.wait_for( lambda: self.browser.find_element(By.CSS_SELECTOR, "input[name=email]") ) navbar = self.browser.find_element(By.CSS_SELECTOR, ".navbar") self.assertNotIn(email, navbar.text) ---- ==== Hmm, that's not bad. But I'm not quite happy with the amount of duplication of `wait_for` stuff in here. Let's make a note to come back to it and let's first get these helpers working: [role="scratchpad"] ***** * 'Clean up wait_for stuff in base.py.' ***** First, we use them in 'test_login.py': [role="sourcecode"] .src/functional_tests/test_login.py (ch22l003) ==== [source,python] ---- def test_login_using_magic_link(self): [...] # she is logged in! self.wait_to_be_logged_in(email=TEST_EMAIL) # Now she logs out self.browser.find_element(By.CSS_SELECTOR, "#id_logout").click() # She is logged out self.wait_to_be_logged_out(email=TEST_EMAIL) ---- ==== Just to make sure we haven't broken anything, we rerun the login test: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_login*] [...] OK ---- And now we can write a placeholder for the "My lists" test, to see if our pre-authenticated session creator really does work: [role="sourcecode"] .src/functional_tests/test_my_lists.py (ch22l004) ==== [source,python] ---- def test_logged_in_users_lists_are_saved_as_my_lists(self): email = "edith@example.com" self.browser.get(self.live_server_url) self.wait_to_be_logged_out(email) # Edith is a logged-in user self.create_pre_authenticated_session(email) self.browser.get(self.live_server_url) self.wait_to_be_logged_in(email) ---- ==== That gets us: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_my_lists*] [...] OK ---- That's a good place for a commit: [subs="specialcharacters,quotes"] ---- $ *git add src/functional_tests* $ *git commit -m "test_my_lists: precreate sessions, move login checks into base"* ---- .JSON Test Fixtures Considered Harmful ******************************************************************************* ((("JSON fixtures"))) ((("fixtures", "JSON fixtures"))) ((("test fixtures")))((("Django framework", "fixtures"))) When we pre-populate the database with test data—as we've done here with the `User` object and its associated `Session` object—what we're doing is setting up what's called a "test fixture". If you look up "Django fixtures", you'll find that Django has a built-in way of saving objects from your database using JSON (using `manage.py dumpdata`), and automatically loading them in your test runs using the `fixtures` class attribute on `TestCase`. You'll find people out there saying https://oreil.ly/Nklcr[not to use JSON fixtures], and I tend to agree. They're a nightmare to maintain when your model changes. Plus, it's difficult for the reader to tell which of the many attribute values specified in the JSON are critical for the behaviour under test, and which of them are just filler. Finally, even if tests start out sharing fixtures, sooner or later one test will want slightly different versions of the data, and you end up copying the whole thing around to keep them isolated. Again, it's hard to tell what's relevant to the test and what is just happenstance. It's usually much more straightforward to just load the data directly using the Django ORM. TIP: Once you have more than a handful of fields on a model, and/or several related models, you'll want to factor out some nice helper methods with descriptive names to build out your data. A lot of people also like https://factoryboy.readthedocs.org[`factory_boy`], but I think the most important thing is the descriptive names. ******************************************************************************* === Our Final Explicit Wait Helper: A Wait Decorator ((("decorators", "wait decorator", id="Dwait20"))) ((("explicit and implicit waits", id="exp20"))) ((("implicit and explicit waits", id="imp20"))) ((("helper methods", id="help20"))) ((("wait_for_row_in_list_table helper method"))) ((("self.wait_for helper method"))) ((("wait_to_be_logged_in/out")))((("waits", "explicit wait helper, wait decorator", id="ix_waitdec"))) We've used decorators a few times in our code so far, but it's time to learn how they actually work by making one of our own. First, let's imagine how we might want our decorator to work. It would be nice to be able to replace all the custom wait/retry/timeout logic in `wait_for_row_in_list_table()` and the inline `self.wait_fors()` in the `wait_to_be_logged_in/out`. Something like this would look lovely: // TODO: there's a change to the rows= here, backport. // DAVID: I didn't realise that I was meant to paste this in yet - // be more explicit? [role="sourcecode"] .src/functional_tests/base.py (ch22l005) ==== [source,python] ---- @wait def wait_for_row_in_list_table(self, row_text): rows = self.browser.find_elements(By.CSS_SELECTOR, "#id_list_table tr") self.assertIn(row_text, [row.text for row in rows]) @wait def wait_to_be_logged_in(self, email): self.browser.find_element(By.CSS_SELECTOR, "#id_logout") navbar = self.browser.find_element(By.CSS_SELECTOR, ".navbar") self.assertIn(email, navbar.text) @wait def wait_to_be_logged_out(self, email): self.browser.find_element(By.CSS_SELECTOR, "input[name=email]") navbar = self.browser.find_element(By.CSS_SELECTOR, ".navbar") self.assertNotIn(email, navbar.text) ---- ==== Are you ready to dive in? Although decorators are quite difficult to wrap your head around,footnote:[I know it took me a long time before I was comfortable with them, and I still have to think about them quite carefully whenever I make one.] the nice thing is that we've already dipped our toes into functional programming in our `self.wait_for()` helper function. That's a function that takes another function as an argument—and a decorator is the same. The difference is that the decorator doesn't actually execute any code itself; it returns a modified version of the function that it was given. Our decorator wants to return a new function, which will keep retrying the function being decorated—catching our usual exceptions until a timeout occurs. Here's a first cut: [role="sourcecode"] .src/functional_tests/base.py (ch22l006) ==== [source,python] ---- def wait(fn): #<1> def modified_fn(): #<3> start_time = time.time() while True: #<4> try: return fn() #<5> except (AssertionError, WebDriverException) as e: #<4> if time.time() - start_time > MAX_WAIT: raise e time.sleep(0.5) return modified_fn #<2> ---- ==== // JAN: Why not use functools.wraps here? <1> A decorator is a way of modifying a function; it takes a function as an [keep-together]#argument...# <2> ...and returns another function as the modified (or "decorated") version. <3> Here's where we define our modified function. <4> And here's our familiar loop, which will keep catching those exceptions until the timeout. <5> And as always, we call our original function and return immediately if there are no [keep-together]#exceptions#. //IDEA: discuss the fact that multiple calls to fn() may have side-effects? That's _almost_ right, but not quite; try running it? [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_my_lists*] [...] self.wait_to_be_logged_out(email) TypeError: wait.<locals>.modified_fn() takes 0 positional arguments but 2 were given ---- [role="pagebreak-before"] Unlike in `self.wait_for`, the decorator is being applied to functions that have [keep-together]#arguments#: [role="sourcecode currentcontents"] .src/functional_tests/base.py ==== [source,python] ---- @wait def wait_to_be_logged_in(self, email): self.browser.find_element(By.CSS_SELECTOR, "#id_logout") [...] ---- ==== `wait_to_be_logged_in` takes `self` and `email` as positional arguments. But when it's decorated, it's replaced with `modified_fn`, which currently takes no arguments. How do we magically make it so our `modified_fn` can handle the same arguments as whatever function the decorator is given?((("variadic arguments")))((("kwargs"))) The answer is a bit of Python magic, +++<code>*args</code>+++ and +++<code>**kwargs</code>+++, more formally known as https://docs.python.org/3/tutorial/controlflow.html#arbitrary-argument-lists["variadic arguments"] (apparently—I only just learned that): [role="sourcecode"] .src/functional_tests/base.py (ch22l007) ==== [source,python] ---- def wait(fn): def modified_fn(*args, **kwargs): #<1> start_time = time.time() while True: try: return fn(*args, **kwargs) #<2> except (AssertionError, WebDriverException) as e: if time.time() - start_time > MAX_WAIT: raise e time.sleep(0.5) return modified_fn ---- ==== <1> Using +++<code>*args</code>+++ and +++<code>**kwargs</code>+++, we specify that `modified_fn()` may take any arbitrary positional and keyword arguments. <2> As we've captured them in the function definition, we make sure to pass those same arguments to `fn()` when we actually call it. One of the fun things this can be used for is to make a decorator that changes the arguments of a function. But we won't get into that now. The main thing is that our decorator now works! // SEBASTIAN: that's actually an awful idea, making it harder to leverage type hints. I wouldn't be giving people such ideas :D [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_my_lists*] [...] OK ---- And do you know what's truly satisfying? We can use our `wait` decorator for our `self.wait_for` helper as well! Like this: [role="sourcecode"] .src/functional_tests/base.py (ch22l008) ==== [source,python] ---- @wait def wait_for(self, fn): return fn() ---- ==== Lovely! Now all our wait/retry logic is encapsulated in a single place, and we have a nice easy way of applying those waits—either inline in our FTs using `self.wait_for()`, or on any helper function using the `@wait` decorator. Let's just check all the FTs still pass of course: ---- Ran 8 tests in 19.379s OK ---- Do a commit, and we're good to cross off that scratchpad item: [role="scratchpad"] ***** * '[strikethrough line-through]#Clean up wait_for stuff in base.py.#' ***** In the next chapter, we'll try to deploy our code to staging, and use the pre-authenticated session fixtures on the server. As we'll see, it'll help us catch a little bug or two! ((("waits", "explicit wait helper, wait decorator", startref="ix_waitdec")))((("", startref="Dwait20"))) ((("", startref="exp20"))) ((("", startref="imp20"))) [role="pagebreak-before less_space"] .Lessons Learned ******************************************************************************* Decorators:: Decorators can be a great way of abstracting out different levels of concerns. They let us write our test assertions without having to think about waits at the same time. ((("decorators", "benefits of"))) De-duplicating your FTs, with caution:: Every single FT doesn't need to test every single part of your application. In our case, we wanted to avoid going through the full login process for every FT that needs an authenticated user, so we used a test fixture to "cheat" and skip that part. You might find other things you want to skip in your FTs. A word of caution, however: functional tests are there to catch unpredictable interactions between different parts of your application, so be wary of pushing de-duplication to the extreme. ((("duplication, eliminating"))) Test fixtures:: Test fixtures refers to test data that needs to be set up as a precondition before a test is run--often this means populating the database with some information, but as we've seen (with browser cookies), it can involve other types of preconditions. ((("test fixtures"))) Avoiding JSON fixtures:: Django makes it easy to save and restore data from the database in JSON format (and others) using the `dumpdata` and `loaddata` management commands. I would tend to recommend against them, as they are painful to manage when your database schema changes. Use the ORM, with some nicely named helper functions instead. ((("JSON fixtures"))) ((("dumpdata command"))) ((("loaddata command"))) ((("fixtures", "JSON fixtures"))) ******************************************************************************* ================================================ FILE: chapter_23_debugging_prod.asciidoc ================================================ [[chapter_23_debugging_prod]] == Debugging and Testing Server Issues Popping a few layers off the stack of things we're working on: we have nice wait-for helpers; what were we using them for? Oh yes, waiting to be logged in. And why was that? Ah yes, we had just built a way of pre-authenticating a user. Let's see how that works against Docker and our staging server. === The Proof Is in the Pudding: Using Docker to Catch Final Bugs Remember the deployment checklist from <<chapter_18_second_deploy>>? Let's see if it can't come in handy today!((("Docker", "using to catch bugs in authentication system"))) First, we rebuild and start our Docker container locally, on port `8888`: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/container.db.sqlite3",target=/home/nonroot/db.sqlite3 \ -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -e DJANGO_DB_PATH=/home/nonroot/db.sqlite3 \ -it superlists* [...] => => naming to docker.io/library/superlists [...] [2025-01-27 22:37:02 +0000] [7] [INFO] Starting gunicorn 22.0.0 [2025-01-27 22:37:02 +0000] [7] [INFO] Listening at: http://0.0.0.0:8888 (7) [2025-01-27 22:37:02 +0000] [7] [INFO] Using worker: sync [2025-01-27 22:37:02 +0000] [8] [INFO] Booting worker with pid: 8 ---- // TODO: we really should have shellscripts for this NOTE: If you see an error saying `bind source path does not exist`, you've lost your container database somehow. Create a new one with `touch container.db.sqlite3`. Now let's make sure our container database is fully up to date, by running `migrate` inside the container: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *docker exec $(docker ps --filter=ancestor=superlists -q) python manage.py migrate* Operations to perform: Apply all migrations: accounts, auth, contenttypes, lists, sessions Running migrations: [...] ---- NOTE: That little `$(docker ps --filter=ancestor=superlists -q)` is a neat way to avoid manually looking up the container ID. An alternative would be to just set the container name explicitly in our `docker run` commands, using `--name`. And now, let's do an FT run: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests*] [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: #id_logout; [...] [...] AssertionError: 'Check your email' not found in 'Server Error (500)' [...] FAILED (failures=1, errors=1) ---- We can't log in--either with the real email system or with our pre-authenticated session. Looks like our nice new authentication system is crashing when we run it in Docker. Let's practice a bit of production debugging! === Inspecting the Docker Container Logs ((("logging", "inspecting Docker container logs"))) ((("Gunicorn", "logging setup"))) When Django fails with a 500 or "unhandled exception" and `DEBUG` is off, it doesn't print the tracebacks to your web browser. But it will send them to your logs instead. [role="pagebreak-before less_space"] .Check Our Django LOGGING Settings ******************************************************************************* It's worth double-checking at this point that your _settings.py_ still contains the `LOGGING` settings that will actually send stuff to the console: [role="sourcecode currentcontents"] .src/superlists/settings.py ==== [source,python] ---- LOGGING = { "version": 1, "disable_existing_loggers": False, "handlers": { "console": {"class": "logging.StreamHandler"}, }, "loggers": { "root": {"handlers": ["console"], "level": "INFO"}, }, } ---- ==== Rebuild and restart the Docker container if necessary, and then either rerun the FT, or just try to log in manually. ******************************************************************************* If you switch to the terminal that's running your Docker image, you should see the traceback printed out in there: [role="skipme"] [subs="specialcharacters,macros"] ---- Internal Server Error: /accounts/send_login_email Traceback (most recent call last): [...] File "/src/accounts/views.py", line 16, in send_login_email send_mail( ~~~~~~~~~^ "Your login link for Superlists", ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<2 lines>... [email], ^^^^^^^^ ) ^ [...] self.connection.sendmail( ~~~~~~~~~~~~~~~~~~~~~~~~^ from_email, recipients, message.as_bytes(linesep="\r\n") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/usr/local/lib/python3.14/smtplib.py", line 876, in sendmail raise SMTPSenderRefused(code, resp, from_addr) smtplib.SMTPSenderRefused: (530, b'5.7.0 Authentication Required. [...] ---- Sure enough, that looks like a pretty good clue as to what's going on: we're getting a "sender refused" error when trying to send our email. Good to know our local Docker setup can reproduce the error on the server! ((("", startref="Dockercatch21"))) === Another Environment Variable in Docker So, Gmail is refusing to let us send emails, is it?((("environment variables", "email password in Docker")))((("Docker", "adding email password environment variable to local container"))) Now why might that be? "Authentication required", you say? Oh, whoops; we haven't told the server what our password is! As you might remember from earlier chapters, our _settings.py_ expects to get the email server password from an environment variable named `EMAIL_PASSWORD`: [role="sourcecode currentcontents"] .src/superlists/settings.py ==== [source,python] ---- EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_PASSWORD") ---- ==== Let's add this new environment variable to our local Docker container `run` command. First, set your email password in your terminal if you need to: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *echo $EMAIL_PASSWORD* # if that's empty, let's set it: $ *export EMAIL_PASSWORD="yoursekritpasswordhere"* ---- Now let's pass that environment variable through to our Docker container using one more `-e` flag—this one fishing the env var out of the shell we're in: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/container.db.sqlite3",target=/home/nonroot/db.sqlite3 \ -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -e DJANGO_DB_PATH=/home/nonroot/db.sqlite3 \ -e EMAIL_PASSWORD \ -it superlists* ---- TIP: If you use `-e` without the `=something` argument, it sets the env var inside Docker to the same value set in the current shell. It's like saying `-e EMAIL_PASSWORD=$EMAIL_PASSWORD`. [role="pagebreak-before"] And now we can rerun our FT again. We'll narrow it down to just the `test_login` test, because that's the main one that has a problem: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests.test_login*] [...] ERROR: test_login_using_magic_link (functional_tests.test_login.LoginTest.test_login_using_magic_link) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/functional_tests/test_login.py", line 32, in test_login_using_magic_link email = mail.outbox.pop() IndexError: pop from empty list ---- Well, not a pass, but the tests do get a little further. It looks like our server _can_ now send emails.((("mail.outbox attribute", "not working outside of Django"))) (If you check the Docker logs, you'll see there are no more errors.) But our FT is saying it can't see any emails appearing in `mail.outbox`. ==== mail.outbox Won't Work Outside Django's Test Environment The reason is that `mail.outbox` is a local, in-memory variable in Django, so that's only going to work when our tests and our server are running in the same process—like they do with unit tests or with `LiveServerTestCase` FTs. When we run against another process, be it Docker or an actual server, we can't access the same `mail.outbox` variable. If we want to actually inspect the emails that the server sends we need another technique in our tests against Docker (or later, against the staging server). [[options-for-testing-real-email]] === Deciding How to Test "Real" Email Sending This is a point at which we have to explore some trade-offs.((("emails", "testing real email sending", id="ix_emltstreal"))) There are a few different ways we could test email sending: 1. We could build a "real" end-to-end test, and have our tests log in to an email server using the POP3 protocol to retrieve the email from there. That's what I did in the first and second editions of this book. 2. We can use a service like Mailinator or Mailsac, which gives us an email account to send to, along with APIs for checking what mail has been delivered. 3. We can use an alternative, fake email backend whereby Django will save the emails to a https://docs.djangoproject.com/en/5.2/topics/email/#file-backend[file on disk], for example, and we can inspect them there. 4. We could give up on testing email on the server. If we have a minimal smoke test confirming that the server _can_ send emails, then we don't need to test that they are actually delivered. [role="pagebreak-before"] <<testing_strategy_table>> lays out some of the pros and cons. [[testing_strategy_table]] .Testing strategy trade-offs [options="header"] |======= | Strategy | Pros | Cons | End-to-end with POP3 | Maximally realistic, tests the whole system | Slow, fiddly, unreliable | Email testing service e.g., Mailinator/Mailsac| As realistic as real POP3, with better APIs for testing| Slow, possibly expensive (and I don't want to endorse any particular commercial provider) | File-based fake email backend | Faster, more reliable, no network calls, tests end-to-end (albeit with fake components) | Still fiddly, requires managing database and filesystem side effects | Giving up on testing email on the server/Docker | Fast, simple | Less confidence that things work "for real" |======= We're exploring a common problem in testing integration with external systems; how far should we go? How realistic should we make our tests? In this case, I'm going to suggest we go for the last option, which is _not_ to test email sending on the server or in Docker. Email itself is a well-understood protocol (reader, it's been around since _before I was born_, and that's a while ago now), and Django has supported sending email for more than a decade. So, I think we can afford to say, in this case, that the costs of building testing tools for email outweigh the benefits. // RITA: Although the sentence has a lot of your voice, I don't think mentioning your birthday is necessary. The reader probably has no idea how old you are. It would be enough to say that email has been around for a while now. I'm going to suggest we stick to using `mail.outbox` when we're running local tests, and we configure our FTs to just check that Docker (or, later, the staging server) _seems_ to be able to send email (in the sense of "not crashing"). We can skip the bit where we check the email contents in our FT. Remember, we also have unit tests for the email content! NOTE: I explore some of the difficulties involved in getting these kinds of tests to work in https://www.obeythetestinggoat.com/book/appendix_fts_for_external_dependencies.html[Online Appendix: Functional Tests for External Dependencies], so go check that out if this feels like a bit of a cop-out! [role="pagebreak-before"] Here's where we can put an((("early return"))) early return in the FT: [role="sourcecode"] .src/functional_tests/test_login.py (ch23l009) ==== [source,python] ---- # A message appears telling her an email has been sent self.wait_for( lambda: self.assertIn( "Check your email", self.browser.find_element(By.CSS_SELECTOR, "body").text, ) ) if self.test_server: # Testing real email sending from the server is not worth it. return # She checks her email and finds a message email = mail.outbox.pop() ---- ==== This test will still fail if you don't set `EMAIL_PASSWORD` to a valid value in Docker or on the server, meaning it would still have warned us of the bug we started the chapter with—so that's good enough for now. Here's how we populate the `FunctionalTest.test_server` attribute: [role="sourcecode"] .src/functional_tests/base.py (ch23l010) ==== [source,python] ---- class FunctionalTest(StaticLiveServerTestCase): def setUp(self): self.browser = webdriver.Firefox() self.test_server = os.environ.get("TEST_SERVER") # <1> if self.test_server: self.live_server_url = "http://" + self.test_server ---- ==== <1> We upgrade `test_server` to be an attribute on the test object, so we can access it in various places in our FTs (we'll see several examples later). Sad to see our walrus go, though! And you can confirm that the FT fails if you _don't_ set `EMAIL_PASSWORD` in Docker, or passes, if you do: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests.test_login*] [...] OK ---- Now let's see if we can get our FTs to pass against the server. First, we'll need to figure out how to set that env var on the server.((("emails", "testing real email sending", startref="ix_emltstreal"))) === An Alternative Method for Setting Secret Environment Variables on the Server ((("environment variables", "secret, alternative method for setting on server", id="ix_envvarset")))((("secrets", "setting secret environment variables on server", id="ix_secrenvvar"))) ((("secret values"))) In <<chapter_12_ansible>>, we dealt with setting the `SECRET_KEY` by generating a random value, and then saving it to a file on the server. We could use a similar technique here. But, just to give you an alternative, I'll show how to pass the environment variable directly up to the container, without storing it in a file: [role="sourcecode"] .infra/deploy-playbook.yaml (ch23l012) ==== [source,python] ---- env: DJANGO_DEBUG_FALSE: "1" DJANGO_SECRET_KEY: "{{ secret_key.content | b64decode }}" DJANGO_ALLOWED_HOST: "{{ inventory_hostname }}" DJANGO_DB_PATH: "/home/nonroot/db.sqlite3" EMAIL_PASSWORD: "{{ lookup('env', 'EMAIL_PASSWORD') }}" # <1> ---- ==== <1> `lookup()` with `env` as its argument is how you look up _local_ environment variables—i.e., the ones set on the computer you're running Ansible from. This means you'll need the `EMAIL_PASSWORD` environment variable to be set on your local machine every time you want to run Ansible. Let's consider some pros and cons of the two approaches: * Saving the secret to a file on the server means you don't need to "remember" or store the secret anywhere on your own machine. * In contrast, always passing it up from the local environment does mean you can change the value of the secret at any time. * In terms of security, they are fairly equivalent—in either case, the environment variable is visible via `docker inspect`. If we rerun our full FT suite against the server, you should see that the login test passes, and we're down to just one failure, in +test_⁠l⁠o⁠g⁠g⁠e⁠d​_⁠i⁠n⁠_⁠u⁠s⁠e⁠r⁠s⁠_lists_are_saved_as_my_lists()+: [role="skipme small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=staging.ottg.co.uk python src/manage.py test functional_tests*] [...] ERROR: test_logged_in_users_lists_are_saved_as_my_lists (functional_tests.test_my_lists.MyListsTest.test_logged_in_users_lists_are_saved_[...] ---------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/src/functional_tests/test_my_lists.py", line 36, in test_logged_in_users_lists_are_saved_as_my_lists self.wait_to_be_logged_in(email) ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^ [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: #id_logout; [...] [...] --------------------------------------------------------------------- Ran 8 tests in 30.087s FAILED (errors=1) ---- Let's look into that next.((("secrets", "setting secret environment variables on server", startref="ix_secrenvvar")))((("environment variables", "secret, alternative method for setting on server", startref="ix_envvarset"))) === Debugging with SQL Let's switch back to testing locally against our Docker container:((("SQL", "debugging creation of pre-authenticated sessions with", id="ix_SQLdbg"))) [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests.test_my_lists*] [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: #id_logout; [...] FAILED (errors=1) ---- It looks like the attempt to create pre-authenticated sessions doesn't work, so we're not being logged in. Let's do a bit of debugging with SQL. First, try logging in to your local "runserver" instance (where things definitely work) and take a look in the normal local database, _src/db.sqlite3_: [role="skipme"] [subs="specialcharacters,macros,callouts"] ---- $ pass:[<strong>sqlite3 src/db.sqlite3</strong>] SQLite version 3.43.2 2023-10-10 13:08:14 Enter ".help" for usage hints. sqlite> pass:[<strong>select * from accounts_token;</strong>] <1> pass:[1|obeythetestinggoat@gmail.com|11d3e26d-32a3-4434-af71-5e0f62fefc52] pass:[2|obeythetestinggoat@gmail.com|25a570c8-736f-42e4-931b-ed5c410b5b51] sqlite> pass:[<strong>select * from django_session;</strong>] <2> tv2m5byccfs05gfpkc1l8k4pep097y3c|.eJxVjEsKg0AMQO-StcwBurI9gTcYYgwzo[...] ---- <1> We can do a `SELECT *` in our tokens table to see some of the tokens we've been creating for our users. <2> And we can take a look in the `django_session` table. You should find the first column matches the session ID you'll see in your DevTools. Let's do a bit of debugging. Take a look in _container.db.sqlite3_: [role="skipme"] [subs="specialcharacters,macros,callouts"] ---- $ pass:[<strong>sqlite3 container.db.sqlite3</strong>] SQLite version 3.43.2 2023-10-10 13:08:14 Enter ".help" for usage hints. sqlite> pass:[<strong>select * from accounts_token;</strong>] <1> sqlite> pass:[<strong>select * from django_session;</strong>] <2> ---- <1> The users table is empty. (If you do see `edith@example.com` in here, it's from a previous test run. Delete and re-create the database if you want to be sure.) <2> And the sessions table is definitely empty. Now, let's try manually. If you visit `localhost:8888` and log in—getting the token from your email—you'll see it works. You can also run +f⁠u⁠n⁠c⁠t⁠i⁠o⁠n⁠a⁠l⁠_​t⁠e⁠s⁠t⁠s⁠.test_login+ and you'll see _that_ pass. If we look in the database again, we'll see some more data: [role="skipme"] [subs="specialcharacters,macros,callouts"] ---- $ pass:[<strong>sqlite3 container.db.sqlite3</strong>] SQLite version 3.43.2 2023-10-10 13:08:14 Enter ".help" for usage hints. sqlite> pass:[<strong>select * from accounts_token;</strong>] 3|obeythetestinggoat@gmail.com|115812a3-7d37-485c-9c15-337b12293f69 4|edith@example.com|a901bee9-88aa-4965-9277-a13723a6bfe1 sqlite> pass:[<strong>select * from django_session;</strong>] 09df51nmvpi137mpv5bwjoghh2a4y5lh|.eJxVjEsKg0AMQO-[...] ---- So, there's nothing _fundamentally_ wrong with the Docker environment. It's seems like it's specifically our test utility function `create_pre_authenticated_session()` that isn't working. At this point, a little niggle in your head might be growing louder, reminding us of a problem we anticipated in the last chapter: `LiveServerTestCase` only lets us talk to the in-memory database. That's where our pre-authenticated sessions are ending up!((("SQL", "debugging creation of pre-authenticated sessions with", startref="ix_SQLdbg"))) === Managing Fixtures in Real Databases We need a way to make changes to the database inside Docker or on the server. Essentially, we want to run some code outside the context of the tests (and the test database) and in the context of the server and its database.((("fixtures", "managing in real databases", id="ix_fxtDB"))) ==== A Django Management Command to Create Sessions ((("scripts, building standalone")))((("sessions", "Django management command to create")))((("management command (Django) to create sessions"))) When trying to build a standalone script that works with Django (i.e., can talk to the database and so on), there are some fiddly issues you need to get right, like setting the `DJANGO_SETTINGS_MODULE` environment variable and setting `sys.path` correctly. Instead of messing about with all that, Django lets you create your own "management commands" (commands you can run with `python manage.py`), which will do all that path-mangling for you. They live in a folder called _management/commands_ inside your apps: [subs=""] ---- $ <strong>mkdir -p src/functional_tests/management/commands</strong> $ <strong>touch src/functional_tests/management/__init__.py</strong> $ <strong>touch src/functional_tests/management/commands/__init__.py</strong> ---- The boilerplate in a management command is a class that inherits from `django.core.management.BaseCommand`, and that defines a method called `handle`: [role="sourcecode"] .src/functional_tests/management/commands/create_session.py (ch23l014) ==== [source,python] ---- from django.conf import settings from django.contrib.auth import BACKEND_SESSION_KEY, SESSION_KEY, get_user_model from django.contrib.sessions.backends.db import SessionStore from django.core.management.base import BaseCommand User = get_user_model() class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("email") def handle(self, *args, **options): session_key = create_pre_authenticated_session(options["email"]) self.stdout.write(session_key) def create_pre_authenticated_session(email): user = User.objects.create(email=email) session = SessionStore() session[SESSION_KEY] = user.pk session[BACKEND_SESSION_KEY] = settings.AUTHENTICATION_BACKENDS[0] session.save() return session.session_key ---- ==== We've taken the code for `create_pre_authenticated_session` from 'test_my_lists.py'. `handle` will pick up an email address from the parser, and then return the session key that we'll want to add to our browser cookies, and the management command prints it out at the command line. [role="pagebreak-before"] Try it out: [role="ignore-errors"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py create_session a@b.com*] Unknown command: 'create_session'. Did you mean clearsessions? ---- One more step: we need to add `functional_tests` to our 'settings.py' so that it's recognised as a real app that might have management commands as well as tests: [role="sourcecode"] .src/superlists/settings.py (ch23l015) ==== [source,python] ---- +++ b/superlists/settings.py @@ -42,6 +42,7 @@ INSTALLED_APPS = [ "accounts", "lists", + "functional_tests", ] ---- ==== WARNING: Beware of the security implications here. We're now adding some remotely executable code for bypassing authentication to our default configuration. Yes, someone exploiting this would need to have already gained access to the server, so it was game over anyway, but nonetheless, this is a sensitive area. If you were doing something like this in a real application, you might consider adding an `if environment != prod`, or similar. Now it works: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py create_session a@b.com*] qnslckvp2aga7tm6xuivyb0ob1akzzwl ---- NOTE: If you see an error saying the `auth_user` table is missing, you may need to run `manage.py migrate`. In case that doesn't work, delete the _db.sqlite3_ file and run `migrate` again to get a clean slate. [role="pagebreak-before less_space"] ==== Getting the FT to Run the Management Command on the Server Next, we need to adjust `test_my_lists` so that it runs the local function when we're using the local in-memory test server from `LiveServerTestCase`. And, if we're running against the Docker container or staging server, it should run the management command instead.((("management command (Django) to create sessions", "getting it to run on server"))) [role="sourcecode"] .src/functional_tests/test_my_lists.py (ch23l016) ==== [source,python] ---- from django.conf import settings from .base import FunctionalTest from .container_commands import create_session_on_server # <1> from .management.commands.create_session import create_pre_authenticated_session class MyListsTest(FunctionalTest): def create_pre_authenticated_session(self, email): if self.test_server: # <2> session_key = create_session_on_server(self.test_server, email) else: session_key = create_pre_authenticated_session(email) ## to set a cookie we need to first visit the domain. ## 404 pages load the quickest! self.browser.get(self.live_server_url + "/404_no_such_url/") self.browser.add_cookie( dict( name=settings.SESSION_COOKIE_NAME, value=session_key, path="/", ) ) [...] ---- ==== <1> Programming by wishful thinking, let's imagine we'll have a module called `container_commands` with a function called `create_session_on_server()` in it. <2> Here's the `if` where we decide which of our two session-creation functions to execute. ==== Running Commands Using Docker Exec and (Optionally) SSH You may remember `docker exec` from <<chapter_09_docker>>; it lets us run commands inside a running Docker container. That's fine for when we're running against the local Docker, but when we're against the server, we need to SSH in first.((("Docker", "running commands using docker exec")))((("SSH", "running commands on Docker container running on the server"))) There's a bit of plumbing here, but I've tried to break things down into small chunks: [role="sourcecode"] .src/functional_tests/container_commands.py (ch23l018) ==== [source,python] ---- import subprocess USER = "elspeth" def create_session_on_server(host, email): return _exec_in_container( host, ["/venv/bin/python", "/src/manage.py", "create_session", email] # <1> ) def _exec_in_container(host, commands): if "localhost" in host: # <2> return _exec_in_container_locally(commands) else: return _exec_in_container_on_server(host, commands) def _exec_in_container_locally(commands): print(f"Running {commands} on inside local docker container") return _run_commands(["docker", "exec", _get_container_id()] + commands) # <3> def _exec_in_container_on_server(host, commands): print(f"Running {commands!r} on {host} inside docker container") return _run_commands( ["ssh", f"{USER}@{host}", "docker", "exec", "superlists"] + commands # <4> ) def _get_container_id(): return subprocess.check_output( # <5> ["docker", "ps", "-q", "--filter", "ancestor=superlists"] # <3> ).strip() def _run_commands(commands): process = subprocess.run( # <5> commands, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False, ) result = process.stdout.decode() if process.returncode != 0: raise Exception(result) print(f"Result: {result!r}") return result.strip() ---- ==== // DAVID: In _run_commands, why not do `check=True`, then we can omit the returncode / exception handling? <1> We invoke our management command with the path to the virtualenv Python, the `create_session` command name, and pass in the email we want to create a session for. <2> We dispatch to two slightly different ways of running a command inside a container, with the assumption that a host on "localhost" is a local Docker container, and the others are on the staging server. <3> To run a command on the local Docker container, we're going to use `docker exec`, and we have a little extra hop first to get the correct container ID. <4> To run a command on the Docker container that's on the staging server, we still use `docker exec`, but we do it inside an SSH session. In this case we don't need the container ID, because the container is always named "superlists". <5> Finally, we use Python's `subprocess` module to actually run a command. You can see a couple of different ways of running it here, which differ based on how we're handing errors and output; the details don't matter too much. ==== Recap: Creating Sessions Locally Versus Staging ((("staging sites", "local versus staged sessions")))((("sessions", "creating locally versus staging"))) Does that all make sense? Perhaps a little ASCII-art diagram will help: ===== Locally: [role="skipme small-code"] ---- +-----------------------------------+ +-------------------------------------+ | MyListsTest | | .management.commands.create_session | | .create_pre_authenticated_session | --> | .create_pre_authenticated_session | | (locally) | | (locally) | +-----------------------------------+ +-------------------------------------+ ---- ===== Against Docker locally: [role="skipme small-code"] ---- +-----------------------------------+ +-------------------------------------+ | MyListsTest | | .management.commands.create_session | | .create_pre_authenticated_session | | .create_pre_authenticated_session | | (locally) | | (in Docker) | +-----------------------------------+ +-------------------------------------+ | ^ v | +----------------------------+ | | server_tools | +-------------+ +----------------------------+ | .create_session_on_server | --> | docker exec | --> | ./manage.py create_session | | (locally) | +-------------+ | (in Docker) | +----------------------------+ +----------------------------+ ---- ===== Against Docker on the server: [role="skipme small-code"] ---- +-----------------------------------+ +-------------------------------------+ | MyListsTest | | .management.commands.create_session | | .create_pre_authenticated_session | | .create_pre_authenticated_session | | (locally) | | (on server) | +-----------------------------------+ +-------------------------------------+ | ^ v | +----------------------------+ | | server_tools | +-----+ +--------+ +----------------------------+ | .create_session_on_server | -> | ssh | -> | docker | -> | ./manage.py create_session | | (locally) | | | | exec | | (on server) | +----------------------------+ +-----+ +--------+ +----------------------------+ ---- We do love a bit of ASCII art now and again! .An Alternative for Managing Test Database Content: Talking Directly to the Database ********************************************************************** An alternative way of managing database content inside Docker, or on a server, would be to talk directly to the database.((("databases", "alternative for managing test database content"))) Because we're using SQLite, that involves writing to the file directly. This can be fiddly to get right, because when we're running inside Django's test runner, Django takes over the test database creation, so you end up having to write raw SQL and manage your connections to the database directly. There are also some tricky interactions with the filesystem mounts and Docker, as well as the need to have the `SECRET_KEY` env var set to the same value as on the server. If we were using a "classic" database server like PostgreSQL or MySQL, we'd be able to talk directly to the database over its port, and that's an approach https://oreil.ly/Uk1y_[I've used successfully in the past] but it's still quite tricky, and usually requires writing your own SQL. ********************************************************************** === Testing the Management Command In any case, let's see if this whole rickety pipeline works.((("fixtures", "managing in real databases", startref="ix_fxtDB")))((("management command (Django) to create sessions", "testing the command"))) First, locally, to check that we didn't break anything: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_my_lists*] [...] OK ---- [role="pagebreak-before"] Next, against Docker—rebuild first: [role="small-code"] [subs="specialcharacters,quotes"] ---- $ *docker build -t superlists . && docker run \ -p 8888:8888 \ --mount type=bind,source="$PWD/container.db.sqlite3",target=/home/nonroot/db.sqlite3 \ -e DJANGO_SECRET_KEY=sekrit \ -e DJANGO_ALLOWED_HOST=localhost \ -e DJANGO_DB_PATH=/home/nonroot/db.sqlite3 \ -e EMAIL_PASSWORD \ -it superlists* ---- And then we run the FT (that uses our fixture) against Docker: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests.test_my_lists*] [...] OK ---- Next, we run it against the server. First, we re-deploy to make sure our code on the server is up to date: [role="against-server small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*ansible-playbook --user=elspeth -i staging.ottg.co.uk, infra/deploy-playbook.yaml -vv*] ---- // CSANAD: at some point I deleted my .venv and reinstalled the pip packages on // my dedicated book-development environment. I just noticed I forgot about // installing ansible. Just a thought, maybe we could mention in a footnote, // perhaps in chapter 11 (after installing ansible), that it's a common practice // to create a separate requirements-dev.txt and we could list selenium, ansible // and requests in ours. And now we run the test: [role="against-server small-code"] [subs=""] ---- $ <strong>TEST_SERVER=staging.ottg.co.uk python src/manage.py test \ functional_tests.test_my_lists</strong> Found 1 test(s). Creating test database for alias 'default'... System check identified no issues (0 silenced). Running '/venv/bin/python /src/manage.py create_session edith@example.com' on staging.ottg.co.uk inside docker container Result: '7n032ogf179t2e7z3olv9ct7b3d4dmas\n' . --------------------------------------------------------------------- Ran 1 test in 4.515s OK Destroying test database for alias 'default'... ---- Looking good! We can rerun all the tests to make sure... [role="against-server small-code"] [subs=""] ---- $ <strong>TEST_SERVER=staging.ottg.co.uk python src/manage.py test functional_tests</strong> [...] [elspeth@staging.ottg.co.uk] run: ~/sites/staging.ottg.co.uk/.venv/bin/python [...] Ran 8 tests in 89.494s OK ---- Hooray! [role="pagebreak-before less_space"] === Test Database Cleanup One more thing to be aware of: now that we're running against a real database, we don't get cleanup for free any more.((("database testing", "test database cleanup", id="ix_DBtstcln"))) If you try running the tests twice—locally or against Docker—you'll run into this error: [role="small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests.test_my_lists*] [...] django.db.utils.IntegrityError: UNIQUE constraint failed: accounts_user.email ---- It's because the user we created the first time we ran the tests is still in the database. When we're running against Django's test database, Django cleans up for us. Let's try and emulate that when we're running against a real database: [role="sourcecode"] .src/functional_tests/container_commands.py (ch23l019) ==== [source,python] ---- def reset_database(host): return _exec_in_container( host, ["/venv/bin/python", "/src/manage.py", "flush", "--noinput"] ) ---- ==== And let's add the call to `reset_database()` in our base test `setUp()` method: [role="sourcecode"] .src/functional_tests/base.py (ch23l020) ==== [source,python] ---- from .container_commands import reset_database [...] class FunctionalTest(StaticLiveServerTestCase): def setUp(self): self.browser = webdriver.Firefox() self.test_server = os.environ.get("TEST_SERVER") if self.test_server: self.live_server_url = "http://" + self.test_server reset_database(self.test_server) ---- ==== If you try to run your tests again, you'll find they pass happily: [role="dofirst-ch23l021 small-code"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*TEST_SERVER=localhost:8888 python src/manage.py test functional_tests.test_my_lists*] [...] OK ---- Probably a good time for a commit! :)((("database testing", "test database cleanup", startref="ix_DBtstcln"))) [role="pagebreak-before less_space"] .Warning: Be Careful Not to Run Test Code Against the Production Server! ******************************************************************************* ((("database testing", "safeguarding production databases"))) ((("production databases"))) We're in dangerous territory now that we have code that can directly affect a database on the server. You want to be very, very careful that you don't accidentally blow away your production database by running FTs against the wrong host. You might consider putting some safeguards in place at this point. You almost definitely want to put staging and production on different servers, for example, and make it so that they use different key pairs for authentication, with different passphrases. I also mentioned not including the FT management commands in `INSTALLED_APPS` for production environments. This is similarly dangerous territory to running tests against clones of production data. I could tell you a little story about accidentally sending thousands of duplicate invoices to clients, for example. LFMF! And tread carefully. ******************************************************************************* === Wrap-Up Actually getting your new code up and running on a server always tends to flush out some last-minute bugs and unexpected issues. We had to do a bit of work to get through them, but we've ended up with several useful things as a result. We now have a lovely generic `wait` decorator, which will be a nice Pythonic helper for our FTs from now on. We've got some more robust logging configuration. We have test fixtures that work both locally and on the server, and we've come out with a pragmatic approach for testing email integration. But before we can deploy our actual production site, we'd better actually give the users what they wanted--the next chapter describes how to give them the ability to save their lists on a "My lists" page.((("debugging", "catching bugs in staging"))) [role="pagebreak-before less_space"] .Lessons Learned Catching Bugs in Staging ******************************************************************************* It's nice to be able to repro things locally.:: The effort we put into adapting our app to use Docker is paying off. We discovered an issue in staging, and were able to reproduce it locally. That gives us the ability to experiment and get feedback much quicker than trying to do experiments on the server itself. Fixtures also have to work remotely.:: `LiveServerTestCase` makes it easy to interact with the test database using the Django ORM for tests running locally. Interacting with the database inside Docker is not so straightforward. One solution is `docker exec` and Django management commands, as I've shown, but you should explore what works for you--connecting directly to the database over SSH tunnels, for example. ((("fixtures", "staging and"))) ((("staging sites", "fixtures and"))) Be very careful when resetting data on your servers.:: A command that can remotely wipe the entire database on one of your servers is a dangerous weapon, and you want to be really, really sure it's never accidentally going to hit your production data. ((("database testing", "safeguarding production databases"))) ((("production databases"))) Logging is critical to debugging issues on the server.:: At the very least, you'll want to be able to see any error messages that are being generated by the server. For thornier bugs, you'll also want to be able to do the occasional "debug print", and see it end up in a file somewhere. ((("logging"))) ((("debugging", "server-side", "baking in logging code"))) ******************************************************************************* ================================================ FILE: chapter_24_outside_in.asciidoc ================================================ [[chapter_24_outside_in]] == Finishing "My Lists": Outside-In TDD ((("Test-Driven Development (TDD)", "outside-in technique", id="TTDoutside22"))) In this chapter, I'd like to talk about a technique called outside-in TDD. It's pretty much what we've been doing all along. Our "double-loop" TDD process, in which we write the functional test first and then the unit tests, is already a manifestation of outside-in--we design the system from the outside, and build up our code in layers. Now I'll make it explicit, and talk about some of the common issues involved. === The Alternative: Inside-Out The alternative to "outside-in" is to work "inside-out", which is the way most people intuitively work before they encounter TDD.((("inside-out TDD"))) After coming up with a design, the natural inclination is sometimes to implement it starting with the innermost, lowest-level components first. For example, when faced with our current problem, providing users with a "My lists" page of saved lists, the temptation is to start at the models layer: we probably want to add an "owner" attribute to the `List` model object, reasoning that an attribute like this is "obviously" going to be required. Once that's in place, we would modify the more peripheral layers of code—such as views and templates—taking advantage of the new attribute, and then finally add URL routing to point to the new view. It feels comfortable because it means you're never working on a bit of code that is dependent on something that hasn't yet been implemented. Each bit of work on the inside is a solid foundation on which to build the next layer out. But working inside-out like this also has some weaknesses. === Why Prefer "Outside-In"? ((("outside-in TDD", "versus inside-out", secondary-sortas="inside-out"))) ((("inside-out TDD", "versus outside-in"))) The most obvious problem with inside-out TDD is that it requires us to stray from a TDD workflow. Our functional test's first failure might be due to missing URL routing, but we decide to ignore that and go off adding attributes to our database model objects instead. We might have ideas in our head about the desired behaviour of our inner layers like database models, and often these ideas will be pretty good—but they are actually just speculation about what's really required, because we haven't yet built the outer layers that will use them. One problem that can occur is building inner components that are more general or more capable than we actually need, which is a waste of time and an added source of complexity for your project. Another common problem is that you create inner components with an API that is convenient for their own internal design, but which later turns out to be inappropriate for the calls that your outer layers would like to make...worse still, you might end up with inner components which, you later realise, don't actually solve the problem that your outer layers need solved. In contrast, working outside-in enables you to use each layer to imagine the most convenient API you could want from the layer beneath it. Let's see it in action. === The FT for "My Lists" ((("functional tests (FTs)", "outside-in technique", id="ix_FToutin"))) As we work through the following functional test, we start with the most outward-facing (presentation layer), through to the view functions (or "controllers"), and lastly the innermost layers, which in this case will be model code. See <<outside-in-layers>>. [[outside-in-layers]] .The layer in our application image::images/tdd3_2401.png["A diagram of the layers of the application, with the user at the top, then a presentation layer (aka views in mvc) containing some templates, below that the views layer (aka controllers in mvc) containg views and forms, then the models layer with django models, and finally an icon for the database at the bottom. there is also an arrow from top to bottom, with the top labelled OUTSIDE! and the bottom labelled IN!"] While we're drawing diagrams, would it help to sketch out what we're imagining? See <<my-lists-page-sketch>>. [[my-lists-page-sketch]] .A sketch of the "My lists" page image::images/tdd3_2402.png["A sketch of the My Lists page, showing the users email in the page header, and several lists with a callout saying that we will use the first item text as the hyperlink text for each list."] [role="pagebreak-before"] Let's incarnate this idea in FT form. We know our `create_pre_authenticated_session` code works now, so we can just fill out the actual body of the test to describe how a user might interact with this prospective "My lists" page: [role="sourcecode"] .src/functional_tests/test_my_lists.py (ch24l001) ==== [source,python] ---- from selenium.webdriver.common.by import By [...] def test_logged_in_users_lists_are_saved_as_my_lists(self): # Edith is a logged-in user self.create_pre_authenticated_session("edith@example.com") # She goes to the home page and starts a list self.browser.get(self.live_server_url) self.add_list_item("Reticulate splines") # <1> self.add_list_item("Immanentize eschaton") first_list_url = self.browser.current_url # She notices a "My lists" link, for the first time. self.browser.find_element(By.LINK_TEXT, "My lists").click() # She sees her email is there in the page heading self.wait_for( lambda: self.assertIn( "edith@example.com", self.browser.find_element(By.CSS_SELECTOR, "h1").text, ) ) # And she sees that her list is in there, # named according to its first list item self.wait_for( lambda: self.browser.find_element(By.LINK_TEXT, "Reticulate splines") ) self.browser.find_element(By.LINK_TEXT, "Reticulate splines").click() self.wait_for( lambda: self.assertEqual(self.browser.current_url, first_list_url) ) ---- ==== <1> We'll define this `add_list_item()` shortly. As you can see, we create a list with a couple of items. Then, we check that this list appears on a new "My lists" page, and that it's "named" after the first item in the list. [role="pagebreak-before"] Let's validate that it really works by creating a second list, and seeing that appear on the "My lists" page as well. The FT continues, and while we're at it, we check that only logged-in users can see the "My lists" page: [role="sourcecode small-code"] .src/functional_tests/test_my_lists.py (ch24l002) ==== [source,python] ---- [...] self.wait_for( lambda: self.assertEqual(self.browser.current_url, first_list_url) ) # She decides to start another list, just to see self.browser.get(self.live_server_url) self.add_list_item("Click cows") second_list_url = self.browser.current_url # Under "my lists", her new list appears self.browser.find_element(By.LINK_TEXT, "My lists").click() self.wait_for(lambda: self.browser.find_element(By.LINK_TEXT, "Click cows")) self.browser.find_element(By.LINK_TEXT, "Click cows").click() self.wait_for( lambda: self.assertEqual(self.browser.current_url, second_list_url) ) # She logs out. The "My lists" option disappears self.browser.find_element(By.CSS_SELECTOR, "#id_logout").click() self.wait_for( lambda: self.assertEqual( self.browser.find_elements(By.LINK_TEXT, "My lists"), [], ) ) ---- ==== Our FT uses a new helper method, `add_list_item()`, which abstracts away the process of entering text into the right input box. We define it in _base.py_: [role="sourcecode small-code"] .src/functional_tests/base.py (ch24l003) ==== [source,python] ---- from selenium.webdriver.common.keys import Keys [...] def add_list_item(self, item_text): num_rows = len(self.browser.find_elements(By.CSS_SELECTOR, "#id_list_table tr")) self.get_item_input_box().send_keys(item_text) self.get_item_input_box().send_keys(Keys.ENTER) item_number = num_rows + 1 self.wait_for_row_in_list_table(f"{item_number}: {item_text}") ---- ==== And while we're at it, we can use it in a few of the other FTs—like this, for example: [role="sourcecode dofirst-ch24l004-1"] .src/functional_tests/test_layout_and_styling.py (ch24l004-2) ==== [source,diff] ---- # She starts a new list and sees the input is nicely # centered there too - inputbox.send_keys("testing") - inputbox.send_keys(Keys.ENTER) - self.wait_for_row_in_list_table("1: testing") + self.add_list_item("testing") + ---- ==== I think it makes the FTs a lot more readable. I made a total of six changes--see if you agree with me. Let's do a quick run of all FTs, a commit, and then back to the FT we're working on. The first error should look like this: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_my_lists*] [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: My lists; [...] ---- === The Outside Layer: Presentation and Templates ((("functional tests (FTs)", "outside-in technique", startref="ix_FToutin")))((("outside-in TDD", "outside layer"))) The test is currently failing because it can't find a link saying "My lists". We can address that at the presentation layer, in _base.html_, in our navigation bar. Here's the minimal code change: [role="sourcecode small-code"] .src/lists/templates/base.html (ch24l005) ==== [source,html] ---- <nav class="navbar"> <div class="container-fluid"> <a class="navbar-brand" href="/">Superlists</a> {% if user.email %} <a class="navbar-link" href="#">My lists</a> <span class="navbar-text">Logged in as {{ user.email }}</span> <form method="POST" action="{% url 'logout' %}"> [...] ---- ==== Of course the `href="#"` means that link doesn't actually go anywhere, but it _does_ get our FT along to the next failure: [subs=""] ---- $ <strong>python src/manage.py test functional_tests.test_my_lists</strong> [...] lambda: self.assertIn( ~~~~~~~~~~~~~^ "edith@example.com", ^^^^^^^^^^^^^^^^^^^^ self.browser.find_element(By.CSS_SELECTOR, "h1").text, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ AssertionError: 'edith@example.com' not found in 'Your To-Do list' ---- That is telling us that we're going to have to build a page that at least has the user's email in its header. Let's start with the basics--a URL and a placeholder template for it. Again, we can go outside-in, starting at the presentation layer with just the URL and nothing else: [role="sourcecode"] .src/lists/templates/base.html (ch24l006) ==== [source,html] ---- {% if user.email %} <a class="navbar-link" href="{% url 'my_lists' user.email %}">My lists</a> ---- ==== // TODO: mention urlencoding emails === Moving Down One Layer to View Functions (the Controller) ((("controller layer (outside-in TDD)")))((("outside-in TDD", "controller layer"))) That will cause a template error in the FT: [subs=""] ---- $ <strong>./src/manage.py test functional_tests.test_my_lists</strong> [...] Internal Server Error: / [...] File "...goat-book/src/lists/views.py", line 8, in home_page return render(request, "home.html", {"form": ItemForm()}) [...] django.urls.exceptions.NoReverseMatch: Reverse for 'my_lists' not found. 'my_lists' is not a valid view function or pattern name. [...] ERROR: test_logged_in_users_lists_are_saved_as_my_lists [...] [...] selenium.common.exceptions.NoSuchElementException: [...] ---- To fix it, we'll need to start moving from working at the presentation layer, gradually into the controller layer—Django's URLs and views. As always, we start with a test. In this layer, a unit test is the way to go: [role="sourcecode"] .src/lists/tests/test_views.py (ch24l007) ==== [source,python] ---- class MyListsTest(TestCase): def test_my_lists_url_renders_my_lists_template(self): response = self.client.get("/lists/users/a@b.com/") self.assertTemplateUsed(response, "my_lists.html") ---- ==== That gives: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test lists*] [...] AssertionError: No templates used to render the response ---- [role="pagebreak-before"] That's because the URL doesn't exist yet, and a 404 has no template. Let's start our fix in _urls.py_: [role="sourcecode"] .src/lists/urls.py (ch24l008) ==== [source,python] ---- urlpatterns = [ path("new", views.new_list, name="new_list"), path("<int:list_id>/", views.view_list, name="view_list"), path("users/<str:email>/", views.my_lists, name="my_lists"), ] ---- ==== That gives us a new test failure, which informs us of what we should do. As you can see, it's pointing us at a _views.py_. We're clearly in the controller layer: ---- path("users/<str:email>/", views.my_lists, name="my_lists"), ^^^^^^^^^^^^^^ AttributeError: module 'lists.views' has no attribute 'my_lists' ---- Let's create a minimal placeholder then: [role="sourcecode"] .src/lists/views.py (ch24l009) ==== [source,python] ---- def my_lists(request, email): return render(request, "my_lists.html") ---- ==== Let's also create a minimal template, with no real content except for the header that shows the user's email address: [role="sourcecode"] .src/lists/templates/my_lists.html (ch24l010) ==== [source,html] ---- {% extends 'base.html' %} {% block header_text %}{{ user.email }}'s Lists{% endblock %} ---- ==== That gets our unit tests passing: [subs="specialcharacters,quotes"] ---- $ *./src/manage.py test lists* [...] OK ---- And hopefully it will address the current error in our FT: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_my_lists*] [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: Reticulate splines; [...] ---- Step by step! Sure enough, the FT gets a little further. It can now find the email in the `<h1>`, but it's now saying that the "My lists" page doesn't yet show any lists. It wants them to appear as clickable links, named after the first item. === Another Pass, Outside-In ((("outside-in TDD", "FT-driven development", id="OITDDft22"))) ((("functional tests (FTs)", "FT-driven development, outside-in technique", id="ix_FToutin2")))At each stage, we're still letting the FT drive what development we do. Starting again at the outside layer, in the template, we begin to write the template code we'd like to use to get the "My lists" page to work the way we want it to. As we do so, we start to specify the API we want from the code at the layers below. // Programming by wishful thinking, as always! ==== A Quick Restructure of Our Template Composition ((("templates", "composition"))) Let's take a look at our base template, _base.html_. It currently has a lot of content that's specific to editing to-do lists, which our "My lists" page doesn't need: [role="sourcecode currentcontents small-code"] .src/lists/templates/base.html ==== [source,html] ---- <div class="container"> <nav class="navbar"> [...] </nav> {% if messages %} [...] {% endif %} <div class="row justify-content-center p-5 bg-body-tertiary rounded-3"> <div class="col-lg-6 text-center"> <h1 class="display-1 mb-4">{% block header_text %}{% endblock %}</h1> <form method="POST" action="{% block form_action %}{% endblock %}" > <1> [...] </form> </div> </div> <div class="row justify-content-center"> <div class="col-lg-6"> {% block table %} <2> {% endblock %} </div> </div> </div> <script src="/static/lists.js"></script> <3> [...] ---- ==== [role="pagebreak-before"] <1> The `<form>` tag is definitely something we only want on pages where we edit lists. Everything else up to this point is generic enough to be on any page. <2> Similarly, the `{% block table %}` isn't something we'd need on the "My lists" page. <3> Finally, the `<script>` tag is specific to lists too. So, we'll want to change things so that _base.html_ is a bit more generic. Let's recap. We've got three actual pages we want to render: 1. The home page (where you can enter a first to-do item to create a new list) 2. The "List" page (where you can view an existing list and add to it) 3. The "My lists" page (which is a list of all your existing lists) And the home page and list page both share the same "form" elements and the _lists.js_ JavaScript. But the "List" page is the only one that needs to show the full table of list items. The "My lists" page doesn't need anything related to editing or displaying lists. So, we have some things shared between all three, and some only shared between the first and second. So far, we've been using inheritance to share the common parts of our templates, but this is a good place to start using composition instead. At the moment, we're saying that "home" is a type of "base" template, but with the "table" section switched off, which is a bit awkward. Let's not make it even more awkward by saying that "list" is a "base" template with both the form and the table switched off! It might make more sense to say that "home" is a type of base template that includes a list form, but no table, and that "list" includes both the list form and the list table. TIP: People often say "prefer composition over inheritance",((("composition over inheritance principle"))) because inheritance can become hard to reason about as the inheritance hierarchy grows. Composition is more flexible and often makes more sense. For a lengthy discussion of this topic, see https://hynek.me/articles/python-subclassing-redux/[Hynek Schlawack's definitive article on subclassing in Python]. [role="pagebreak-before"] So, let's do the following: 1. Pull out the `<form>` tag and the _lists.js_ `<script>` tag into into some blocks we can "include" in our home page and lists page. 2. Move the `<table>` block so it only exists in the list page. 3. Take all the list-specific stuff out of the _base.html_ template, making it into a more generic page with a header and a placeholder for generic content. We'll use what's called an https://docs.djangoproject.com/en/5.2/ref/templates/builtins/#include[`include`] to compose reusable template fragments when we don't want to use inheritance. ==== An Early Return So We're Refactoring Against Green Before we start refactoring, let's put an early return in our FT, so we're refactoring against green tests: [role="sourcecode"] .src/functional_tests/test_my_lists.py (ch24l010-0) ==== [source,python] ---- # She sees her email is there in the page heading self.wait_for( lambda: self.assertIn( "edith@example.com", self.browser.find_element(By.CSS_SELECTOR, "h1").text, ) ) return # TODO: resume here after templates refactor # And she sees that her list is in there, # named according to its first list item [...] ---- ==== Verify the FTs are all green: ---- Ran 8 tests in 19.712s OK ---- [role="pagebreak-before less_space"] ==== Factoring Out Two Template includes First let's pull out the form and the script tag from _base.html_: [role="sourcecode small-code"] .src/lists/templates/base.html (ch24l010-1) ==== [source,diff] ---- @@ -58,43 +58,19 @@ <div class="col-lg-6 text-center"> <h1 class="display-1 mb-4">{% block header_text %}{% endblock %}</h1> - <form method="POST" action="{% block form_action %}{% endblock %}" > - {% csrf_token %} - <input - id="id_text" - name="text" - class="form-control - form-control-lg - {% if form.errors %}is-invalid{% endif %}" - placeholder="Enter a to-do item" - value="{{ form.text.value }}" - aria-describedby="id_text_feedback" - required - /> - {% if form.errors %} - <div id="id_text_feedback" class="invalid-feedback"> - {{ form.errors.text.0 }} - </div> - {% endif %} - </form> + {% block extra_header %} + {% endblock %} + </div> </div> - <div class="row justify-content-center"> - <div class="col-lg-6"> - {% block table %} - {% endblock %} - </div> - </div> + {% block content %} + {% endblock %} </div> - <script src="/static/lists.js"></script> - <script> - window.onload = () => { - initialize("#id_text"); - }; - </script> + {% block scripts %} + {% endblock %} </body> </html> ---- ==== [role="pagebreak-before"] You can see we've replaced all the list-specific stuff with three new blocks: . `extra_header` for anything we want to put in the big header section . `content` for the main content of the page . `scripts` for any JavaScript we want to include Let's paste the `<form>` tag into a file at _src/lists/templates/includes/form.html_ (having a subfolder in templates for includes is a common practice): [role="sourcecode small-code"] .src/lists/templates/includes/form.html (ch24l010-2) ==== [source,html] ---- <form method="POST" action="{{ form_action }}" > <1> {% csrf_token %} <input id="id_text" name="text" class="form-control form-control-lg {% if form.errors %}is-invalid{% endif %}" placeholder="Enter a to-do item" value="{{ form.text.value | default:'' }}" aria-describedby="id_text_feedback" required /> {% if form.errors %} <div id="id_text_feedback" class="invalid-feedback"> {{ form.errors.text.0 }} </div> {% endif %} </form> ---- ==== <1> This is the only change; we've replaced the `{% block form_action %}` with `{{ form_action }}`. Let's paste the ++script++ tags verbatim into a new file at _includes/scripts.html_: [role="sourcecode"] .src/lists/templates/includes/scripts.html (ch24l010-3) ==== [source,html] ---- <script src="/static/lists.js"></script> <script> window.onload = () => { initialize("#id_text"); }; </script> ---- ==== [role="pagebreak-before"] Now let's look at how to use the `include`, and how the `form_action` change plays out in the changes to _home.html_: [role="sourcecode small-code"] .src/lists/templates/home.html (ch24l010-4) ==== [source,html] ---- {% extends 'base.html' %} {% block header_text %}Start a new To-Do list{% endblock %} {% block extra_header %} {% url 'new_list' as form_action %} <1> {% include "includes/form.html" with form=form form_action=form_action %} <2> {% endblock %} {% block scripts %} <3> {% include "includes/scripts.html" %} {% endblock %} ---- ==== <1> The `{% url ... as %}` syntax lets us define a template variable inline. <2> Then we use `{% include ... with key=value... %}` to pull in the contents of the `form.html` template, with the appropriate context variables passed in--a bit like calling a function.footnote:[ Strictly speaking, you could have omitted the `with=` in this case, as included templates automatically get the context of their parent. But sometimes you want to pass a context variable under a different name, so I like the `with`, for consistency and explicitness.] <3> The `scripts` block is just a straightforward `include` with no variables. [role="pagebreak-before"] Now let's see it in _list.html_: [role="sourcecode"] .src/lists/templates/list.html (ch24l010-5) ==== [source,diff] ---- @@ -2,12 +2,24 @@ {% block header_text %}Your To-Do list{% endblock %} -{% block form_action %}{% url 'view_list' list.id %}{% endblock %} -{% block table %} +{% block extra_header %} <1> + {% url 'view_list' list.id as form_action %} + {% include "includes/form.html" with form=form form_action=form_action %} +{% endblock %} + +{% block content %} <2> +<div class="row justify-content-center"> + <div class="col-lg-6"> <table class="table" id="id_list_table"> {% for item in list.item_set.all %} <tr><td>{{ forloop.counter }}: {{ item.text }}</td></tr> {% endfor %} </table> + </div> +</div> +{% endblock %} + +{% block scripts %} <3> + {% include "includes/scripts.html" %} {% endblock %} ---- ==== <1> The `block table` becomes an `extra_header` block, and we use the `include` to pull in the form. <2> The `block table` becomes a `content` block, with all the HTML we need for our table. <3> And the `scripts` block is the same as the one from _home.html_. Now a little rerun of all our FTs to make sure we haven't broken anything: ---- Ran 8 tests in 19.712s OK ---- [role="pagebreak-before"] OK, let's remove the early return: [role="sourcecode"] .src/functional_tests/test_my_lists.py (ch24l010-6) ==== [source,diff] ---- @@ -44,7 +44,6 @@ class MyListsTest(FunctionalTest): self.browser.find_element(By.CSS_SELECTOR, "h1").text, ) ) - return # TODO: resume here after templates refactor # And she sees that her list is in there, # named according to its first list item ---- ==== // CSANAD something somewhere broke my styling tests, even though right now I work from the book-example // commit-to-commit (no manual changes, just `git checkout` the next commit). // DAVID: For some reason I got an extra failure in test_layout_and_styling. Running it again, it passed. // Is it possible it's flakey? (I appreciate this is a terribly vague bug report so feel free to ignore.) And we'll commit that as a nice refactor: [subs="specialcharacters,quotes"] ---- $ *git add src/lists/templates* $ *git commit -m "refactor templates to use composition/includes"* ---- Now let's get back to our outside-in process, and to working in our template to drive out the requirements for our views layer. ==== Designing Our API Using the Template With the early return removed, our FT is back to telling us that we need to actually show our lists—named after their first items—on the new "My lists" page: [subs="specialcharacters,quotes"] ---- $ *./src/manage.py test functional_tests.test_my_lists* [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: Reticulate splines; [...] ---- (If you haven't taken a look around the site recently, it does look pretty blank—see <<empty-my-lists-page>>.) [[empty-my-lists-page]] .Not much to see here image::images/tdd3_2403.png["A screenshot of the My Lists page, showing the user's email in the page header, but no lists."] ((("templates", "designing APIs using"))) So, in _my_lists.html_, we can now work in the `content` block: [role="sourcecode"] .src/lists/templates/my_lists.html (ch24l010-7) ==== [source,html] ---- [...] {% block content %} <h2>{{ owner.email }}'s lists</h2> <1> <ul> {% for list in owner.lists.all %} <2> <li><a href="{{ list.get_absolute_url }}">{{ list.name }}</a></li> <3> {% endfor %} </ul> {% endblock %} ---- ==== // TODO: look into changing the user.email at the top to owner.email as well // the trouble is that changing it at this point introduces a regression // in the FT. We've made several design decisions in this template that are going to filter their way down through the code: <1> We want a variable called `owner` to represent the user in our template. This is what will allow one user to view another user's lists. <2> We want to be able to iterate through the lists created by that user using `owner.lists.all`. (I happen to know how to make this work with the Django ORM.) <3> We want to use `list.name` to print out the "name" of the list, which is currently specified as the text of its first element. .Programming by Wishful Thinking Again, Still ******************************************************************************* The phrase "programming by wishful thinking" was first popularised by the amazing, mind-expanding textbook https://oreil.ly/5EZNI[Structure and Interpretation of Computer Programs (SICP)], which I _cannot_ recommend highly enough. In it, the authors use it as a way to think about and write code at a higher level of abstraction, without worrying about the details of a lower level that might not even exist yet. For them, it's a key tool for designing programs and managing complexity. We've been doing a lot of "programming by wishful thinking" in this book. We've talked about how TDD itself is a form of wishful thinking; our tests express that we _wish_ we had code that worked in such-and-such a way. Outside-in TDD is very much an extension of this philosophy. We start writing code at the higher levels based on what we _wish_ we had at the lower levels, even though it doesn't exist yet... YAGNI also comes into it. By driving our development from the outside in, each piece of code we write is only there because we know it's actually needed by a higher layer and, ultimately, by the user. ******************************************************************************* We can rerun our FTs to check that we didn't break anything, and to see whether we've gotten any further: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests*] [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: Reticulate splines; [...] --------------------------------------------------------------------- Ran 8 tests in 77.613s FAILED (errors=1) ---- Well, no further—but at least we didn't break anything. Time for a commit: [subs="specialcharacters,quotes"] ---- $ *git add src/lists* $ *git diff --staged* # urls+views.py, templates $ *git commit -m "url, placeholder view, and first-cut templates for my_lists"* ---- [role="pagebreak-before less_space"] ==== Moving Down to the Next Layer: What the View Passes to the Template ((("templates", "views layer and"))) Now our views layer needs to respond to the requirements we've laid out in the template layer, by giving it the objects it needs—in this case, the list owner: [role="sourcecode"] .src/lists/tests/test_views.py (ch24l011) ==== [source,python] ---- from accounts.models import User [...] class MyListsTest(TestCase): def test_my_lists_url_renders_my_lists_template(self): [...] def test_passes_correct_owner_to_template(self): User.objects.create(email="wrong@owner.com") correct_user = User.objects.create(email="a@b.com") response = self.client.get("/lists/users/a@b.com/") self.assertEqual(response.context["owner"], correct_user) ---- ==== That gives: ---- KeyError: 'owner' ---- So: [role="sourcecode"] .src/lists/views.py (ch24l012) ==== [source,python] ---- from accounts.models import User [...] def my_lists(request, email): owner = User.objects.get(email=email) return render(request, "my_lists.html", {"owner": owner}) ---- ==== That gets our new test passing, but we'll also see an error from the previous test. We just need to add a user for it as well: [role="sourcecode"] .src/lists/tests/test_views.py (ch24l013) ==== [source,python] ---- def test_my_lists_url_renders_my_lists_template(self): User.objects.create(email="a@b.com") [...] ---- ==== And we get to an OK: ((("functional tests (FTs)", "FT-driven development, outside-in technique", startref="ix_FToutin2")))((("", startref="OITDDft22"))) ---- OK ---- === The Next "Requirement" from the Views Layer: [.keep-together]#New Lists Should Record Owner# ((("outside-in TDD", "views layer"))) Before we move down to the model layer, there's another part of the code at the view layer that will need to use our model: we need some way for newly created lists to be assigned to an owner, if the current user is logged in to the site. Here's a first crack at writing the test: [role="sourcecode"] .src/lists/tests/test_views.py (ch24l014) ==== [source,python] ---- class NewListTest(TestCase): [...] def test_list_owner_is_saved_if_user_is_authenticated(self): user = User.objects.create(email="a@b.com") self.client.force_login(user) #<1> self.client.post("/lists/new", data={"text": "new item"}) new_list = List.objects.get() self.assertEqual(new_list.owner, user) ---- ==== <1> `force_login()` is the way you get the test client to make requests with a logged-in user. The test fails as follows: ---- AttributeError: 'List' object has no attribute 'owner' ---- To fix it, let's first try writing code like this: [role="sourcecode"] .src/lists/views.py (ch24l015) ==== [source,python] ---- def new_list(request): form = ItemForm(data=request.POST) if form.is_valid(): nulist = List.objects.create() nulist.owner = request.user <1> nulist.save() <2> form.save(for_list=nulist) return redirect(nulist) else: return render(request, "home.html", {"form": form}) ---- ==== <1> We'll set the `.owner` attribute on our new list. <2> And we'll try and save it to the database. [role="pagebreak-before"] But it won't actually work, because we don't know _how_ to save a list owner yet: ---- self.assertEqual(new_list.owner, user) ^^^^^^^^^^^^^^ AttributeError: 'List' object has no attribute 'owner' ---- ==== A Decision Point: Whether to Proceed to the Next Layer with a Failing Test ((("outside-in TDD", "model layer", id="OITDDmodel21"))) In order to get this test passing, as it's written now, we have to move down to the model layer. However, it means doing more work with a failing test, which is not ideal.((("isolation of tests", "using mocks for")))((("mocks", "isolating tests using"))) The alternative is to rewrite the test to make it more _isolated_ from the level below, using mocks. On the one hand, it's a lot more effort to use mocks, and it can lead to tests that are harder to read. On the other hand, advocates of London-school TDD are very keen on the approach. You can read an exploration of this approach in https://www.obeythetestinggoat.com/book/appendix_purist_unit_tests.html[Online Appendix: Test Isolation and "Listening to Your Tests"]. For now, we'll accept the trade-off: moving down one layer with failing tests, but avoiding the extra mocks. [[revisit_this_point_with_isolated_tests]] Let's do a commit, and then `tag` the commit as a way of remembering our position if we want to revisit this decision later: [subs="specialcharacters,quotes"] ---- $ *git commit -am "new_list view tries to assign owner but cant"* $ *git tag revisit_this_point_with_isolated_tests* ---- === Moving Down to the Model Layer Our outside-in design has driven out two requirements for the model layer: we want to be able to assign an owner to a list using the attribute `.owner`, and we want to be able to access the list's owner with the API `owner.lists.all()`. Let's write a test for that: [role="sourcecode"] .src/lists/tests/test_models.py (ch24l018) ==== [source,python] ---- from accounts.models import User [...] class ListModelTest(TestCase): def test_get_absolute_url(self): [...] def test_list_items_order(self): [...] def test_lists_can_have_owners(self): user = User.objects.create(email="a@b.com") mylist = List.objects.create(owner=user) self.assertIn(mylist, user.lists.all()) ---- ==== And that gives us a new unit test failure: ---- mylist = List.objects.create(owner=user) [...] TypeError: List() got unexpected keyword arguments: 'owner' ---- The naive implementation would be this: [role="skipme"] [source,python] ---- from django.conf import settings [...] class List(models.Model): owner = models.ForeignKey(settings.AUTH_USER_MODEL) ---- But we want to make sure the list owner is optional. Explicit is better than implicit, and tests are documentation, so let's have a test for that too: [role="sourcecode"] .src/lists/tests/test_models.py (ch24l020) ==== [source,python] ---- def test_list_owner_is_optional(self): List.objects.create() # should not raise ---- ==== The correct implementation is this: [role="sourcecode"] .src/lists/models.py (ch24l021) ==== [source,python] ---- class List(models.Model): owner = models.ForeignKey( "accounts.User", related_name="lists", blank=True, null=True, on_delete=models.CASCADE, ) def get_absolute_url(self): return reverse("view_list", args=[self.id]) ---- ==== Now running the tests gives the usual database error: ---- return super().execute(query, params) ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^ django.db.utils.OperationalError: table lists_list has no column named owner_id ---- Because we need to make some migrations: [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py makemigrations*] Migrations for 'lists': src/lists/migrations/0007_list_owner.py + Add field owner to list ---- //22 We're almost there; a couple more failures in some of our old tests: ---- ERROR: test_can_save_a_POST_request [...] ValueError: Cannot assign "<SimpleLazyObject: <django.contrib.auth.models.AnonymousUser object at 0x1069852e>>": "List.owner" must be a "User" instance. [...] ERROR: test_redirects_after_POST [...] ValueError: Cannot assign "<SimpleLazyObject: <django.contrib.auth.models.AnonymousUser object at 0x106a1b440>>": "List.owner" must be a "User" instance. ---- We're moving back up to the views layer now, just doing a little tidying up. Notice that these are in the existing test for the `new_list` view, when we haven't got a logged-in user. The tests are reminding us to think of this use case too: we should only save the list owner when the user is actually logged in. The `.is_authenticated` attribute we came across in <<chapter_19_spiking_custom_auth>> comes in useful now:footnote:[When they're not logged in, Django represents users using a class called `AnonymousUser`, whose `.is_authenticated` is always `False`.] [role="sourcecode"] .src/lists/views.py (ch24l023) ==== [source,python] ---- if form.is_valid(): nulist = List.objects.create() if request.user.is_authenticated: nulist.owner = request.user nulist.save() form.save(for_list=nulist) return redirect(nulist) [...] ---- ==== [role="pagebreak-before"] And that gets us passing! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test lists*] [...] Ran 36 tests in 0.237s OK ---- This is a good time for a commit: [subs="specialcharacters,quotes"] ---- $ *git add src/lists* $ *git commit -m "lists can have owners, which are saved on creation."* ---- === Final Step: Feeding Through the .name API from the Template The last thing our outside-in design wanted ((("outside-in TDD", "accessing list name through the template")))came from the templates, which want to be able to access a list "name" based on the text of its first item: [role="sourcecode"] .src/lists/tests/test_models.py (ch24l024) ==== [source,python] ---- def test_list_name_is_first_item_text(self): list_ = List.objects.create() Item.objects.create(list=list_, text="first item") Item.objects.create(list=list_, text="second item") self.assertEqual(list_.name, "first item") ---- ==== [role="sourcecode"] .src/lists/models.py (ch24l025) ==== [source,python] ---- @property def name(self): return self.item_set.first().text ---- ==== And that, believe it or not, actually gets us a passing test and a working "My lists" page (see <<my-lists-page>>)! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests*] [...] Ran 8 tests in 93.819s OK ---- [[my-lists-page]] .The "My lists" page, in all its glory (and proof I did test on Windows) image::images/tdd3_2404.png["Screenshot of new My Lists page"] // DAVID: At the moment it's possible to see other users' list pages. (That's by design, right?) // But then - I hate to say it - we probably shouldn't call it the 'My Lists' page, except in // the link to the page. .The @property Decorator in Python ******************************************************************************* ((("@property decorator"))) ((("decorators", "property decorator"))) ((("Python 3", "@property decorator"))) If you haven't seen it before, the `@property` decorator transforms a method on a class to make it appear to the outside world like an attribute. ((("duck typing"))) This is a powerful feature of the language, because it makes it easy to implement "duck typing"—to change the implementation of a property without changing the interface of the class. In other words, if we decide to change `.name` into being a "real" attribute on the model, stored as text in the database, then we will be able to do so entirely transparently--as far as the rest of our code is concerned, will still be able to just access `.name` and get the list name, without needing to know about the implementation. Raymond Hettinger gave a https://oreil.ly/WQ2CX[classic, beginner-friendly talk on this topic at PyCon back in 2013], which I enthusiastically recommend (it covers about a million good practices for Pythonic class design besides). Of course, in the Django template language, `.name` would still call the method even if it didn't have `@property`, but that's a particularity of Django, and doesn't apply to Python in general... ******************************************************************************* // SEBASTIAN: While @property indeed is a helpful gimmick, I consider // @property doing DB operations or causing other side-effects an anti-pattern. // I wonder if readers of the book are also not already knowing that. // What I suggest is to consider whether to keep it in this chapter or not. // It seems to be a bit off. Might be as "quick hack" we're doing constantly to make // tests pass but I wouldn't settle on leaving it as it is. ((("", startref="OITDDmodel21"))) In the next chapter, it's time to recruit some computers to do more of the work for us. Let's talk about continuous integration (CI). // RITA: Later, let's check to make sure the end-of-chapter sidebars are consistent throughout the book. Some of them are called "Lessons Learned," others "Lessons Learned in X," and others are called just the topic name. .Outside-In TDD ******************************************************************************* Outside-in TDD:: This is methodology for building code, driven by tests, which proceeds by starting from the "outside" layers (presentation, GUI), and moving "inwards" step by step, via view/controller layers, down towards the model layer. The idea is to drive the design of your code from how it will be used, rather than trying to anticipate requirements from the bottom up. ((("outside-in TDD", "defined"))) // SEBASTIAN: Might be worth mentioning that outside-in plays nicely with API-first // or, at the very least, that it may also mean writing test at the API level // if we have a SPA Programming by wishful thinking:: The outside-in process is sometimes called "programming by wishful thinking". Actually, any kind of TDD involves some wishful thinking. We're always writing tests for things that don't exist yet. ((("programming by wishful thinking"))) The pitfalls of outside-in:: Outside-in isn't a silver bullet. It encourages us to focus on things that are immediately visible to the user, but it won't automatically remind us to write other critical tests that are less user-visible--things like security, for example. You'll need to remember them yourself. ((("", startref="TTDoutside22"))) ((("outside-in TDD", "drawbacks of"))) ******************************************************************************* ================================================ FILE: chapter_25_CI.asciidoc ================================================ [[chapter_25_CI]] == CI: Continuous Integration ((("continuous integration (CI)", id="CI24"))) ((("continuous integration (CI)", "benefits of"))) As our site grows, it takes longer and longer to run all of our functional tests. If this continues, the danger is that we're going to stop bothering.((("CI", see="continuous integration"))) Rather than let that happen, we can automate the running of functional tests by setting up "continuous integration", or CI. That way, in day-to-day development, we can just run the FT that we're working on at that time, and rely on CI to run all the other tests automatically and let us know if we've broken anything accidentally. The unit tests should stay fast enough that we can keep running the full suite locally, every few seconds. NOTE: Continuous integration is another practice that was popularised by Kent Beck's https://martinfowler.com/bliki/ExtremeProgramming.html[extreme programming (XP)] movement in the 1990s. As we'll see, one of the great frustrations of configuring CI is that the feedback loop is so much slower than working locally. As we go along, we'll look for ways to optimise for that where we can. While debugging, we'll also touch on the theme of _reproducibility_. It's the idea that we want to be able to reproduce behaviours of our CI environment locally—in the same way that we try and make our production and dev environments as similar as possible. [role="pagebreak-before less_space"] === CI in Modern Development Workflows We use CI for a number of reasons: * As mentioned, it can patiently run the full suite of tests, even if they've grown too large to run locally. * It can act as a "gate" in your deployment/release process, to ensure that you never deploy code that isn't passing all the tests. * In open source projects that use a "pull request" workflow, it's a way to ensure that any code submitted by potentially unknown contributors passes all your tests, before you consider merging it. * It's (sadly) increasingly common in corporate environments to see this pull request process and its associated CI checks to be used as the default way for teams to merge all code changes.footnote:[ I say "sadly" because you _should_ be able to trust your colleagues, not put them through a process designed for open source projects to de-risk code contributions from random strangers on the internet. Look up "trunk-based development" if you want to see more old people shouting at clouds on this topic.] === Choosing a CI Service ((("continuous integration (CI)", "choosing a service"))) In the early days, CI would be implemented by configuring a server (perhaps under a desk in the corner of the office) with software on it that could pull down all the code from the main branch at the end of each day, and scripts to compile all the code and run all the tests—a process that became known as a "build". Then, each morning, developers would take a look at the results, and deal with any broken builds. As the practice spread, and feedback cycles grew faster, CI software matured. CI has become a common cloud-based service, designed to integrate with code hosting providers like GitHub—or even provided directly by the same providers. GitHub has "GitHub Actions", and because it's like, _right there_, it's probably the most popular choice for open source projects these days. In a corporate environment, you might come across other solutions like CircleCI, Travis CI, and GitLab. It is still absolutely possible to download and self-host your own CI server; in the first and second editions of this book, I demonstrated the use of Jenkins, a popular tool at the time. But the installation and subsequent admin/maintenance burden is not effort-free, so for this edition I wanted to pick a service more like the kind of thing you're likely to encounter in your day job—while trying not to endorse the largest commercial provider. There's nothing wrong with GitHub Actions! It just doesn't need any _more_ help dominating the market. So I've decided to use GitLab in this book. It is absolutely a commercial service, but it retains an open source version, and you can self-host it if you want to. The syntax (it's always YAML...) and core concepts are common across all providers, so the things you learn here will be replicable in whichever service you encounter in future. Like most of the services out there, GitLab has a free tier, which will work fine for our purposes. === Getting Our Code into GitLab GitLab is primarily a code hosting service, like GitHub, so the first thing to do is get our code up there.((("GitLab", "getting code into", id="ix_GitL"))) ==== Signing Up Head over to https://gitlab.com[GitLab.com], and sign up for a free account. Then, head over to your profile page, and find the SSH Keys section, and upload a copy of your public key. ==== Starting a Project Then, use the New Project -> Create Blank Project option, as in <<gitlab-new-blank-project>>. Feel free to name the project whatever you want; you can see I've fancifully named mine with a "z". I'm a free spirit, what can I say. .Creating a new repo on GitLab [[gitlab-new-blank-project]] image::images/tdd3_2501.png["New Blank Project"] ==== Pushing Our Code Up Using Git Push First, we set up GitLab as a "remote" for our project: [role="skipme"] [subs="specialcharacters,quotes"] ---- # substitute your username and project name as necessary $ *git remote add gitlab git@gitlab.com:yourusername/superlists.git* $ *git remote -v* gitlab git@gitlab.com:hjwp/superlistz.git (fetch) gitlab git@gitlab.com:hjwp/superlistz.git (push) origin git@github.com:hjwp/book-example.git (fetch) origin git@github.com:hjwp/book-example.git (push) # (as you can see i already had a remote for github called 'origin') ---- Now we can push up our code with `git push`: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *git push gitlab* Enumerating objects: 706, done. Counting objects: 100% (706/706), done. Delta compression using up to 11 threads Compressing objects: 100% (279/279), done. Writing objects: 100% (706/706), 918.72 KiB | 131.25 MiB/s, done. Total 706 (delta 413), reused 682 (delta 408), pack-reused 0 (from 0) remote: Resolving deltas: 100% (413/413), done. To gitlab.com:hjwp/superlistz.git * [new branch] main -> main branch 'main' set up to track 'gitlab/main'. ---- If you refresh the GitLab UI, you should now see your code, as in <<gitlab_files_ui>>. .CI project files on GitLab [[gitlab_files_ui]] image::images/tdd3_2502.png["GitLab UI showing project files"] === Setting Up a First Cut of a CI Pipeline The "pipeline" terminology was popularised by Dave Farley and Jez Humble in their book _Continuous Delivery_ (Addison-Wesley).((("GitLab", "getting code into", startref="ix_GitL")))((("pipelines (CI)")))((("continuous integration (CI)", "setting up CI pipeline, first cut", id="ix_CIpipe1"))) The name alludes to the fact that a CI build typically has a series, where the process flows from one to another. Go to Build -> Pipelines, and you'll see a list of example templates. When getting to know a new configuration language, it's nice to be able to start with something that works, rather than a blank slate. I chose the Python example template and made a few customisations, but you could just as easily start from a blank slate and paste what I have here (YAML, once again, folks!): [role="sourcecode"] ..gitlab-ci.yml (ch25l001) ==== [source,yaml] ---- # Use the same image as our Dockerfile image: python:slim # These two settings let us cache pip-installed packages, # it came from the default template variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" cache: paths: - .cache/pip # "setUp" phase, before the main build before_script: - python --version ; pip --version # For debugging - pip install virtualenv - virtualenv .venv - source .venv/bin/activate # This is the main build test: script: - pip install -r requirements.txt # <1> # unit tests - python src/manage.py test lists accounts # <2> # (if those pass) all tests, incl. functional. - pip install selenium # <3> - cd src && python manage.py test # <4> ---- ==== <1> We start by installing our core requirements. <2> I've decided to run the unit tests first. This gives us an "early failure" if there's any problem at this stage, and saves us from having to run—and more importantly, wait for—the FTs to run. <3> Then we need Selenium for the functional tests. Again, I'm delaying this `pip install` until it's absolutely necessary, to get feedback as quickly as possible. <4> And here is a full test run, including the functional tests. TIP: It's a good idea in CI pipelines to try and run the quickest tests first, so that you can get feedback as quickly as possible. You can use the GitLab web UI to edit your pipeline YAML, and then when you save it, you can go check for results straight away. But it is also just a file in your repo! So as we go on through the chapter, you can also just edit it locally. You'll need to commit it and then `git push` up to GitLab, and then go check the Jobs section in the Build UI to see the results((("continuous integration (CI)", "setting up CI pipeline", startref="ix_CIpipe1"))) of your changes: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *git push gitlab* ---- === First Build! (and First Failure) // IDEA: consider deliberately forgetting to pip install selenium Whichever way you click through the UI, you should be able to find your way to see the output of the build job, as in <<gitlab_first_build>>.((("continuous integration (CI)", "building the pipeline", id="ix_CIbld")))((("GitLab", "building a CI pipeline in"))) .First build on GitLab [[gitlab_first_build]] image::images/tdd3_2503.png["GitLab UI showing the output of the first build"] [role="pagebreak-before"] Here's a selection of what I saw in the output console: [role="skipme small-code"] ---- Running with gitlab-runner 17.7.0~pre.103.g896916a8 (896916a8) on green-1.saas-linux-small-amd64.runners-manager.gitlab.com/default JLgUopmM, system ID: s_deaa2ca09de7 Preparing the "docker+machine" executor 00:20 Using Docker executor with image python:latest ... Pulling docker image python:latest ... [...] $ python src/manage.py test lists accounts Creating test database for alias 'default'... Found 55 test(s). System check identified no issues (0 silenced). ................../builds/hjwp/book-example/.venv/lib/python3.14/site-packages/django/core /handlers/base.py:61: UserWarning: No directory at: /builds/hjwp/book-example/src/static/ mw_instance = middleware(adapted_handler) ..................................... --------------------------------------------------------------------- Ran 53 tests in 0.129s OK Destroying test database for alias 'default'... $ pip install selenium Collecting selenium Using cached selenium-4.28.1-py3-none-any.whl.metadata (7.1 kB) Collecting urllib3<3,>=1.26 (from urllib3[socks]<3,>=1.26->selenium) [...] Successfully installed attrs-25.1.0 certifi-2025.1.31 h11-0.14.0 idna-3.10 outcome-1.3.0.post0 pysocks-1.7.1 selenium-4.28.1 sniffio-1.3.1 sortedcontainers-2.4.0 trio-0.29.0 trio-websocket-0.12.1 typing_extensions-4.12.2 urllib3-2.3.0 websocket-client-1.8.0 wsproto-1.2.0 $ cd src && python manage.py test Creating test database for alias 'default'... Found 63 test(s). System check identified no issues (0 silenced). ......../builds/hjwp/book-example/.venv/lib/python3.14/site-packages/django/core/handlers /base.py:61: UserWarning: No directory at: /builds/hjwp/book-example/src/static/ mw_instance = middleware(adapted_handler) ...............................................EEEEEEEE ====================================================================== ERROR: test_layout_and_styling (functional_tests.test_layout_and_styling. LayoutAndStylingTest.test_layout_and_styling) --------------------------------------------------------------------- Traceback (most recent call last): File "/builds/hjwp/book-example/src/functional_tests/base.py", line 30, in setUp self.browser = webdriver.Firefox() ~~~~~~~~~~~~~~~~~^^ [...] selenium.common.exceptions.WebDriverException: Message: Process unexpectedly closed with status 255 --------------------------------------------------------------------- Ran 61 tests in 8.658s FAILED (errors=8) selenium.common.exceptions.WebDriverException: Message: Process unexpectedly closed with status 255 ---- NOTE: If GitLab won't run your build at this point, you may need to go through some sort of identity-verification process. Check your profile page. You can see we got through the unit tests, and then in the full test run we have 8 errors out of 63 tests. The FTs are all failing. I'm "lucky" because I've done this sort of thing many times before, so I know what to expect: it's failing because Firefox isn't installed in the image we're using.((("Firefox", "installing in container image"))) Let's modify the script, and add an `apt install`. Again we'll do it as late as possible: [role="sourcecode"] ..gitlab-ci.yml (ch25l002) ==== [source,yaml] ---- # This is the main build test: script: - pip install -r requirements.txt # unit tests - python src/manage.py test lists accounts # (if those pass) all tests, incl. functional. - apt update -y && apt install -y firefox-esr # <1> - pip install selenium - cd src && python manage.py test ---- ==== <1> We use the Debian Linux `apt` package manager to install Firefox. `firefox-esr` is the "extended support release", which is a more stable version of Firefox to test against. [role="pagebreak-before"] When you save that change (and commit and push if necessary), the pipeline will run again. If you wait a bit, you'll see we get a slightly different failure: [role="skipme small-code"] ---- $ apt-get update -y && apt-get install -y firefox-esr Get:1 http://deb.debian.org/debian bookworm InRelease [151 kB] Get:2 http://deb.debian.org/debian bookworm-updates InRelease [55.4 kB] Get:3 http://deb.debian.org/debian-security bookworm-security InRelease [48.0 kB] [...] The following NEW packages will be installed: adwaita-icon-theme alsa-topology-conf alsa-ucm-conf at-spi2-common at-spi2-core dbus dbus-bin dbus-daemon dbus-session-bus-common dbus-system-bus-common dbus-user-session dconf-gsettings-backend dconf-service dmsetup firefox-esr fontconfig fontconfig-config [...] Get:117 http://deb.debian.org/debian-security bookworm-security/main amd64 firefox-esr amd64 128.7.0esr-1~deb12u1 [69.8 MB] [...] Selecting previously unselected package firefox-esr. Preparing to unpack .../105-firefox-esr_128.7.0esr-1~deb12u1_amd64.deb ... Adding 'diversion of /usr/bin/firefox to /usr/bin/firefox.real by firefox-esr' Unpacking firefox-esr (128.7.0esr-1~deb12u1) ... [...] Setting up firefox-esr (128.7.0esr-1~deb12u1) ... update-alternatives: using /usr/bin/firefox-esr to provide /usr/bin/x-www-browser (x-www-browser) in auto mode [...] ====================================================================== ERROR: test_multiple_users_can_start_lists_at_different_urls (functional_tests.test_simple_list_creation.NewVisitorTest. test_multiple_users_can_start_lists_at_different_urls) --------------------------------------------------------------------- Traceback (most recent call last): File "/builds/hjwp/book-example/src/functional_tests/base.py", line 30, in setUp self.browser = webdriver.Firefox() ~~~~~~~~~~~~~~~~~^^ [...] selenium.common.exceptions.WebDriverException: Message: Process unexpectedly closed with status 1 --------------------------------------------------------------------- Ran 61 tests in 3.654s FAILED (errors=8) ---- We can see Firefox installing OK, but we still get an error. This time, it's exit code 1. [role="pagebreak-before less_space"] ==== Trying to Reproduce a CI Error Locally The cycle of "change _.gitlab-ci.yml_, push, wait for a build, check results" is painfully slow. Let's see if we can reproduce this error locally.((("errors", "reproducing CI error locally"))) To reproduce the CI environment locally, I put together a quick Dockerfile, by copy-pasting the steps in the `script` section and prefixing them with `RUN` commands: [role="sourcecode"] .infra/Dockerfile.ci (ch25l003) ==== [source,dockerfile] ---- FROM python:slim RUN pip install virtualenv RUN virtualenv .venv # this won't work # RUN source .venv/bin/activate # use full path to venv instead. COPY requirements.txt requirements.txt RUN .venv/bin/pip install -r requirements.txt RUN apt update -y && apt install -y firefox-esr RUN .venv/bin/pip install selenium COPY infra/debug-ci.py debug-ci.py CMD .venv/bin/python debug-ci.py ---- ==== And let's add a little debug script at _debug-ci.py_: [role="sourcecode small-code"] .infra/debug-ci.py (ch25l004) ==== [source,python] ---- from selenium import webdriver # just try to open a selenium session webdriver.Firefox().quit() ---- ==== [role="pagebreak-before"] We build and run it like this: [role="skipme small-code"] [subs="specialcharacters,macros"] ---- $ pass:specialcharacters,quotes[*docker build -f infra/Dockerfile.ci -t debug-ci . && \ docker run -it debug-ci*] [...] => [internal] load build definition from infra/Dockerfile.ci 0.0s => => transferring dockerfile: [...] => [internal] load metadata for docker.io/library/python:slim [...] => [1/8] FROM docker.io/library/python:slim@sha256:[...] => CACHED [2/8] RUN pip install virtualenv 0.0s => CACHED [3/8] RUN virtualenv .venv 0.0s => CACHED [4/8] COPY requirements.txt requirements.txt 0.0s => CACHED [5/8] RUN .venv/bin/pip install -r requirements.txt 0.0s => CACHED [6/8] RUN apt update -y && apt install -y firefox-esr 0.0s => CACHED [7/8] RUN .venv/bin/pip install selenium 0.0s => [8/8] COPY infra/debug-ci.py debug-ci.py 0.0s => exporting to image 0.0s => => exporting layers 0.0s => => writing image sha256:[...] => => naming to docker.io/library/debug-ci 0.0s Traceback (most recent call last): File "//.venv/lib/python3.14/site-packages/selenium/webdriver/common/driver_finder.py", line 67, in _binary_paths output = SeleniumManager().binary_paths(self._to_args()) [...] selenium.common.exceptions.WebDriverException: Message: Unsupported platform/architecture combination: linux/aarch64 The above exception was the direct cause of the following exception: Traceback (most recent call last): File "//debug-ci.py", line 4, in <module> webdriver.Firefox().quit() ~~~~~~~~~~~~~~~~~^^ [...] selenium.common.exceptions.NoSuchDriverException: Message: Unable to obtain driver for firefox; For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors/driver_location ---- You might not see this--that "Unsupported platform/architecture combination" error is spurious; it's because I was on a Mac. Let's try again with: // SEBASTIAN: Might use extra sentence of explanation why being on Mac requires you to // do a cross-build [role="ignore-errors"] [subs="specialcharacters,macros"] ---- $ pass:specialcharacters,quotes[*docker build -f infra/Dockerfile.ci -t debug-ci --platform=linux/amd64 . && \ docker run --platform=linux/amd64 -it debug-ci*] [...] Traceback (most recent call last): File "//debug-ci.py", line 4, in <module> webdriver.Firefox().quit() [...] selenium.common.exceptions.WebDriverException: Message: Process unexpectedly closed with status 1 ---- OK, that's a reproduction of our issue. But no further clues yet! ==== Enabling Debug Logs for Selenium/Firefox/Webdriver Getting debug information out of Selenium can be a bit fiddly.((("logging", "enabling debug logs for Firefox/Selenium/Webdriver")))((("Webdriver", "enabling debug logs for")))((("Firefox", "enabling debug logs for")))((("Selenium", "enabling debug logs for"))) I tried two avenues: setting `options` and setting the `service`. The former doesn't really work as far as I can tell, but the latter does: [role="sourcecode"] .infra/debug-ci.py (ch25l005) ==== [source,python] ---- import subprocess from selenium import webdriver options = webdriver.FirefoxOptions() # <1> options.log.level = "trace" service = webdriver.FirefoxService( # <2> log_output=subprocess.STDOUT, service_args=["--log", "trace"] ) # just try to open a selenium session webdriver.Firefox(options=options, service=service).quit() ---- ==== <1> This is how I attempted to increase the log level using `options`. I had to reverse-engineer it from the source code, and it doesn't seem to work anyway, but I thought I'd leave it here for future reference. There is some limited info in the https://www.selenium.dev/documentation/webdriver/browsers/firefox/#log-output[Selenium docs]. <2> This is the `FirefoxService` config class, which _does_ seem to let you print some debug info. I'm configuring it to print to standard output. Sure enough, we can see some output now! [role="ignore-errors small-code"] [subs="specialcharacters,macros"] ---- $ pass:specialcharacters,quotes[*docker build -f infra/Dockerfile.ci -t debug-ci --platform=linux/amd64 . && \ docker run --platform=linux/amd64 -it debug-ci*] [...] 1234567890111 geckodriver INFO Listening on 127.0.0.1:XXXX 1234567890112 webdriver::server DEBUG -> POST /session {"capabilities": {"firstMatch": [{}], "alwaysMatch": {"browserName": "firefox", "acceptInsecureCerts": true, ... , "moz:firefoxOptions": {"binary": "/usr/bin/firefox", "prefs": {"remote.active-protocols": 1}, "log": {"level": "trace"}}}}} 1234567890111 geckodriver::capabilities DEBUG Trying to read firefox version from ini files 1234567890111 geckodriver::capabilities DEBUG Trying to read firefox version from binary 1234567890111 geckodriver::capabilities DEBUG Found version 128.10.1esr 1740029792102 mozrunner::runner INFO Running command: MOZ_CRASHREPORTER="1" MOZ_CRASHREPORTER_NO_REPORT="1" MOZ_CRASHREPORTER_SHUTDOWN="1" [...] "--remote-debugging-port" [...] "-no-remote" "-profile" "/tmp/rust_mozprofile[...] 1234567890111 geckodriver::marionette DEBUG Waiting 60s to connect to browser on 127.0.0.1 1234567890111 geckodriver::browser TRACE Failed to open /tmp/rust_mozprofile[...] 1234567890111 geckodriver::marionette TRACE Retrying in 100ms Error: no DISPLAY environment variable specified 1234567890111 geckodriver::browser DEBUG Browser process stopped: exit status: 1 1234567890112 webdriver::server DEBUG <- 500 Internal Server Error {"value":{"error":"unknown error","message":"Process unexpectedly closed with status 1","stacktrace":""}} Traceback (most recent call last): File "//debug-ci.py", line 13, in <module> webdriver.Firefox(options=options, service=service).quit() [...] selenium.common.exceptions.WebDriverException: Message: Process unexpectedly closed with status 1 ---- // DAVID: Pasting this into an LLM gave some good suggestions. Well, it wasn't immediately obvious what's going on there, but I did eventually get a clue from the line that says `no DISPLAY environment variable specified`. Out of curiosity, I thought I'd try running `firefox` directly:footnote:[ If you remember from <<chapter_09_docker>>, `docker run` by default runs the command specified in `CMD`, but you can override that by specifying a different command to run at the end of the parameter list.] [role="ignore-errors"] [subs="specialcharacters,quotes"] ---- $ *docker build -f infra/Dockerfile.ci -t debug-ci --platform=linux/amd64 . && \ docker run --platform=linux/amd64 -it debug-ci firefox* [...] Error: no DISPLAY environment variable specified ---- Sure enough, the same error. ==== Enabling Headless Mode for Firefox If you search around for this error, you'll eventually find enough pointers to the answer: Firefox is crashing because it can't find a display.((("headless mode")))((("Firefox", "enabling headless mode for"))) Servers are "headless", meaning they don't have a screen. Thankfully, Firefox has a headless mode, which we can enable by setting an environment variable, `MOZ_HEADLESS`. Let's confirm that locally. We'll use the `-e` flag for `docker run`: [subs="specialcharacters,macros"] ---- $ pass:specialcharacters,quotes[*docker build -f infra/Dockerfile.ci -t debug-ci --platform=linux/amd64 . && \ docker run -e MOZ_HEADLESS=1 --platform=linux/amd64 -it debug-ci*] 1234567890111 geckodriver INFO Listening on 127.0.0.1:43137 [...] *** You are running in headless mode. [...] 1234567890112 webdriver::server DEBUG Teardown [...] 1740030525996 Marionette DEBUG Closed connection 0 1234567890111 geckodriver::browser DEBUG Browser process stopped: exit status: 0 1234567890112 webdriver::server DEBUG <- 200 OK [...] ---- It takes quite a long time to run, and there's lots of debug out, but...it looks OK! That's no longer an error. Let's set that environment variable in our CI script: [role="sourcecode"] ..gitlab-ci.yml (ch25l006) ==== [source,yaml] ---- variables: # Put pip-cache in home folder so we can use gitlab cache PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # Make Firefox run headless. MOZ_HEADLESS: "1" ---- ==== TIP: Using a local Docker image to reproduce the CI environment is a hint that it might be worth investing time in running CI in a custom Docker image that you fully control; this is another way of improving _reproducibility_. We won't have time to go into detail in this book though. And we'll see what happens when we do `git push gitlab` again. [role="pagebreak-before less_space"] === A Common Bugbear: Flaky Tests Did it work for you? For me, it _almost_ did.((("continuous integration (CI)", "building the pipeline", startref="ix_CIbld")))((("flaky tests"))) All but one of the FTs passed, but I did see one unexpected error: [role="skipme small-code"] ---- + python manage.py test functional_tests ......F. ====================================================================== FAIL: test_can_start_a_todo_list (functional_tests.test_simple_list_creation.NewVisitorTest) --------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/functional_tests/test_simple_list_creation.py", line 38, in test_can_start_a_todo_list self.wait_for_row_in_list_table('2: Use peacock feathers to make a fly') File "...goat-book/functional_tests/base.py", line 51, in wait_for_row_in_list_table raise e File "...goat-book/functional_tests/base.py", line 47, in wait_for_row_in_list_table self.assertIn(row_text, [row.text for row in rows]) AssertionError: '2: Use peacock feathers to make a fly' not found in ['1: Buy peacock feathers'] --------------------------------------------------------------------- ---- Now, you might not see this error, but it's common for the switch to CI to flush out some "flaky" tests—things that will fail intermittently. In CI, a common cause is the "noisy neighbour" problem, where the CI server might be much slower than your own machine, thus flushing out some race conditions—or in this case, just randomly hanging for a few seconds, taking us past the default timeout. Let's give ourselves some tools to help debug though. === Taking Screenshots ((("continuous integration (CI)", "screenshots", id="CIscreen24"))) ((("screenshots", id="screen24"))) ((("debugging", "screenshots for", id="DBscreen24"))) To be able to debug unexpected failures that happen on a remote server, it would be good to see a picture of the screen at the moment of the failure, and maybe also a dump of the page's HTML. We can do that using some custom logic in our FT class `tearDown`. We'll need to do a bit of introspection of `unittest` internals (a private attribute called `._outcome`) but this will work:footnote:[...or at least until the next Python version. Using private APIs is risky, but I couldn't find a better way.] [role="sourcecode"] .src/functional_tests/base.py (ch25l007) ==== [source,python] ---- import os import time from datetime import datetime from pathlib import Path [...] MAX_WAIT = 5 SCREEN_DUMP_LOCATION = Path(__file__).absolute().parent / "screendumps" [...] class FunctionalTest(StaticLiveServerTestCase): def setUp(self): [...] def tearDown(self): if self._test_has_failed(): if not SCREEN_DUMP_LOCATION.exists(): SCREEN_DUMP_LOCATION.mkdir(parents=True) self.take_screenshot() self.dump_html() self.browser.quit() super().tearDown() def _test_has_failed(self): # slightly obscure but couldn't find a better way! return self._outcome.result.failures or self._outcome.result.errors ---- ==== We first create a directory for our screenshots if necessary, and then we take our screenshot and dump the HTML. Let's see how those will work: [role="sourcecode"] .src/functional_tests/base.py (ch25l008) ==== [source,python] ---- def take_screenshot(self): path = SCREEN_DUMP_LOCATION / self._get_filename("png") print("screenshotting to", path) self.browser.get_screenshot_as_file(str(path)) def dump_html(self): path = SCREEN_DUMP_LOCATION / self._get_filename("html") print("dumping page HTML to", path) path.write_text(self.browser.page_source) ---- ==== And finally, here's a way of generating a unique filename identifier, which includes the name of the test and its class, as well as a timestamp: [role="sourcecode small-code"] .src/functional_tests/base.py (ch25l009) ==== [source,python] ---- def _get_filename(self, extension): timestamp = datetime.now().isoformat().replace(":", ".")[:19] return ( f"{self.__class__.__name__}.{self._testMethodName}-{timestamp}.{extension}" ) ---- ==== You can test this first locally by deliberately breaking one of the tests—with a `self.fail()` half-way through, for example—and you'll see something like this: [role="dofirst-ch25l010"] [subs="specialcharacters,quotes"] ---- $ *./src/manage.py test functional_tests.test_my_lists* [...] Fscreenshotting to ...goat-book/src/functional_tests/screendumps/MyListsTest.te st_logged_in_users_lists_are_saved_as_my_lists-[...] dumping page HTML to ...goat-book/src/functional_tests/screendumps/MyListsTest. test_logged_in_users_lists_are_saved_as_my_lists-[...] Fscreenshotting to ...goat-book/src/functional_tests/screendumps/MyListsTest.te st_logged_in_users_lists_are_saved_as_my_lists-2025-02-18T11.29.00.png dumping page HTML to ...goat-book/src/functional_tests/screendumps/MyListsTest. test_logged_in_users_lists_are_saved_as_my_lists-2025-02-18T11.29.00.html ---- Why not try and open one of those files up? It's kind of satisfying. === Saving Build Outputs (or Debug Files) as Artifacts We also need to tell GitLab to "save" these files, for us to be able to actually look at them.((("GitLab", "saving build outputs as artifacts")))((("artifacts"))) Those are called _artifacts_: [role="sourcecode"] ..gitlab-ci.yml (ch25l012) ==== [source,yaml] ---- test: [...] script: [...] artifacts: # <1> when: always # <2> paths: # <1> - src/functional_tests/screendumps/ ---- ==== <1> `artifacts` is the name of the key, and the `paths` argument is fairly self-explanatory. You can use wildcards here—more info in the https://docs.gitlab.com/ci/jobs/job_artifacts[GitLab docs]. <2> One thing the docs _didn't_ make obvious is that you need `when: always`, because otherwise it won't save artifacts for failed jobs. That was annoyingly hard to figure out! In any case, that should work. If you commit the code and then push it back to GitLab, we should be able to see a new build job: [role="dofirst-ch25l010-1"] [subs="specialcharacters,quotes"] ---- $ *echo "src/functional_tests/screendumps" >> .gitignore* $ *git commit -am "add screenshot on failure to FT runner"* $ *git push* ---- [role="pagebreak-before"] In its output, we'll see the screenshots and HTML dumps being saved: [role="skipme small-code"] ---- screendumps/LoginTest.test_can_get_email_link_to_log_in-window0-2014-01-22T17.45.12.html Fscreenshotting to /builds/hjwp/book-example/src/functional_tests/screendumps/ NewVisitorTest.test_can_start_a_todo_list-2025-02-17T17.51.01.png dumping page HTML to /builds/hjwp/book-example/src/functional_tests/screendumps/ NewVisitorTest.test_can_start_a_todo_list-2025-02-17T17.51.01.html Not Found: /favicon.ico .screenshotting to /builds/hjwp/book-example/src/functional_tests/screendumps/ NewVisitorTest.test_multiple_users_can_start_lists_at_different_urls-2025-02-17T17. 51.06.png dumping page HTML to /builds/hjwp/book-example/src/functional_tests/screendumps/ NewVisitorTest.test_multiple_users_can_start_lists_at_different_urls-2025-02-17T17.51. 06.html ====================================================================== FAIL: test_can_start_a_todo_list (functional_tests.test_simple_list_creation.NewVisitorTest. test_can_start_a_todo_list) [...] ---- And to the right, some new UI options appear to Browse the artifacts, as in <<gitlab_ui_for_browse_artifacts>>. .Artifacts appear on the right of the build job [[gitlab_ui_for_browse_artifacts]] image::images/tdd3_2504.png["GitLab UI tab showing the option to browse artifacts"] [role="pagebreak-before"] And if you navigate through, you'll see something like <<gitlab_ui_show_screenshot>>. .Our screenshot in the GitLab UI, looking unremarkable [[gitlab_ui_show_screenshot]] image::images/tdd3_2505.png["GitLab UI showing a normal-looking screenshot of the site"] // TODO: this errors if there are no screenshots. === If in Doubt, Try Bumping the Timeout! ((("", startref="CIscreen24"))) ((("", startref="screen24"))) ((("", startref="DBscreen24"))) ((("continuous integration (CI)", "timeout bumping"))) Your build might be clear, but mine was still failing, and those screenshots didn't offer any obvious clues. Hmm. Well, when in doubt, bump the timeout—as the old adage goes: [role="sourcecode skipme"] .src/functional_tests/base.py ==== [source,python] ---- MAX_WAIT = 10 ---- ==== Then we can rerun the build by pushing, and confirm it now works. [role="pagebreak-before less_space"] === A Successful Python Test Run At this point, we should get a working pipeline (see <<gitlab_pipeline_success>>). .A successful GitLab pipeline [[gitlab_pipeline_success]] image::images/tdd3_2506.png["GitLab UI showing a successful pipeline run"] === Running Our JavaScript Tests in CI ((("continuous integration (CI)", "setting up CI pipeline", startref="ix_CIpipe1")))((("continuous integration (CI)", "QUnit JavaScript tests", id="CIjs5"))) ((("JavaScript testing", "in CI", secondary-sortas="CI", id="JSCI"))) There's a set of tests we almost forgot--the JavaScript tests. Currently our "test runner" is an actual web browser. To get them running in CI, we need a command-line test runner. NOTE: Our JavaScript tests currently test the interaction between our code and the Bootstrap framework/CSS, so we still need a real browser to be able to make our visibility checks work. Thankfully, the Jasmine docs point us straight towards the kind of tool we need: https://github.com/jasmine/jasmine-browser-runner[Jasmine browser runner]. ==== Installing Node.js It's time to stop pretending we're not in the JavaScript game. We're doing web development; that means we do JavaScript; that means we're going to end up with Node.js on our computers.((("Node.js", "installing"))) It's just the way it has to be. Follow the instructions on the http://nodejs.org[Node.js home page]. It should guide you through installing the "node version manager" (nvm), and then to getting the latest version of node: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *nvm install --lts* Installing Node v22.17.0 (arm64) [...] $ *node -v* v22.17.0 ---- ==== Installing and Configuring the Jasmine Browser Runner The docs suggest we install it ((("Jasmine", "installing and configuring browser runner", id="ix_Jasbrwsrun")))((("browsers", "browser-based test runner (Jasmine)", id="ix_brwststrun")))like this, and then run the `init` command to generate a default config file: // IDEA: unskip. should be able to do some sort of rule=with-cd thingie [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *cd src/lists/static* $ *npm install --save-dev jasmine-browser-runner jasmine-core* [...] added 151 packages in 4s $ *cat package.json* # this is the equivalent of requirements.txt { "devDependencies": { "jasmine-browser-runner": "^3.0.0", "jasmine-core": "^5.6.0" } } $ *ls node_modules/* # will show several dozen directories $ *npx jasmine-browser-runner init* Wrote configuration to spec/support/jasmine-browser.mjs. ---- Well, we now have about a million files in _node_modules/_ (which is JavaScript's version of a virtualenv, essentially), and we also have a new config file in _spec/support/jasmine-browser.mjs_. That's not the ideal place, because we've said our tests live in a folder called _tests_. So, let's move the config file in there: [subs="specialcharacters,quotes"] ---- $ *mv spec/support/jasmine-browser.mjs tests/jasmine-browser-runner.config.mjs* $ *rm -rf spec* ---- [role="pagebreak-before"] Then let's edit it slightly, to specify a few things correctly: [role="sourcecode"] .src/lists/static/tests/jasmine-browser-runner.config.mjs (ch25l013) ==== [source,js] ---- export default { srcDir: ".", // <1> srcFiles: [ "*.js" ], specDir: "tests", // <2> specFiles: [ "**/*[sS]pec.js" ], helpers: [ "helpers/**/*.js" ], env: { stopSpecOnExpectationFailure: false, stopOnSpecFailure: false, random: true, forbidDuplicateNames: true }, listenAddress: "localhost", hostname: "localhost", browser: { name: "headlessFirefox" // <3> } }; ---- ==== // DAVID: srcFiles was "**/*.js", should it be changed too? <1> Our source files are in the current directory, _src/lists/static_—i.e., _lists.js_. <2> Our spec files are in _tests/_. <3> And here we say we want to use the headless version of Firefox. (We could have done this by setting `MOZ_HEADLESS` at the command line again, but this saves us from having to remember.) [role="pagebreak-before"] Let's try running it now. We use the `--config` option to pass it to the now non-standard path to the config file: [role="skipme small-code"] [subs="specialcharacters,quotes"] ---- $ *npx jasmine-browser-runner runSpecs \ --config=tests/jasmine-browser-runner.config.mjs* Jasmine server is running here: http://localhost:62811 Jasmine tests are here: ...goat-book/src/lists/static/tests Source files are here: ...goat-book/src/lists/static Running tests in the browser... Randomized with seed 17843 Started .F. Failures: 1) Superlists tests error message should be hidden on input Message: Expected true to be false. Stack: <Jasmine> @http://localhost:62811/__spec__/Spec.js:46:40 <Jasmine> 3 specs, 1 failure Finished in 0.014 seconds Randomized with seed 17843 (jasmine-browser-runner runSpecs --seed=17843) ---- Could be worse! One failure out of three specs. Unfortunately, it's the most important test: [role="sourcecode currentcontents"] .src/lists/static/tests/Spec.js ==== [source,python] ---- it("should hide error message on input", () => { initialize(inputSelector); textInput.dispatchEvent(new InputEvent("input")); expect(errorMsg.checkVisibility()).toBe(false); }); ---- ==== Ah yes, if you remember, I said that the main reason we need to use a browser-based test runner is because our visibility checks depend on the Bootstrap CSS framework. In the HTML spec runner we've configured so far, we load Bootstrap using a `<link>` tag: [role="sourcecode currentcontents"] .src/lists/static/tests/SpecRunner.html ==== [source,html] ---- <!-- Bootstrap CSS --> <link href="../bootstrap/css/bootstrap.min.css" rel="stylesheet"> ---- ==== [role="pagebreak-before"] And here's how we load it for `jasmine-browser-runner`: [role="sourcecode"] .src/lists/static/tests/jasmine-browser-runner.config.mjs (ch25l014) ==== [source,js] ---- export default { srcDir: ".", srcFiles: [ "*.js" ], specDir: "tests", specFiles: [ "**/*[sS]pec.js" ], cssFiles: [ // <1> "bootstrap/css/bootstrap.min.css" // <1> ], helpers: [ "helpers/**/*.js" ], ---- ==== <1> The `cssFiles` key is how you tell the runner to load, er, some CSS. I found that out in the https://jasmine.github.io/api/browser-runner/edge/Configuration.html[docs]. Let's give that a go... [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *npx jasmine-browser-runner runSpecs \ --config=tests/jasmine-browser-runner.config.mjs* Jasmine server is running here: http://localhost:62901 Jasmine tests are here: .../goat-book/src/lists/static/tests Source files are here: .../goat-book/src/lists/static Running tests in the browser... Randomized with seed 06504 Started ... 3 specs, 0 failures Finished in 0.016 seconds Randomized with seed 06504 (jasmine-browser-runner runSpecs --seed=06504) ---- Hooray! That works locally—let's get it into CI: [role="skipme"] [subs="specialcharacters,quotes"] ---- $ *cd -* # go back to the project root # add the package.json, which saves our node depenencies $ *git add src/lists/static/package.json src/lists/static/package-lock.json* # ignore the node_modules/ directory $ *echo "node_modules/" >> .gitignore* # and our config file $ *git add src/lists/static/tests/jasmine-browser-runner.config.mjs* $ *git add .gitignore* $ *git commit -m "config for node + jasmine-browser-runner for JS tests"* ---- //015,016,017 ==== Adding a Build Step for JavaScript ((("Jasmine", "installing and configuring browser runner", startref="ix_Jasbrwsrun")))((("browsers", "browser-based test runner (Jasmine)", startref="ix_brwststrun"))) We now want two different build steps, so let's rename `test` to `test-python` and move all its specific bits like `variables` and `before_script` inside it, and then create a separate step called `test-js`, with a similar structure: [role="sourcecode"] ..gitlab-ci.yml (ch25l018) ==== [source,yaml] ---- test-python: # Use the same image as our Dockerfile image: python:slim # <1> variables: # <1> # Put pip-cache in home folder so we can use gitlab cache PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # Make Firefox run headless. MOZ_HEADLESS: "1" cache: # <1> paths: - .cache/pip # "setUp" phase, before the main build before_script: # <1> - python --version ; pip --version # For debugging - pip install virtualenv - virtualenv .venv - source .venv/bin/activate script: - pip install -r requirements.txt # unit tests - python src/manage.py test lists accounts # (if those pass) all tests, incl. functional. - apt update -y && apt install -y firefox-esr - pip install selenium - cd src && python manage.py test artifacts: when: always paths: - src/functional_tests/screendumps/ test-js: # <2> image: node:slim script: - apt update -y && apt install -y firefox-esr # <3> - cd src/lists/static - npm install # <4> - npx jasmine-browser-runner runSpecs --config=tests/jasmine-browser-runner.config.mjs # <5> ---- ==== <1> `image`, `variables`, `cache`, and `before_script` all move out of the top level and into the `test-python` step, as they're all specific to this step only now. <2> Here's our new step, `test-js`. <3> We install Firefox into the node image, just like we do for the Python one. <4> We don't need to specify _what_ to `npm install`, because that's all in the _package-lock.json_ file. <5> And here's our command to run the tests. And slap me over the head with a wet fish if that doesn't pass on the first go! See <<gitlab_pipeline_js_success>> for a successful pipeline run. .Wow, there are those JavaScript tests, passing on the first attempt! [[gitlab_pipeline_js_success]] image::images/tdd3_2507.png["GitLab UI showing a successful pipeline run with JavaScript tests"] ((("", startref="CIjs5"))) ((("", startref="JSCI"))) [role="pagebreak-before less_space"] === Tests Now Pass And there we are! A complete CI build featuring all of our tests! See <<gitlab_pipeline_overview_success.png>>. .Here are both our jobs in all their green glory [[gitlab_pipeline_overview_success.png]] image::images/tdd3_2508.png["GitLab UI the pipeline overview, with both build jobs green"] Nice to know that, no matter how lazy I get about running the full test suite on my own machine, the CI server will catch me. Another one of the Testing Goat's agents in cyberspace, watching over us... .Alternatives: Woodpecker and Forgejo ******************************************************************************* I want to give a shout out to https://woodpecker-ci.org[Woodpecker CI] and https://forgejo.org[Forgejo], two of the newer self-hosted CI options. And while I'm at it, to https://www.jenkins.io[Jenkins], which did a great job for the first and second editions, and still does for many people.((("continuous integration (CI)", "self-hosted CI options"))) // CSANAD: I just found framagit.org by Framasoft. Maybe we could mention them? Although // it might be important to ask them first, in case they need to handle the // expected additional traffic. If you want true independence from overly commercial interests, then self-hosted is the way to go. You'll need your own server for both of these. I tried both, and managed to get them working within an hour or two. Their documentation is good. If you do decide to give them a go, I'd say, be a bit cautious about security options. For example, you might decide you don't want any old person from the internet to be able to sign up for an account on your server: [role="skipme"] ---- DISABLE_REGISTRATION: true ---- But more power to you for giving it a go! ******************************************************************************* === Some Things We Didn't Cover CI is a big topic and, inevitably, I couldn't cover everything. Here's a few pointers to things you might want to learn about. ==== Defining a Docker Image for CI We spent quite a bit of time debugging—for example, the unhelpful messages when Firefox wasn't installed.((("continuous integration (CI)", "defining Docker image for")))((("Docker", "defining container image for CI"))) Just as we did when preparing our deployment, it's a big help having an environment that you can run on your local machine that's as close as possible to what you have remotely; that's why we chose to use a Docker image. In CI, our tests also run a Docker image (`python:slim` and `node:slim`), so one common pattern is to define a Docker image within your repo that you'll use for CI. Ideally, it should also be as similar as possible to the one you use in production! A typical solution here is to use multistage Docker builds—with a base stage, a prod stage, and a dev/CI stage. In our case, the last stage would have Firefox, Selenium, and other test-only dependencies in it, which we don't need for prod. You can then run your tests locally inside the same Docker image that's used in CI.((("reproducibility"))) TIP: _Reproducibility_ is one of the key attributes we're aiming for. The more your project grows in complexity, the more it's worth investing in minimising the differences between local dev, CI, and prod. ==== Caching We touched on the use of caches in CI for the `pip` download cache, but as CI pipelines grow in maturity, you'll find you can make more and more use of caching. ((("caching in CI pipelines")))For example, it might be a good idea to cache your _node_modules/_ directory. It's a topic for another time, but this is yet another way of trying to speed up the feedback cycle. ==== Automated Deployment, aka Continuous Delivery (CD) The natural next step is to finish our journey into automation, and set up a pipeline that will deploy our code all the way to production, each time we push code...as long as the tests pass!((("continuous delivery (CD)")))((("automated deployment")))((("deployment", "continuous delivery"))) I work through an example of how to do that in the https://www.obeythetestinggoat.com/book/appendix_CD.html[Online Appendix: Continuous Deployment (CD)]. If you're feeling inspired, I'd encourage you to take a look. Now, onto our last chapter of coding, everyone! .Best Practices for CI (Including Selenium Tips) ******************************************************************************* Set up CI as soon as possible for your project.:: As soon as your functional tests take more than a few seconds to run, you'll find yourself avoiding running them. Give this job to a CI server, to make sure that all your tests are being run somewhere. ((("Selenium", "best CI practices"))) ((("continuous integration (CI)", "tips"))) Optimise for fast feedback.:: CI feedback loops can be frustratingly slow. Optimising things to get results quicker is worth the effort. Run your fastest tests first, and use caches to try to minimise time spent on, for example, dependency installation. Set up screenshots and HTML dumps for failures.:: Debugging test failures is easier if you can see what the page looked like when the failure occurred. This is particularly useful for debugging CI failures, but it's also very useful for tests that you run locally. ((("screenshots"))) ((("debugging", "screenshots for"))) ((("HTML", "screenshot dumps"))) Be prepared to bump your timeouts.:: A CI server may not be as speedy as your laptop—especially if it's under load, running multiple tests at the same time. Be prepared to be even more generous with your timeouts, in order to minimise the chance of random failures. ((("flaky tests"))) Take the next step, CD (continuous deployment).:: Once we're running tests automatically, we can take the next step, which is to automate our deployments (when the tests pass). See the https://www.obeythetestinggoat.com/book/appendix_CD.html[Online Appendix: Continuous Deployment (CD)]. ((("continuous deployment (CD)"))) ******************************************************************************* ================================================ FILE: chapter_26_page_pattern.asciidoc ================================================ [[chapter_26_page_pattern]] == The Token Social Bit, the Page Pattern, [.keep-together]#and an Exercise for the Reader# //// DAVID The format of this chapter works really well! I wonder if there is a way of introducing some of this earlier in the book in two or three places, maybe in smaller ways. They could commit beforehand and then try to solve certain problems on their own, then undoing their work afterwards and replacing it with how you did it. //// ((("functional tests (FTs)", "with multiple users", secondary-sortas="multiple users", id="FTmultiple25"))) ((("functional tests (FTs)", "structuring test code", id="FTstructure25"))) Are jokes about how "everything has to be social now" slightly old hat? Yes, Harry; they were old hat 10 years ago when you started writing this book, and they're positively prehistoric now. _Irregardless_, let's say lists are often better shared. We should allow our users to collaborate on their lists with other users. Along the way, we'll improve our FTs by starting to implement something called the "page object pattern". Then, rather than showing you explicitly what to do, I'm going to let you write your unit tests and application code by yourself. Don't worry; you won't be totally on your own! I'll give an outline of the steps to take, as well as some hints and tips. But still--if you haven't already, this is the chapter where you get a chance to spread your wings. Enjoy! [role="pagebreak-before less_space"] === An FT with Multiple Users, and addCleanup ((("Page pattern", "FT with multiple user"))) Let's get started--we'll need two users for this FT: [role="sourcecode small-code"] .src/functional_tests/test_sharing.py (ch26l001) ==== [source,python] ---- from selenium import webdriver from selenium.webdriver.common.by import By from .base import FunctionalTest def quit_if_possible(browser): try: browser.quit() except: pass class SharingTest(FunctionalTest): def test_can_share_a_list_with_another_user(self): # Edith is a logged-in user self.create_pre_authenticated_session("edith@example.com") edith_browser = self.browser self.addCleanup(lambda: quit_if_possible(edith_browser)) # Her friend Onesiphorus is also hanging out on the lists site oni_browser = webdriver.Firefox() self.addCleanup(lambda: quit_if_possible(oni_browser)) self.browser = oni_browser self.create_pre_authenticated_session("onesiphorus@example.com") # Edith goes to the home page and starts a list self.browser = edith_browser self.browser.get(self.live_server_url) self.add_list_item("Get help") # She notices a "Share this list" option share_box = self.browser.find_element(By.CSS_SELECTOR, 'input[name="sharee"]') self.assertEqual( share_box.get_attribute("placeholder"), "your-friend@example.com", ) ---- ==== The interesting feature to note about this section is the `addCleanup` function, whose documentation you can find https://docs.python.org/3/library/unittest.html#unittest.TestCase.addCleanup[online]. It can be used as an alternative to the `tearDown` function as a way of cleaning up resources used during the test. It's most useful when the resource is only allocated halfway through a test, so you don't have to spend time in `tearDown` with a bunch of conditional logic designed to clean up resources that may or may not have been used by the point the test failed. `addCleanup` is run after `tearDown`, which is why we need that `try/except` formulation for `quit_if_possible`. By the time the test ends, the browser assigned to `self.browser`—whether it was `edith_browser` or `oni_browser`—will already have been quit by `tearDown()`. We'll also need to move `create_pre_authenticated_session` from _test_my_lists.py_ into _base.py_, so we can use it in more than one test. OK, let's see if that all works: [role="dofirst-ch26l002"] [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_sharing*] [...] Traceback (most recent call last): File "...goat-book/src/functional_tests/test_sharing.py", line 33, in test_can_share_a_list_with_another_user [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: input[name="sharee"]; [...] ---- Great! It seems to have made it through creating the two user sessions, and it gets onto an expected failure--there is no input for an email address of a person to share a list with on the page. Let's do a commit at this point, because we've got at least a placeholder for our FT, we've got a useful modification of the `create_pre_authenticated_session` function, and we're about to embark on a bit of an FT refactor: [subs="specialcharacters,quotes"] ---- $ *git add src/functional_tests* $ *git commit -m "New FT for sharing, move session creation stuff to base"* ---- === The Page Pattern ((("Page pattern", "reducing duplication with", id="POPduplic25"))) ((("duplication, eliminating", id="dup25"))) Before we go any further, I want to show an alternative method for reducing duplication in your FTs, called https://www.selenium.dev/documentation/test_practices/encouraged/page_object_models["page objects"]. We've already built several helper methods for our FTs—including `add_list_item`, which we've used here—but if we just keep adding more and more, it's going to get very crowded. I've worked on a base FT class that was over 1,500 lines long, and that got pretty unwieldy. [role="pagebreak-before"] Page objects are an alternative that encourage us to store all the information and helper methods about the different types of pages on our site in a single place. Let's see how that might look for our site, starting with a class to represent any lists page: [role="sourcecode small-code"] .src/functional_tests/list_page.py ==== [source,python] ---- from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from .base import wait class ListPage: def __init__(self, test): self.test = test # <1> def get_table_rows(self): # <3> return self.test.browser.find_elements(By.CSS_SELECTOR, "#id_list_table tr") @wait def wait_for_row_in_list_table(self, item_text, item_number): # <2> expected_row_text = f"{item_number}: {item_text}" rows = self.get_table_rows() self.test.assertIn(expected_row_text, [row.text for row in rows]) def get_item_input_box(self): # <2> return self.test.browser.find_element(By.ID, "id_text") def add_list_item(self, item_text): # <2> new_item_no = len(self.get_table_rows()) + 1 self.get_item_input_box().send_keys(item_text) self.get_item_input_box().send_keys(Keys.ENTER) self.wait_for_row_in_list_table(item_text, new_item_no) return self # <4> ---- ==== //003 <1> It's initialised with an object that represents the current test. That gives us the ability to make assertions, access the browser instance via `self.test.browser`, and use the `self.test.wait_for` function. <2> I've copied across some of the existing helper methods from _base.py_, but I've tweaked them slightly... <3> For example, this new method is used in the new versions of the old helper methods. <4> Returning `self` is just a convenience. It enables https://oreil.ly/I1Sr7[method chaining], which we'll see in action immediately. [role="pagebreak-before"] Let's see how to use it in our test: [role="sourcecode"] .src/functional_tests/test_sharing.py (ch26l004) ==== [source,python] ---- from .list_page import ListPage [...] # Edith goes to the home page and starts a list self.browser = edith_browser self.browser.get(self.live_server_url) list_page = ListPage(self).add_list_item("Get help") ---- ==== Let's continue rewriting our test, using the page object whenever we want to access elements from the lists page: [role="sourcecode"] .src/functional_tests/test_sharing.py (ch26l008) ==== [source,python] ---- # She notices a "Share this list" option share_box = list_page.get_share_box() self.assertEqual( share_box.get_attribute("placeholder"), "your-friend@example.com", ) # She shares her list. # The page updates to say that it's shared with Onesiphorus: list_page.share_list_with("onesiphorus@example.com") ---- ==== We add the following three functions to our `ListPage`: [role="sourcecode"] .src/functional_tests/list_page.py (ch26l009) ==== [source,python] ---- def get_share_box(self): return self.test.browser.find_element( By.CSS_SELECTOR, 'input[name="sharee"]', ) def get_shared_with_list(self): return self.test.browser.find_elements( By.CSS_SELECTOR, ".list-sharee", ) def share_list_with(self, email): self.get_share_box().send_keys(email) self.get_share_box().send_keys(Keys.ENTER) self.test.wait_for( lambda: self.test.assertIn( email, [item.text for item in self.get_shared_with_list()] ) ) ---- ==== The idea behind the page pattern is that it should capture all the information about a particular page in your site. That way, if you later want to go and make changes to that page--even just simple tweaks to its HTML layout--you'll have a single place to adjust your functional tests, rather than having to dig through dozens of FTs. The next step would be to pursue the FT refactor through our other tests. I'm not going to show that here, but it's something you could do for practice, to get a feel for what the trade-offs are like between "don't repeat yourself" (DRY) and test readability... ((("", startref="POPduplic25"))) ((("", startref="dup25"))) === Extend the FT to a Second User, and the "My Lists" Page ((("Page pattern", "adding a second Page object"))) Let's spec out just a little more detail of what we want our sharing user story to be. Edith has seen on her list page that the list is now "shared with" Onesiphorus, and then we can have Onesiphorus log in and see the list on his "My lists" page—maybe in a section called "lists shared with me": [role="sourcecode"] .src/functional_tests/test_sharing.py (ch26l010) ==== [source,python] ---- from .my_lists_page import MyListsPage [...] list_page.share_list_with("onesiphorus@example.com") # Onesiphorus now goes to the lists page with his browser self.browser = oni_browser MyListsPage(self).go_to_my_lists_page("onesiphorus@example.com") # He sees Edith's list in there! self.browser.find_element(By.LINK_TEXT, "Get help").click() ---- ==== [role="pagebreak-before"] That means another function in our `MyListsPage` class: [role="sourcecode"] .src/functional_tests/my_lists_page.py (ch26l011) ==== [source,python] ---- from selenium.webdriver.common.by import By class MyListsPage: def __init__(self, test): self.test = test def go_to_my_lists_page(self, email): self.test.browser.get(self.test.live_server_url) self.test.browser.find_element(By.LINK_TEXT, "My lists").click() self.test.wait_for( lambda: self.test.assertIn( email, self.test.browser.find_element(By.TAG_NAME, "h1").text, ) ) return self ---- ==== Once again, this is a function that would be good to carry across into _test_my_lists.py_, along with maybe a `MyListsPage` object. In the meantime, Onesiphorus can also add things to the list: [role="sourcecode"] .src/functional_tests/test_sharing.py (ch26l012) ==== [source,python] ---- # On the list page, Onesiphorus can see says that it's Edith's list self.wait_for( lambda: self.assertEqual(list_page.get_list_owner(), "edith@example.com") ) # He adds an item to the list list_page.add_list_item("Hi Edith!") # When Edith refreshes the page, she sees Onesiphorus's addition self.browser = edith_browser self.browser.refresh() list_page.wait_for_row_in_list_table("Hi Edith!", 2) ---- ==== That's another addition to our `ListPage` object: [role="sourcecode"] .src/functional_tests/list_page.py (ch26l013) ==== [source,python] ---- class ListPage: [...] def get_list_owner(self): return self.test.browser.find_element(By.ID, "id_list_owner").text ---- ==== [role="pagebreak-before"] It's long past time to run the FT and check if all of this works! [subs="specialcharacters,macros"] ---- $ pass:quotes[*python src/manage.py test functional_tests.test_sharing*] [...] File "...goat-book/src/functional_tests/test_sharing.py", line 35, in test_can_share_a_list_with_another_user share_box = list_page.get_share_box() [...] return self.test.browser.find_element( ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ By.CSS_SELECTOR, ^^^^^^^^^^^^^^^^ 'input[name="sharee"]', ^^^^^^^^^^^^^^^^^^^^^^^ [...] selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: input[name="sharee"]; [...] ---- That's the expected failure; we don't have an input for email addresses of people to share with. Let's do a commit: [subs="specialcharacters,quotes"] ---- $ *git add src/functional_tests* $ *git commit -m "Create Page objects for list pages, use in sharing FT"* ---- === An Exercise for the Reader [quote, Iain H. (reader)] ______________________________________________________________ I probably didn’t _really_ understand what I was doing until after having completed the "exercise for the reader" in the page pattern chapter. ______________________________________________________________ ((("Page pattern", "practical exercise"))) There's nothing that cements learning like taking the training wheels off, and getting something working on your own, so I hope you'll give this a go. By this point in the book, you should have all the elements you need to test-drive this new feature, from the outside in. The FT is there to guide you, and this feature should take you down into both the views and the models layers. So, give it a go! ==== Step-by-step Guide If you'd like a bit more help, here's an outline of the steps you could take: 1. You'll need a new section in _list.html_, initially with just a form containing an input box for an email address. That should get the FT one step further. 2. Next, you'll need a view for the form to submit to. Start by defining the URL in the template—maybe something like 'lists/<list_id>/share'. 3. Then, you'll have your first unit test. It can be just enough to get a placeholder view in. You want the view to respond to POST requests and respond with a redirect back to the list page. The test could be called something like `ShareListTest.test_post_redirects_to_lists_page`. 4. You build out your placeholder view, as just a two-liner that finds a list and redirects to it. 5. You can then write a new unit test that creates a user and a list, does a POST with their email address, and checks that the user is added to `mylist.shared_with.all()` (a similar ORM usage to "My lists"). That `shared_with` attribute won't exist yet; you're going outside-in. 6. So, before you can get this test to pass, you'll have to move down to the model layer. The next test, in _test_models.py_, can check that a list has a `shared_with.add()` method that works with a user’s email address, and that `shared_with.all()` subsequently includes that user. 7. You'll then need a `ManyToManyField`. You'll probably see an error message about a clashing `related_name`, which you'll find a solution for if you look around the Django docs. 8. It will need a database migration. 9. That should get the model tests passing. Pop back up to fix the view test. 10. You may find that the redirect view test fails, because it's not sending a valid POST request. You can either choose to ignore invalid inputs, or adjust the test to send a valid POST. 11. Then, head back up to the template level; on the "My lists" page, you'll want a `<ul>` with a +for+ loop of the lists shared with the user. On the lists page, you also want to show who the list is shared with, and mention who the list owner is. Look back at the FT for the correct classes and IDs to use. You could have brief unit tests for each of these if you like, as well. 12. You might find that spinning up the site with `runserver` helps you iron out any bugs and fine-tune the layout and aesthetics. If you use a private browser session, you'll be able to log multiple users in. [role="pagebreak-before"] By the end, you might end up with something that looks like <<list-sharing-example>>. [[list-sharing-example]] .Sharing lists image::images/tdd3_2601.png["Screenshot of list sharing UI"] [role="pagebreak-before less_space"] .The Page Pattern, and the Real Exercise for the Reader ******************************************************************************* Applying DRY to your functional tests:: Once your FT suite starts to grow, you'll find different tests using similar parts of the UI. Try to avoid having constants—like the HTML IDs or classes of particular UI elements—duplicated across your FTs. ((("Don’t Repeat Yourself (DRY)"))) The page pattern:: Moving helper methods into a base `FunctionalTest` class can become unwieldy. Consider using individual page objects to hold all the logic for dealing with particular parts of your site. ((("Page pattern", "benefits of"))) An exercise for the reader:: I hope you've actually tried this out! Try to follow the outside-in method, and occasionally try things out manually if you get stuck. The real exercise for the reader, of course, is to apply TDD to your next project. I hope you'll enjoy it! ((("", startref="FTmultiple25"))) ((("", startref="FTstructure25"))) ******************************************************************************* In the next chapter, we'll wrap up with a discussion of the trade-offs in testing, and some of the considerations involved in choosing which kinds of tests to use, and when. ================================================ FILE: chapter_27_hot_lava.asciidoc ================================================ [[chapter_27_hot_lava]] == Fast Tests, Slow Tests, and Hot Lava [quote, Casey Kinsey] ______________________________________________________________ The database is hot lava! ______________________________________________________________ We've come to the end of the book, and the end of our journey with this to-do app and its tests. Let's recap our test structure so far: * We have a suite of functional tests that use Selenium to test that the whole app really works. On several occasions, the FTs have saved us from shipping broken code--whether it was broken CSS, a broken database due to filesystem permissions, or broken email integration. * And we have a suite of unit tests that use Django test client, enabling us to test-drive our code for models, forms, views, URLs, and even (to some extent) templates. They've enabled us to build the app incrementally, to refactor with confidence, and they've supported a fast unit-test/code cycle. * We've also spent a good bit of time on our infrastructure, packaging up our app with Docker for ease of deployment, and we've set up a CI pipeline to run our tests automatically on push. However, being a simple app that could fit in a book, there are inevitably some limitations and simplifications in our approach. In this chapter, I'd like to talk about how to carry your testing principles forward, as you move into larger, more complex applications in the real world. Let's find out why someone might say that the database is hot lava!((("tests", "desiderata for effective tests"))) === Why Do We Test? Our Desiderata for Effective Tests At https://testdesiderata.com[testdesiderata.com], Kent Beck and Kelly Sutton outline several desiderata (desirable characteristics) for tests, as outlined in <<test_desiderata>>: [[test_desiderata]] .Test desiderata |=== | *Isolated*: Tests should return the same results regardless of the order in which they are run. | *Composable*: We should be able to test different dimensions of variability separately and combine the results. | *Deterministic*: If nothing changes, the test results shouldn’t change. | *Fast*: Tests should run quickly. | *Writable*: Tests should be cheap to write, relative to the cost of the code being tested. | *Readable*: Tests should be comprehensible for readers, invoking the motivation for writing the particular test. | *Behavioural*: Tests should be sensitive to changes in the behaviour of the code under test. If the behaviour changes, the test result should change. | *Structure-agnostic*: Tests should not change their results if the structure of the code changes. | *Automated*: Tests should run without human intervention. | *Specific*: If a test fails, the cause of the failure should be obvious. | *Predictive*: If the tests all pass, then the code under test should be suitable for production. | *Inspiring*: Passing the tests should inspire confidence. |=== We've talked about almost all of these desiderata in the book: we talked about _isolation_ when we switched to using the Django test runner. We talked about _composability_ when discussing the car factory example in <<chapter_21_mocking_2>>. We talked about tests being _readable_ when we talked about the given-when-then structure and when implementing helper methods in our FTs. We talked about testing _behaviour_ rather than implementation at several points, including in the mocking chapters. We talked about _structure_ in the forms chapters, when we showed that the higher-level views tests enabled us to refactor more freely than the lower-level forms tests. We've talked about splitting up our tests to have fewer assertions to make them more _specific_. We talked about _determinism_ when discussing flaky tests and the use of `wait_for()` in our FTs, for example, as well as in the production debugging chapter. And in this chapter, we're going to talk primarily about _speed_, and about what makes tests _inspiring_. But first, it's worth taking a step back from the list in <<test_desiderata>>, and asking: "What do we want from our tests?" [role="pagebreak-before less_space"] ==== Confidence and Correctness (Preventing Regression) A fundamental part of programming is that, now and again, you need to check whether "it works".((("regression", "preventing"))) Automated testing is the solution to the problem that checking things manually can quickly become tedious and be unreliable. We want our tests to tell us that our code works—both at the low level of individual functions or classes, and at the higher level of "does it all hang together?" ==== A Productive Workflow Our tests need to be fast enough to write, but more importantly, fast to run. We want to get into a smooth, productive workflow, and try to enter that holy credo of programmers—the "flow state". Beyond that, we want our tests to take some of the stress out of programming, encouraging us to work in small increments, with frequent bursts of dopamine from seeing green tests. ==== Driving Better Design And our tests should help us to write _better_ code: first, by enabling fearless refactoring and, second, by giving us feedback on the design of our code.((("code design, better, tests driving"))) Writing the tests first lets us think about our API from the outside in, before we write it--and we've seen that. But in this chapter, we'll also talk about the potential for tests to give you feedback on your design in more subtle ways. As we'll see, designing code to be more testable often leads to code that has clearly identified dependencies, and is more modular and more decoupled. As we continuously think about what kinds of tests to write, we are trying to achieve the optimum balance of these different desiderata. === Were Our Unit Tests Integration Tests All Along? [.keep-together]#What Is That Warm# Glow Coming from the Database? ((("integration tests", "versus unit tests", secondary-sortas="unit"))) ((("unit tests", "versus integration tests", secondary-sortas="integration"))) Almost all of the "unit" tests in the book perhaps should have been called _integration_ tests, because they all rely on Django's test runner, which gives us a real database to talk to. Many also use the Django test client, which does a lot of magic with the middleware layers that sit between requests. The end result is that our tests are heavily integrated with both the database and Django itself. [role="pagebreak-before less_space"] ==== We've Been in the "Sweet Spot" Now, actually, this has been a pretty good thing for us so far. We're very much in the "sweet spot" of Django's testing tools.((("Django framework", "sweet spot of Django's testing tools"))) Our unit tests have been fast enough to enable a smooth workflow, and they've given us a strong reassurance that our application really works—from the models all the way up to the templates. By allying them with a small-ish suite of functional tests, we've got a lot of confidence in our code. And we've been able to use them to get at least a bit of feedback on our design, and to enable lots of refactoring. ==== What Is a "True" Unit Test? Does it Matter? But people will often tell you that a "true" unit test should be more isolated. It's meant to test a single "unit" of software, and your database "should" be outside of that.((("unit tests", "true unit tests"))) Why do they say that (other than for the smugness they get from should-ing us)? As you can tell, I think the argument from _definitions_ is a bit of a red herring. But you might hear instead, "the database is hot lava!"—as Casey Kinsey put it in a memorable DjangoCon talk. There is real feeling and real experience behind these comments. What are people getting at? ==== Integration and Functional Tests Get Slower Over Time The problem is that, as your application and codebase grow, involving the database in every single test starts to carry an unacceptable cost—in terms of execution speed.((("database testing", "functional and integration tests getting slower")))((("integration tests", "involving the database, getting slower")))((("functional tests (FTs)", "involving the database, getting slower"))) Casey's company, for example, was struggling with test suites that took several hours. At PythonAnywhere, our functional test suite didn't just rely on the database; it would spin up a full test cluster of six virtual machines. A full run used to take at least 12 hours, and we'd have to wait overnight for our results. That was one of the least productive parts of an otherwise extraordinary workflow. At Kraken, the full test suite does only take about 45 minutes, which is not bad for nearly 10 million lines of code, but that's only thanks to a quite frankly ridiculous level of parallelisation and associated expenditure on CI. We're now spending a lot of effort on trying to move more of our unit tests to being "true" unit tests. The problem is that these things don't scale linearly. The more database tables you have, the more relationships between them, and that starts to increase geometrically. So you can see why, over time, these kinds of tests are going to fail to meet our desiderata because they're too slow to enable a productive workflow and a fast enough feedback cycle. NOTE: Don't take it from me! Gary Bernhardt, a legend in both the Ruby and Python testing communities, has a talk simply called https://oreil.ly/ga28I["Fast Test, Slow Test"], which is a great tour of the problems I'm discussing here. .The Holy Flow State ******************************************************************************* Thinking sociologically for a moment, we programmers have our own culture and our own "religion" in a way.((("flow, holy state of")))((("holy flow state"))) It has many congregations within it—such as the cult of TDD, to which you are now initiated. There are the followers of Vim and the heretics of Emacs. But one thing we all agree on—one particular spiritual practice, our own transcendental meditation—is the holy flow state. That feeling of pure focus, of concentration, where hours pass like no time at all, where code flows naturally from our fingers, where problems are just tricky enough to be interesting but not so hard that they defeat us... There is absolutely no hope of achieving flow if you spend your time waiting for a slow test suite to run. Anything longer than a few seconds and you're going to let your attention wander, you context-switch, and the flow state is gone. And the flow state is a fragile dream; once it's gone, it takes a long time to come back.footnote:[ Some people say it takes at least 15 minutes to get back into the flow state. In my experience, that's overblown, and I sometimes wonder if it's thanks to TDD. I think TDD reduces the cognitive load of programming. By breaking our work down into small increments, by simplifying our thinking—"What's the current failing test? What's the simplest code I can write to make it pass?"—it's often actually quite easy to context-switch back into coding. Maybe it's less true for the times when we're doing design work and thinking about what the abstractions in our code should be though. But also there's absolutely no hope for you if you've started scrolling social media while waiting for your tests to finish. See you in 20 minutes to an hour!] ******************************************************************************* ==== We're Not Getting the Full Potential Benefits of Testing TDD experts often say, "It should be called test-driven _design_, not test-driven development". What do they mean by that?((("tests", "giving maximum feedback on code design"))) We have definitely seen a bit of the positive influence of TDD on our design. We've talked about how our tests are the first clients of any API we create, and we've talked about the benefits of "programming by wishful thinking" and outside-in. But there's more to it. These same TDD experts also often say that you should "listen to your tests". Unless you've read the https://www.obeythetestinggoat.com/book/appendix_purist_unit_tests.html[online Appendix: Test Isolation and "Listening to Your Tests"], that will still sound like a bit of a mystery. So, how can we get to a position where our tests are giving us maximum feedback on our design? === The Ideal of the Test Pyramid I know I said I didn't want to get bogged down ((("test pyramid")))into arguments based on definitions, but let's set out the way people normally think about these three types of tests: Functional/end-to-end tests:: FTs check that the system works end-to-end, exercising the full stack ((("functional tests (FTs)")))of the application, including all dependencies and connected external systems. An FT is the ultimate test that it all hangs together, and that things are "really" going to work. // CSANAD: I find the expression 'things are "really" going to work' too vague. // I would rather mention User Stories here, since they very often can be turned into // functional/end-to-end tests: they are worded similarly and they both cover specific // functionalities that are valuable for a given user (edit: well, you talk // about this below, under "On Acceptance Tests"). // Furthermore, I would maybe give an example for each: // "Francis starts a new list by entering a new item." Integration tests:: The purpose of an integration test should be to check that the code you write is integrated correctly with some "external" system or dependency.((("integration tests"))) // CSANAD: this one is more tricky to find integration tests for, since we // didn't create separate 'integration tests'. Maybe an example could be // checking whether Bootstrap is loaded correctly, or perhaps the email. // Something like that would be helpful in my opinion, especially because // we promised in Chapter 05 ("Unit Tests Versus Integration Tests, and the Database") // that we would further clarify the difference. (True) unit tests:: Unit tests are the lowest-level tests, and are supposed to test a single "unit" of code or behaviour. The ideal unit test is fully isolated((("unit tests"))) from everything external to the unit under test, such that changes to things outside cannot break the test. // CSANAD: I was trying to find an example of a pure unit test. I recall // we may have had some helper function at some point, for which there was // no need to use Django's TestCase but I can't find it. Maybe I'm // remembering wrong. The canonical advice is that you should aim to have the majority of your tests be unit tests, with a smaller number of integration tests, and an even smaller number of functional tests—as in the classic "test pyramid" of <<test_pyramid>>. // CSANAD: in the HTML, it read: "as in the classic 'Test Pyramid' of The Test Pyramid". [[test_pyramid]] .The test pyramid image::images/tdd3_2701.png["A Pyramid shape, with a large bottom layer of unit tests, a medium layer of integration tests, and a small peak of FTs"] Bottom layer: unit tests (the vast majority):: These isolated tests are fast and they pinpoint failures precisely. We want these to cover the majority of our functionality, and the entirety of our business logic if possible. Middle layer: integration tests (a significant portion):: In an ideal world, these are reserved purely for testing the interactions between our code and external systems—like the database, or even (arguably) Django itself. These are slower, but they give us the confidence that our components work together. Top layer: a minimal set of functional/end-to-end tests:: These tests are there to give us the ultimate reassurance that everything works end-to-end and top-to-bottom. But because they are the slowest and most brittle, we want as few of them as possible. // CSANAD: I think explaining the layers after having explained the types of // tests just above it, seems a little redundant. I wonder if we should combine // them. [[acceptance_tests]] .On Acceptance Tests ******************************************************************************* What about "acceptance tests"? You might have heard this term bandied about. Often, people use it to mean the same thing as functional tests or end-to-end tests.((("acceptance tests"))) But, as taught to me by one of the legends of quality assurance at MADE.com (Hi, Marta!), _any_ kind of test can be an acceptance test if it maps onto one of your acceptance criteria. The point of an acceptance test is to validate a piece of behaviour that's important to the user. In our application, that's how we've been thinking about our FTs. But, ultimately, using FTs to test every single piece of user-relevant functionality is not sustainable. We need to figure out ways to have our integration tests and unit tests do the work of verifying user-visible behaviour, understood at the right level of abstraction. Learn more in https://oreil.ly/Pf8Np[the video on acceptance test-driven development (ATDD)] by Dave Farley. ******************************************************************************* === Avoiding Mock Hell Well that's all very well, Harry (you might say), but our current test setup is nothing like this!((("mocks", "avoiding mock hell"))) How do we get there from _here_? We've seen how to use mocks to isolate ourselves from external dependencies. Are they the solution then? As I was at pains to point out the mocking chapters, the use of mocks comes with painful trade-offs: * They make tests harder to read and write. * They leave your tests tightly coupled to implementation details. * As a result, they tend to impede refactoring. * And, in the extreme, you can sometimes end up with mocks testing mocks, almost entirely disconnected from what the code actually does. Ed Jung calls this https://oreil.ly/sm16H[Mock Hell]. This isn't to say that mocks are always bad! But just that, from experience, attempting to use them as your primary tool for decoupling // CSANAD: I think we could actually argue that by using mocks, we // accept that the code is tightly coupled with its dependencies. your tests from external dependencies is not a viable solution; it carries costs that often outweigh the benefits. NOTE: I'm glossing over the use of mocks in a London-school approach to TDD. See the https://www.obeythetestinggoat.com/book/appendix_purist_unit_tests.html[Online Appendix: Test Isolation and "Listening to Your Tests"]. === The Actual Solutions Are Architectural The actual solution to the problem isn't obvious from where we're standing. It lies in rethinking the architecture of our application.((("architectures of applications"))) In brief, if we can _decouple_ the core business logic of our application from its dependencies, then we can write true, isolated unit tests for it that do not depend on those, um, dependencies. ((("business logic, decoupling from dependencies")))((("dependencies", "decoupling business logic from"))) Integration tests are most necessary at the _boundaries_ of a system--at the points where our code integrates with external systems—like the database, filesystem, network, or a UI.((("boundaries between system components", "integration tests and"))) Similarly, it's at the boundaries that the downsides of test isolation and mocks are at their worst, because it's at the boundaries that you're most likely to be annoyed if your tests are tightly coupled to an implementation, or to need more reassurance that things are integrated properly. Conversely, code at the _core_ of our application--code that's purely concerned with our business domain and business rules, code that's entirely under our control--has no intrinsic need for integration tests.((("core application code"))) So, the way to get what we want is to minimise the amount of our code that has to deal with boundaries. Then we test our core business logic with unit tests, and test the rest with integration and functional tests. But how do we do that? [role="pagebreak-before less_space"] ==== Ports and Adapters/Hexagonal/Onion/Clean Architecture The classic solutions to this problem from the object-oriented world come under different names, but they're all variations of the same trick: identifying the boundaries, creating an interface to define those boundaries, and then using that interface at test time to swap out fake versions of your real dependencies.((("architectures of applications", "Hexagonal/Clean/Onion architectures")))((("adapters, ports and")))((("object-oriented architecture, ports and adapters")))((("Hexagonal Architecture pattern")))((("Clean Architecture pattern")))((("Onion Architecture pattern"))) Steve Freeman and Nat Pryce, in their book <<GOOSGBT, _Growing Object-Oriented Software, Guided by Tests_>>, call this approach "Ports and Adapters" (see <<ports-and-adapters>>). [[ports-and-adapters]] .Ports and Adapters (diagram by Nat Pryce) image::images/tdd3_2702.png["Illustration of ports and adapaters architecture, with isolated core and integration points"] // CSANAD: I haven't found the original diagram by Nat Pryce. I would recommend // maybe a making the next header "Functional Core, Imperative Shell" formatted // differently, making it more obvious that it's an explanation of the diagram. // Or, we could just add a "Legend" under the diagram, explaining what the // nodes, arrows and different shades of the layers depict. This pattern, or variations on it, are known as "Hexagonal Architecture" (by Alistair Cockburn), "Clean Architecture" (by Robert C. Martin, aka Uncle Bob), or "Onion Architecture" (by Jeffrey Palermo). .Time for a Plug! Read More in "Cosmic Python" ******************************************************************************* At the end of the process of writing this book (the first time around) I realised that I was going to have to learn about these architectural solutions, and it was at MADE.com that I met Bob Gregory who was to become my coauthor. There, we explored "ports and adapters" and related architectures, which were quite rare at the time in the Python world. So if you'd like a take on these architectural patterns with a Pythonic twist, check out https://www.cosmicpython.com[_Architecture Patterns with Python_], which we subtitled "Cosmic Python", because "cosmos" is the opposite of "chaos", in Greek. ******************************************************************************* ==== Functional Core, Imperative Shell Gary Bernhardt pushes this further, recommending an architecture he calls "Functional Core, Imperative Shell", whereby the "shell" of the application((("shell of application, Imperative Shell pattern"))) (the place where interaction with boundaries happens) follows the imperative programming paradigm, and can be tested by integration tests, functional tests, or even (gasp!) not at all (if it's kept minimal enough). ((("Functional Core, Imperative Shell Architecture pattern"))) ((("architectures of applications", "Functional Core, Imperative Shell"))) ((("core application code", "functional core, imperative shell"))) But the core of the application is actually written following the functional programming paradigm (complete with the "no side effects" corollary), which allows fully isolated, "pure" unit tests—_without any mocks or fakes_. Check out Gary's presentation titled https://oreil.ly/of8pU["Boundaries"] for more on this approach. ==== The Central Conceit: These Architectures Are "Better" These patterns do not come for free! Introducing the extra indirection and abstraction can add complexity to your code.((("architectures of applications", "upside of architectural patterns"))) In fact, the creator of Ruby on Rails, David Heinemeier Hansson (DHH), has a famous blog post where he describes these architectures as https://dhh.dk/2014/test-induced-design-damage.html[test-induced design damage]. That post eventually led to quite a thoughtful and https://martinfowler.com/articles/is-tdd-dead[nuanced discussion] between DHH, Martin Fowler, and Kent Beck. Like any technique, these patterns can be misused, but I wanted to make the case for their upside: by making our software more testable, we also make it more modular and maintainable. We are forced to clearly separate our concerns, and we make it easier to do things like upgrade our infrastructure when we need to. This is the place where the "improved design" desiderata comes in. TIP: Making our software more testable also often leads to a better design. .Testing in Production ******************************************************************************* I should also make brief mention of the power of observability and monitoring. Kent Beck tells a story about his first few weeks at Facebook, ((("observability and monitoring"))) ((("production, testing in"))) when one of the first tests he wrote turned out to be flaky in the build. Someone just deleted it. Shocked and asking why, he was told, "We know production is up. Your test is just producing noise; we don't need it". footnote:[There's a https://oreil.ly/jhXg8[transcript of this story].] Facebook has such confidence in its production monitoring and observability that it can provide them with most of the feedback they need about whether the system is working. Not everywhere is Facebook! But it's a good indication that automated tests aren't the be-all and end-all. ******************************************************************************* === The Hardest Part: Knowing When to Make the Switch image::images/tdd3_2703.png["An illustration of a frog being slowly boiled in a pan"] When is it time to hop out? For small- to medium-sized applications, as we've seen, the Django test runner and the integration tests it encourages us to write are just fine. The problem is knowing when it's time to make the change to a more decoupled architecture, and to start striving explicitly for the test pyramid.((("test pyramid", "striving explicitly for"))) It's hard to give good advice here, as I've only experienced environments where either someone else made the decision before I joined, or the company is already struggling with a point where it's (at least arguably) too late. One thing to bear in mind, though, is that the longer you leave it, the harder it is. Another is that because the pain is only going to set in gradually, like the apocryphal boiled frogs, you're unlikely to notice until you're past the "perfect" moment to switch. And on top of that, it's _never_ going to be a convenient time to switch. This is one of those things, like tech debt, that is always going to struggle to justify itself in the face of more immediate priorities. So, perhaps one strategy would be an Odysseus pact: tie yourself to the mast, and make a commitment--while the tests are still fast--to set a "red line" for when to switch. For example, "If the tests ever take more than 10 seconds to run locally, then it's time to rethink the architecture". I'm not saying 10 seconds is the right number, by the way. I know plenty of people who are perfectly happy to wait 30 seconds. And I know Gary Bernhardt, for one, would get very nervous at a test suite that takes more than 100 milliseconds. But I think the idea of drawing that line in the sand, wherever it is, _before_ you get there, might be a good way to fight the "boiled frog" problem. Failing all of that, if the best time to make the change was "ages ago", then the second best time is "right now". Other than that, I can only wish you good luck, and hope that by warning you of the dangers, you'll keep an eye on your test suite and spot the problems before they get too large. Happy testing! === Wrap-Up In this book, I've been able to show you how to use TDD, and have talked a bit about why we do it and what makes a good test. But we're inevitably limited by the scope of the project. What that means is that some of the more advanced uses of TDD, particularly the interplay between testing and architecture, have been beyond the scope of this book. But I hope that this chapter has been a bit of a guide to find your way around that topic as your career progresses. [role="pagebreak-before less_space"] ==== Further Reading A few places to go for((("Test-Driven Development (TDD)", "resources for further reading"))) more inspiration: "Fast Test, Slow Test" and "Boundaries":: Gary Bernhardt's talks from Pycon https://oreil.ly/6OJKP[2012] and https://oreil.ly/aw-rF[2013]. His http://www.destroyallsoftware.com[screencasts] are also well worth a look. Integration tests are a scam:: J.B. Rainsberger has a https://oreil.ly/j4ck-[famous rant] about the way integration tests will ruin your life.footnote:[ Rainsberger actually distinguishes "integrated" tests from "integration" tests: an integrated test is any test that's not fully isolated from things outside the unit under test.] Then check out a couple of follow-up posts, particularly http://www.jbrains.ca/permalink/using-integration-tests-mindfully-a-case-study[the defence of acceptance tests], and http://www.jbrains.ca/permalink/part-2-some-hidden-costs-of-integration-tests[the analysis of how slow tests kill productivity]. ((("integration tests", "benefits and drawbacks of"))) Ports and Adapters:: Steve Freeman and Nat Pryce wrote about this in <<GOOSGBT, their book>>. You can also catch a good discussion in http://vimeo.com/83960706[Steve's talk]. See also https://oreil.ly/2UExy[Uncle Bob's description of the clean architecture], and https://alistair.cockburn.us/hexagonal-architecture[Alistair Cockburn coining the term "Hexagonal Architecture"]. The test-double testing wiki:: Justin Searls' online resource is a great source of definitions and discussions on testing pros and cons, and arrives at its own conclusions of the right way to do things: https://github.com/testdouble/contributing-tests/wiki/Test-Driven-Development[testing wiki]. Fowler on unit tests:: Martin Fowler (author of _Refactoring_) offers a http://martinfowler.com/bliki/UnitTest.html[balanced and pragmatic tour] of what unit tests are, and of the trade-offs around speed. A take from the world of functional programming:: _Grokking Simplicity_ by Eric Normand explores the idea of "Functional Core, Imperative Shell". Don't worry; you don't need a crazy functional programming language like Haskell or Clojure to understand it—it's written in perfectly sensible JavaScript. // CSANAD: Shouldn't we provide a link to this book too? // https://www.oreilly.com/library/view/grokking-simplicity/9781617296208/ // O'Reilly resources usually have a different kind of link though. Happy testing! ================================================ FILE: check-links.py ================================================ #!python import asyncio import sys from pathlib import Path import httpx from bs4 import BeautifulSoup def find_links(path): html_content = Path(path).read_text() soup = BeautifulSoup(html_content, "html.parser") links = soup.find_all("a", href=True) return [ link["href"] for link in links if link["href"].startswith("http") and "localhost" not in link["href"] and "127.0.0.1" not in link["href"] ] async def check_url(url, client): print(f"Checking {url}") try: await client.head(url, follow_redirects=True, timeout=5) except httpx.RequestError as e: print(f"Link {url} errored {e}") return False except httpx.HTTPStatusError as e: print(f"Link {url} errored {e.response.status_code}") return False return True async def main(path): links = find_links(path) async with httpx.AsyncClient() as client: tasks = [check_url(link, client) for link in links] results = await asyncio.gather(*tasks) success_count = sum(results) failure_count = len(links) - success_count print( f"Checked {len(links)} links, {success_count} succeeded, {failure_count} failed." ) if failure_count > 0: sys.exit(1) if __name__ == "__main__": path = sys.argv[1] if len(sys.argv) > 1 else "book.html" asyncio.run(main(path)) ================================================ FILE: coderay-asciidoctor.css ================================================ /*! Stylesheet for CodeRay to loosely match GitHub themes | MIT License */ pre.CodeRay{background:#f7f7f8} .CodeRay .line-numbers{border-right:1px solid;opacity:.35;padding:0 .5em 0 0;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none} .CodeRay span.line-numbers{display:inline-block;margin-right:.75em} .CodeRay .line-numbers strong{color:#000} table.CodeRay{border-collapse:separate;border:0;margin-bottom:0;background:none} table.CodeRay td{vertical-align:top;line-height:inherit} table.CodeRay td.line-numbers{text-align:right} table.CodeRay td.code{padding:0 0 0 .75em} .CodeRay .debug{color:#fff!important;background:navy!important} .CodeRay .annotation{color:#007} .CodeRay .attribute-name{color:navy} .CodeRay .attribute-value{color:#700} .CodeRay .binary{color:#509} .CodeRay .comment{color:#998;font-style:italic} .CodeRay .char{color:#04d} .CodeRay .char .content{color:#04d} .CodeRay .char .delimiter{color:#039} .CodeRay .class{color:#458;font-weight:bold} .CodeRay .complex{color:#a08} .CodeRay .constant,.CodeRay .predefined-constant{color:teal} .CodeRay .color{color:#099} .CodeRay .class-variable{color:#369} .CodeRay .decorator{color:#b0b} .CodeRay .definition{color:#099} .CodeRay .delimiter{color:#000} .CodeRay .doc{color:#970} .CodeRay .doctype{color:#34b} .CodeRay .doc-string{color:#d42} .CodeRay .escape{color:#666} .CodeRay .entity{color:#800} .CodeRay .error{color:#808} .CodeRay .exception{color:inherit} .CodeRay .filename{color:#099} .CodeRay .function{color:#900;font-weight:bold} .CodeRay .global-variable{color:teal} .CodeRay .hex{color:#058} .CodeRay .integer,.CodeRay .float{color:#099} .CodeRay .include{color:#555} .CodeRay .inline{color:#000} .CodeRay .inline .inline{background:#ccc} .CodeRay .inline .inline .inline{background:#bbb} .CodeRay .inline .inline-delimiter{color:#d14} .CodeRay .inline-delimiter{color:#d14} .CodeRay .important{color:#555;font-weight:bold} .CodeRay .interpreted{color:#b2b} .CodeRay .instance-variable{color:teal} .CodeRay .label{color:#970} .CodeRay .local-variable{color:#963} .CodeRay .octal{color:#40e} .CodeRay .predefined{color:#369} .CodeRay .preprocessor{color:#579} .CodeRay .pseudo-class{color:#555} .CodeRay .directive{font-weight:bold} .CodeRay .type{font-weight:bold} .CodeRay .predefined-type{color:inherit} .CodeRay .reserved,.CodeRay .keyword{color:#000;font-weight:bold} .CodeRay .key{color:#808} .CodeRay .key .delimiter{color:#606} .CodeRay .key .char{color:#80f} .CodeRay .value{color:#088} .CodeRay .regexp .delimiter{color:#808} .CodeRay .regexp .content{color:#808} .CodeRay .regexp .modifier{color:#808} .CodeRay .regexp .char{color:#d14} .CodeRay .regexp .function{color:#404;font-weight:bold} .CodeRay .string{color:#d20} .CodeRay .string .string .string{background:#ffd0d0} .CodeRay .string .content{color:#d14} .CodeRay .string .char{color:#d14} .CodeRay .string .delimiter{color:#d14} .CodeRay .shell{color:#d14} .CodeRay .shell .delimiter{color:#d14} .CodeRay .symbol{color:#990073} .CodeRay .symbol .content{color:#a60} .CodeRay .symbol .delimiter{color:#630} .CodeRay .tag{color:teal} .CodeRay .tag-special{color:#d70} .CodeRay .variable{color:#036} .CodeRay .insert{background:#afa} .CodeRay .delete{background:#faa} .CodeRay .change{color:#aaf;background:#007} .CodeRay .head{color:#f8f;background:#505} .CodeRay .insert .insert{color:#080} .CodeRay .delete .delete{color:#800} .CodeRay .change .change{color:#66f} .CodeRay .head .head{color:#f4f} ================================================ FILE: colo.html ================================================ <section id="colophon" data-type="colophon" xmlns="http://www.w3.org/1999/xhtml"> <h1>Colophon</h1> <p>The animal on the cover of <em>Test-Driven Development with Python</em> is a cashmere goat. Though all goats can produce a cashmere undercoat, only those goats selectively bred to produce cashmere in commercially viable amounts are typically considered “cashmere goats”. Cashmere goats thus belong to the domestic goat species <em>Capra hircus</em>.</p> <p>The exceptionally fine, soft hair of the undercoat of a cashmere goat grows alongside an outercoat of coarser hair as part of the goat’s double fleece. The cashmere undercoat appears in winter to supplement the protection offered by the outercoat, called "guard hair". The crimped quality of cashmere hair in the undercoat accounts for its lightweight yet effective insulation properties.</p> <p>The name “cashmere” is derived from the Kashmir Valley region on the Indian subcontinent, where the textile has been manufactured for thousands of years. A diminishing population of cashmere goats in modern Kashmir has led to the cessation of exports of cashmere fiber from the area. Most cashmere wool now originates in Afghanistan, Iran, Outer Mongolia, India, and—predominantly—China.</p> <p>Cashmere goats grow hair of varying colors and color combinations. Both males and females have horns, which serve to keep the animals cool in summer and provide the goats’ owners with effective handles during farming activities.</p> <p>Many of the animals on O'Reilly covers are endangered; all of them are important to the world.</p> <p>The cover image is from Wood's Animate Creation. The series design is by Edie Freedman, Ellie Volckhausen, and Karen Montgomery. The cover fonts are Gilroy Semibold and Guardian Sans. The text font is Adobe Minion Pro; the heading font is Adobe Myriad Condensed; the code font is Dalton Maag's Ubuntu Mono; and the Scratchpad font is ORAHand-Medium.</p> </section> ================================================ FILE: copy_html_to_site_and_print_toc.py ================================================ #!/usr/bin/env python import re import subprocess from collections.abc import Iterator from pathlib import Path from typing import NamedTuple from lxml import html DEST = Path("~/workspace/www.obeythetestinggoat.com/content/book").expanduser() EXCLUDE = [ "titlepage.html", "copyright.html", "toc.html", "ix.html", "author_bio.html", "colo.html", ] ADOC_INCLUDE_RE = re.compile(r"include::(.+.asciidoc)\[\]") def _chapters(): for l in Path("book.asciidoc").read_text().splitlines(): if not l.startswith("include::"): continue if m := re.match(ADOC_INCLUDE_RE, l): chap = m.group(1).replace(".asciidoc", ".html") if chap in EXCLUDE: continue yield chap else: raise ValueError(f"Could not parse include line in book.asciidoc: {l}") CHAPTERS = list(_chapters()) class ChapterInfo(NamedTuple): href_id: str chapter_title: str subheaders: list[str] xrefs: list[str] def make_chapters(): for chapter in CHAPTERS: subprocess.check_call(["make", chapter], stdout=subprocess.PIPE) def parse_chapters() -> Iterator[tuple[str, html.HtmlElement]]: for chapter in CHAPTERS: raw_html = Path(chapter).read_text() yield chapter, html.fromstring(raw_html) def get_anchor_targets(parsed_html) -> list[str]: ignores = {"header", "content", "footnotes", "footer", "footer-text"} all_ids = [a.get("id") for a in parsed_html.cssselect("*[id]")] return [i for i in all_ids if not i.startswith("_") and i not in ignores] def get_chapter_info(): chapter_info = {} appendix_numbers = list("ABCDEFGHIJKL") chapter_numbers = list(range(1, 100)) part_numbers = list(range(1, 10)) for chapter, parsed_html in parse_chapters(): print("getting info from", chapter) if not parsed_html.cssselect("h2"): header = parsed_html.cssselect("h1")[0] else: header = parsed_html.cssselect("h2")[0] href_id = header.get("id") if href_id is None: href_id = parsed_html.cssselect("body")[0].get("id") subheaders = [h.get("id") for h in parsed_html.cssselect("h3")] chapter_title = header.text_content() chapter_title = chapter_title.replace("Appendix A: ", "") if chapter.startswith("chapter_"): chapter_no = chapter_numbers.pop(0) chapter_title = f"Chapter {chapter_no}: {chapter_title}" if chapter.startswith("appendix_"): appendix_no = appendix_numbers.pop(0) chapter_title = f"Appendix {appendix_no}: {chapter_title}" if chapter.startswith("part"): part_no = part_numbers.pop(0) chapter_title = f"Part {part_no}: {chapter_title}" if chapter.startswith("epilogue"): chapter_title = f"Epilogue: {chapter_title}" xrefs = get_anchor_targets(parsed_html) chapter_info[chapter] = ChapterInfo(href_id, chapter_title, subheaders, xrefs) return chapter_info def fix_xrefs(contents, chapter, chapter_info): parsed = html.fromstring(contents) links = parsed.cssselect(r"a[href^=\#]") for link in links: for other_chap in CHAPTERS: if other_chap == chapter: continue chapter_id = chapter_info[other_chap].href_id href = link.get("href") targets = ["#" + x for x in chapter_info[other_chap].xrefs] if href == "#" + chapter_id: link.set("href", f"/book/{other_chap}") elif href in targets: link.set("href", f"/book/{other_chap}{href}") return html.tostring(parsed) def fix_appendix_titles(contents, chapter, chapter_info): parsed = html.fromstring(contents) titles = parsed.cssselect("h2") if titles and titles[0].text.startswith("Appendix A"): title = titles[0] title.text = chapter_info[chapter].chapter_title return html.tostring(parsed) def copy_chapters_across_with_fixes(chapter_info, fixed_toc): comments_html = Path("disqus_comments.html").read_text() buy_book_div = html.fromstring(Path("buy_the_book_banner.html").read_text()) analytics_div = html.fromstring(Path("analytics.html").read_text()) load_toc_script = Path("load_toc.js").read_text() for chapter in CHAPTERS: old_contents = Path(chapter).read_text() new_contents = fix_xrefs(old_contents, chapter, chapter_info) new_contents = fix_appendix_titles(new_contents, chapter, chapter_info) parsed = html.fromstring(new_contents) body = parsed.cssselect("body")[0] if parsed.cssselect("#header"): head = parsed.cssselect("head")[0] head.append( html.fragment_fromstring("<script>" + load_toc_script + "</script>") ) body.set("class", "article toc2 toc-left") body.insert(0, buy_book_div) body.append( html.fromstring( comments_html.replace("CHAPTER_NAME", chapter.split(".")[0]) ) ) body.append(analytics_div) fixed_contents = html.tostring(parsed) with open(DEST / chapter, "w") as f: f.write(fixed_contents.decode("utf8")) with open(DEST / "toc.html", "w") as f: f.write(html.tostring(fixed_toc).decode("utf8")) def extract_toc_from_book(): subprocess.check_call(["make", "book.html"], stdout=subprocess.PIPE) parsed = html.fromstring(Path("book.html").read_text()) return parsed.cssselect("#toc")[0] def fix_toc(toc, chapter_info): href_mappings = {} for chapter in CHAPTERS: chap = chapter_info[chapter] if chap.href_id: href_mappings["#" + chap.href_id] = f"/book/{chapter}" for subheader in chap.subheaders: if subheader: href_mappings["#" + subheader] = f"/book/{chapter}#{subheader}" else: print(f"Warning: {chapter} has a subheader with no ID") def fix_link(href): if href in href_mappings: return href_mappings[href] else: return href toc.rewrite_links(fix_link) toc.set("class", "toc2") return toc def print_toc_md(chapter_info): for chapter in CHAPTERS: title = chapter_info[chapter].chapter_title print(f"* [{title}](/book/{chapter})") def rsync_images(): subprocess.run( ["rsync", "-a", "-v", "images/", DEST / "images/"], check=True, ) def main(): make_chapters() toc = extract_toc_from_book() chapter_info = get_chapter_info() fixed_toc = fix_toc(toc, chapter_info) copy_chapters_across_with_fixes(chapter_info, fixed_toc) rsync_images() print_toc_md(chapter_info) if __name__ == "__main__": main() ================================================ FILE: copyright.html ================================================ <section data-type="copyright-page"> <!--TITLE--> <h1>Test-Driven Development with Python</h1> <!--AUTHOR--> <p class="author">by <span class="firstname">Harry</span> <span class="othername mi">J.W.</span> <span class="surname">Percival</span></p> <!-- COPYRIGHT --> <p class="copyright">Copyright © 2025 Harry Percival. All rights reserved.</p> <!--PUBLISHER--> <p class="publisher">Published by <span class="publishername">O’Reilly Media, Inc.</span>, 141 Stony Circle, Suite 195, Santa Rosa, CA 95401.</p> <p>O’Reilly books may be purchased for educational, business, or sales promotional use. Online editions are also available for most titles (<a href="https://oreilly.com"><em>https://oreilly.com</em></a>). For more information, contact our corporate/institutional sales department: 800-998-9938 or <a class="email" href="mailto:corporate@oreilly.com"><em>corporate@oreilly.com</em></a>.</p> <!--STAFF LIST--> <ul class="stafflist"> <li><span class="staffrole">Acquisitions Editor: </span>Brian Guerin</li> <li><span class="staffrole">Development Editor: </span>Rita Fernando</li> <li><span class="staffrole">Production Editor: </span>Christopher Faucher</li> <li><span class="staffrole">Copyeditor: </span>Piper Content Partners</li> <li><span class="staffrole">Proofreader: </span>Kim Cofer</li> <li><span class="staffrole">Indexer: </span>Ellen Troutman-Zaig</li> <li><span class="staffrole">Cover Designer: </span>Susan Brown</li> <li><span class="staffrole">Cover Illustrator: </span>Karen Montgomery</li> <li><span class="staffrole">Interior Designer: </span>David Futato</li> <li><span class="staffrole">Interior Illustrator: </span>Kate Dullea</li> </ul> <!-- PRINTINGS --> <ul class="printings"> <li><span class="printedition">June 2014:</span> First Edition</li> <li><span class="printedition">August 2017:</span> Second Edition</li> <li><span class="printedition">October 2025:</span> Third Edition</li> </ul> <!-- REVISIONS --> <div> <h1 class="revisions">Revision History for the Third Edition</h1> <ul class="releases"> <li><span class="revdate">2025-10-15:</span> First Release</li> </ul> </div> <!-- ERRATA --> <p class="errata">See <a href="https://www.oreilly.com/catalog/errata.csp?isbn=0636920873884">https://www.oreilly.com/catalog/errata.csp?isbn=0636920873884</a> for release details.</p> <!--LEGAL--> <div class="legal"> <p>The O’Reilly logo is a registered trademark of O’Reilly Media, Inc. <em>Test-Driven Development with Python</em>, the cover image, and related trade dress are trademarks of O’Reilly Media, Inc.</p> <p>The views expressed in this work are those of the author, and do not represent the publisher's views. While the publisher and the author have used good faith efforts to ensure that the information and instructions contained in this work are accurate, the publisher and the author disclaim all responsibility for errors or omissions, including without limitation responsibility for damages resulting from the use of or reliance on this work. Use of the information and instructions contained in this work is at your own risk. If any code samples or other technology this work contains or describes is subject to open source licenses or the intellectual property rights of others, it is your responsibility to ensure that your use thereof complies with such licenses and/or rights. <!--PROD: Uncomment the following sentence if appropriate and add it to the above para:--><!--This book is not intended as [legal/medical/financial; use the appropriate reference] advice. Please consult a qualified professional if you require [legal/medical/financial] advice.--></p> </div> <div class="copyright-bottom"> <p class="isbn">978-1-098-14871-3</p> <p class="printer">[LSI]</p> </div> </section> ================================================ FILE: count-todos.py ================================================ import csv import datetime import re import sys from pathlib import Path MARKERS = ["TODO", "RITA", "DAVID", "SEBASTIAN", "JAN", "CSANAD"] out = csv.writer(sys.stdout) out.writerow(["Date", "Chapter"] + MARKERS) today = datetime.date.today() for path in sorted( list(Path(".").rglob("chapter*.asciidoc")) + list(Path(".").rglob("appendix*.asciidoc")) ): chapter_name = str(path).replace(".asciidoc", "") contents = path.read_text() todos = [len(re.findall(rf"\b{thing}\b", contents)) for thing in MARKERS] out.writerow([today.isoformat(), chapter_name] + todos) ================================================ FILE: cover.html ================================================ <figure data-type="cover"> <img src="images/cover.png"/> </figure> ================================================ FILE: disqus_comments.html ================================================ <div class="comments" style="padding: 20px"> <h3>Comments</h3> <div id="disqus_thread"></div> <script type="text/javascript"> var disqus_config = function () { this.page.identifier = 'CHAPTER_NAME'; }; (function() { var d = document, s = d.createElement('script'); s.src = '//obeythetestinggoat.disqus.com/embed.js'; s.setAttribute('data-timestamp', +new Date()); (d.head || d.body).appendChild(s); })(); </script> <noscript>Please enable JavaScript to view the <a href="https://disqus.com/?ref_noscript" rel="nofollow">comments powered by Disqus.</a></noscript> </div> ================================================ FILE: docs/ORM_style_guide.htm ================================================ <!DOCTYPE html> <html><head> <meta http-equiv="content-type" content="text/html; charset=UTF-8"> <title>O'Reilly Style Guide

O’Reilly Style Guide and Word List

About O'Reilly Style

This style guide is for authors, copyeditors, and proofreaders working on books of all formats. As writers and editors, we know that language changes over time, so please check back regularly for updates to terms and conventions. Recent additions will be set in bold for a few months.

Authors, please also consult the authoring documentation for the format in which you’re writing (Asciidoc, HTMLbook, DocBook, or Word). For sponsored projects, please see our statement of editorial independence.

For term conventions, check our guide and word list first, then The Chicago Manual of Style, 17th edition, then Merriam-Webster’s Collegiate Dictionary. Use your book-specific word list (provided by production) to document style choices that differ or are not covered here (e.g., A.M. or a.m., data center or datacenter).

To avoid unintentional bias, when writing about groups of people, check the group’s advocacy organization for guidance on appropriate language. The Conscious Style Guide is one good resource, aggregating links to relevant organizations. The University of Washington has another that is tech-specific. The Disability Language Style Guide is a thorough guide to writing about disabilities with sensitivity. Always follow a person’s preference and note exceptions, if necessary (e.g., quoting research that is decades old).

For questions specific to your book or assignment, please consult with your editor or production editor.

Considering Electronic Formats

Because we use a single set of source files to produce the print and electronic versions of our books, it’s important to keep all formats in mind while writing and editing:

  • Avoid using "above" and "below" to reference figures, tables, examples, unnumbered code blocks, equations, etc. (e.g., "In the example below…"). Using live cross references (e.g., "see Figure 2-1") is best, but when that’s not possible, use "preceding" or "following," as the physical placement of elements could be different in reflowable formats.

  • Anchor URLs to text nodes whenever possible, like you would on a website. See Links for more information.

    Be as descriptive as possible because the print version of your book renders hyperlinks like this: "text anchor (http://url.example.com/)."

    For example, this:

    Download the source code (http://www.url.thisismadeup.com) and install the package"

    is more useful than this:

    "Download the source code from this website (http://www.url.thisismadeup.com) and install the package."

    Avoid anchoring URLs to generic words or phrases such as "here," "this website," etc.

  • Long URLs will be shortened so that they’re easy for print readers to type manually.

    Do not link to products on any sales channels other than oreilly.com, including Apple, Google, or Amazon. Apple and Google will refuse to sell content that links to products on Amazon. Vendors, please flag any links to these sales channels and let the production editor know they exist.

    Saying "XX book is available on Amazon"—sans link—is OK.

O’Reilly Grammar, Punctuation, etc.

For any words or conventions not covered here, refer to The Chicago Manual of Style, 17th edition and Merriam-Webster.

back to top

Abbreviations/Acronyms

  • Generic pronouns should be they/their when needed, not he, she, or he/she.

  • Acronyms should generally be spelled out the first time they appear in a book, as in: "collaborative development environment (CDE)." See the Word List for common exceptions. After the acronym has been defined, you should generally use the acronym only (not the whole term, unless it makes more sense contextually to use the whole term). Usually, acronyms are defined only once per book. But if the author prefers, we can also define certain terms the first time they appear in each chapter.

  • Acronyms should be capitalized when expanded only if the term is a proper noun (and spelled that way by the company). For example, key performance indicator (KPI), but Amazon Web Services (AWS).

  • A.M. and P.M. or a.m. and p.m.—be consistent.

  • K = 1,024; k = 1,000. So a 56 kbps modem is equal to 56,000 bps, while 64 K of memory is equal to 65,536.

  • In units of measure, do not use a hyphen. For example, it’s 32 MB hard drive, not 32-MB hard drive. (Though when the unit is spelled out, use a hyphen, e.g., 32-megabyte hard drive.)

  • University degrees (e.g., B.A., B.S., M.A., M.S., Ph.D., etc.) can appear with or without periods—just be consistent.

  • United States and United Kingdom should be spelled out on first mention. After that, just use the acronym with no periods (so, US or UK).

back to top

Bibliographical Entries and Citations

In general, when referring to another book within a book’s text, include the author name(s) for up to two authors. For three or more authors, state the first author name, followed by “et al.” (be sure to include the period).

On first reference to another book, include author and publisher name. For example, "You can find more information in The Elements of Typographic Style by Robert Bringhurst (H&M)," or "For more information, consult Robert Bringhurst’s The Elements of Typographic Style (H&M)." On subsequent references, just use the book title.

When referencing an O’Reilly book within the text, note only "O’Reilly" in parentheses, not "O’Reilly Media, Inc." References to other O’Reilly books should be linked to the book’s catalog page.

Make sure that the catalog page is anchored to the book’s title, rather than standing on its own like this: "See Programming F# 3.0." NOT THIS: "See Programming F# 3.0 (http://shop.oreilly.com/product/0636920024033.do)."

Citations

When citing other materials in bibliographies, reference lists, or footnotes, use the “Notes and Bibliography” system from the The Chicago Manual of Style, 17th edition. Chicago also has an Author-Date system that some authors prefer, which is perfectly acceptable. If there is no discernible consistency, suggest Chicago's Notes for footnotes and Bibliography for endnotes or back matter.

Let your production editor know which of Chicago's systems you applied by adding a note to the Word List Doc.

Footnotes

  • Footnotes in running text are numbered and start over at 1 in each chapter. Footnote markers in running text should always appear after punctuation.

    This: The following query selects the symbol column and all columns from stocks whose names start with the prefix price.1

    Not this: The following query selects the symbol column and all columns from stocks whose names start with the prefix price1.

  • Footnotes should contain more than just a URL, whether a full citation for the text the URL points to or context for where the link leads.

    This:

    1. The Wikipedia entry on JavaScript (https://en.wikipedia.org/wiki/JavaScript) provides more information.
    2. Grove, John. 2015. “Calhoun and Conservative Reform.” American Political Thought 4, no. 2 (March): 203–27. https://doi.org/10.1086/680389.

    Not this:

    1. https://en.wikipedia.org/wiki/JavaScript
    2. https://doi.org/10.1086/680389

  • Table footnotes are lettered (a, b, c, etc.) and appear directly after the table. They should be kept to a minimum.

More details about styling footnotes in AsciiDoc are in Writing in AsciiDoc.

back to top

Cross References

Here are a few examples of cross references:

  • Chapter: See Chapter 27.

  • Section: See “Treatment” on page xx. (The text “on page xx” will be dynamic in Atlas, updating as page numbers change.)

  • Figure: ...as shown in Figure 1-1.

  • Sidebars: See “A Note for Mac Users” on page xx. (As with section xrefs, the page number will update automatically in Atlas.)

More details on cross-references in Asciidoc are available in our Writing in AsciiDoc guide.

These cross-reference styles are also available in DocBook under various <xref>: formats. Please refer to the DocBook Authoring Guidelines.

For information about styling URLs and hyperlinks, see Considering Electronic Formats.

back to top

Headings

Capitalization in headings:

  • In most of our design templates, A- and B-level headings are initial-capped (or title case): cap the first letter of each word, with the exception of articles, conjunctions, and program names or technical words that are always lowercase.

  • Prepositions of four letters or fewer are not initial-capped, unless they function as part of a verb (e.g., “Set Up Your Operating System”).

  • Hyphenated words in subordinating conjunctions (e.g., as, if, that, because, etc.) are always initial-capped (even if they are four letters or less). Hyphenated words in titles or captions should both be capped if the second word is a main word, but only the first should be capped if the second word isn’t too important (it’s a bit of a judgment call). For example: Big-Endian, Built-in. See The Chicago Manual of Style.

  • C-level headings have initial cap on the first word only (also called sentence-case), with the exception of proper nouns and the first word that follows a colon (unless that word refers to code and should be lowercase).

  • D-level headings (rare) are run-in with the following paragraph and have an initial cap on the first word only, with the exception of proper nouns and the first word that follows a colon (unless that word refers to code and should be lowercase), with a period at the end of the heading.

  • Sidebar titles are initial-capped, or title case (like A- and B-level headings, mentioned previously).

  • Admonition (note/tip/warning) titles are initial-capped, or title case (like A- and B-level headings, mentioned previously). Admonition titles are optional.

Headings should not contain inline code font or style formatting such as bold, italic, or code font.

Headings should always immediately precede body text. Do not follow a heading with an admonition or another heading without some form of introductory or descriptive text.

back to top

Dates and Numbers

What to spell out and when:

  • Spell out numbers from zero to nine and certain round multiples of those numbers unless the same object appears in a sentence with an object 10 or over (five apples; five apples and one hundred oranges; 5 apples and 110 oranges).

  • Whole numbers one through nine followed by hundred, thousand, million, billion, and so forth are usually spelled out (except in the sciences or with monetary amounts).

  • Centuries follow the same zero through nine rule, so those will usually be numerals (i.e., 20th century, 21st century).

  • In most numbers of one thousand or more, commas should be used between groups of three digits, counting from the right (32,904 NOT 32904). Exceptions: page numbers, addresses, port numbers, etc.

  • Use numerals for versions (version 5 or v5).

  • Use a numeral if it’s an actual value (e.g., 5% 7″ $6.00).

  • Always use the symbol % with numerals rather than the spelled out word (percent), and make sure it is closed up to number: 0.05%. Unless the percentage begins a sentence or title/caption, the number should be a numeral with the % symbol.

  • Ordinal numbers: Spell out first through ninth, use numerals for 10th and above. No superscript.

Formatting:

  • Use spaces around inline operators (1 + 1 = 2. NOT 1+1=2).

  • 32-bit integer.

  • 1980s or ’80s.

  • Phone numbers can appear in the format xxx-xxx-xxxx.

  • Use an en dash (–) with negative numbers or for minus signs, rather than a hyphen.

  • Use multiplication symbol “×” for dimensions, not "by" (e.g., "8.5 × 11").

back to top

Figures, Tables, and Examples

Every formally numbered figure, table, and example should be preceded by a specific in-text reference (for example: see Figure 99-1; Example 1-99 shows; Table 1-1 lists, etc.). Formal figures, tables, and examples should not be introduced with colons or phrases like “in the following figure,” or “as shown in this table.” Though we do support unnumbered informal figures/tables/examples, these should be used only for elements whose contents are not discussed at length or referred back to. Lack of specific in-text references may cause incorrect placement of figures. See Cross References above for more detail on including cross references.

If you are writing or copyediting in Word, figure, table, and example numbers should be numbered as follows: 1-2 (note hyphen [-], not en dash [–] between numbers). The first number is the chapter number. This will be soft-coded in production if not during the writing process.

If you are writing or copyediting in Asciidoc, please refer to Writing in AsciiDoc for examples of Asciidoc cross references.

If you are writing or copyediting in DocBook, please reference each figure, table, and example with an <xref>.

Any word groupings within a figure should have an initial cap on the first word only, with the exception of proper nouns. Generally, we don’t use periods at the end of these word groupings.

  • Figure 1-1. Figure captions are sentence-cased, with the exception of proper nouns. Code styling is allowed within the figure name or caption. There is no period after figure captions. Exceptions should be discussed with your production editor (e.g., if several long captions require punctuation, we can collaborate on efficient ways to achieve consistency).

  • Table 1-1. Column heads and table titles are sentence-cased, with the exception of proper nouns. Code styling is allowed within the table name or caption. There is no period after table titles.

  • Example 1-1. Example titles are sentence-cased, with the exception of proper nouns. Code styling is allowed within the example name or caption. There is no period after example titles.

When working in Word, make sure all table cells are tagged with a cell paragraph tag, even if they’re blank. Any bold “headings” that appear below the very first row of a table should be tagged CellSubheading rather than CellHeading.

Also in Word, all figures must be within a FigureHolder paragraph followed directly by a FigureTitle paragraph.

back to top

Code

Line Length

Maximum line length for code varies slightly between book formats. Consult the table below to find the maximum line length for your book’s series within Atlas v2. If writing in Word, please keep code within the margins that appear in the Word template and indicate proper linebreaks and indents for all code. Indent using spaces, not tabs.

Series Body (top-level code) Examples Lists Readeraids Sidebars

Animal

81

85

73

57

77

Animal 6x9

64

68

56

40

60

Report 6x9

64

68

56

40

60

Trade 6x9

76

72

65

80

69

Cookbook

81

85

73

57

77

Make 1-column

89

89

81

66

39

Make 2-column

45

46

35

28

40

Make Getting Started

63

67

60

51

60

Nutshell

71

75

67

60

75

Pocket Ref

51

55

50

42

51

Theory in Practice

81

85

77

51

83

Syntax Highlighting

We use a tool called Pygments to colorize code. In most books, code will appear in black and white in the print book and in color in all electronic formats, including the web pdf. If you’re an author, please consult the list of available lexers and apply them to your code as you write. To apply syntax highlighting in Asciidoc, consult Writing in AsciiDoc. To apply syntax highlighting in DocBook, consult the DocBook Authoring Guidelines. To apply syntax highlighting in Word, consult the O’Reilly Media Word Template Quickstart Guide.

Formatting Code in Word

When copyediting in Word, please do a global search and replace for tabs in code (search for \^t to find them) before submitting files for conversion; tabs will not convert. A general rule of thumb is one tab can be replaced with four spaces (which is the same number that the clean-up macro in the ORA.dot template uses). However, this number can vary, so the most important thing is that copyeditors replace tabs with the numbers of spaces needed to match the indentation and make sure levels of indentation are preserved.

back to top

Lists

Typically, we use three types of lists: numbered lists, for ordered steps or chronological items; variable lists, for terms and explanations/definitions; and bulleted lists, for series of items. List items are sentence-capped. List items should be treated as separate items and should not be strung together with punctuation or conjunctions. Unless one item in a list forms a complete sentence, the list's items do not take periods. If one does form a complete sentence, use periods for all items within that list, even fragments.

NOT O'Reilly style:

  • Here is an item, and
  • here is another item; and
  • here is the final item.

O'Reilly style:

  • Here is an item.
  • Here is another item.
  • Here is the final item.

Following are examples of each type of list.

Numbered list

The following list of step-by-step instructions is an example of a numbered list:

  1. Save Example 2-1 as the file hello.cs.

  2. Open a command window.

  3. From the command line, enter csc /debug hello.cs.

  4. To run the program, enter Hello.

Variable list

The following list of defined terms is an example of a variable list:

Setup project

This creates a setup file that automatically installs your files and resources.

Web setup project

This helps deploy a web-based project.

Bulleted list

The following series of items is an example of a bulleted list:

  • Labels

  • Buttons

  • A text box

“Bulleted” lists nested inside of bulleted lists should have em dashes as bullets.

Frequently, bulleted lists should be converted to variable lists. Any bulleted list whose entries consist of a short term and its definition should be converted. For example, the following bulleted list entries:

  • Spellchecking: process of correcting spelling

  • Pagebreaking—process of breaking pages

should be variable list entries:

Spellchecking

Process of correcting spelling

Pagebreaking

Process of breaking pages

back to top

Punctuation

For anything not covered in this list, please consult the Chicago Manual of Style, 17th Edition.

  • Serial comma (this, that, and the other).

  • Commas and periods go inside quotation marks.

  • Curly quotes and apostrophes (“ ” not " ") in regular text.

  • Straight quotes (" " not “ ”) in constant-width text and all code. Some Unix commands use backticks (`), which must be preserved.

  • No period after list items unless one item forms a complete sentence (then use periods for all items within that list, even fragments).

  • Em dashes are always closed (no space around them).

  • Ellipses are always closed (no space around them).

  • For menu items that end with an ellipsis (e.g., "New Folder…"), do not include ellipsis in running text.

  • Lowercase the first letter after a colon: this is how we do it. (Exception: headings.)

  • Parentheses are always roman, even when the contents are italic. For parentheses within parentheses, use square brackets (here’s the first parenthetical [and here’s the second]).

back to top

Miscellaneous

  • Do not use a hyphen between an adverb and the word it modifies. So, “incredibly wide table” rather than “incredibly-wide table.”

  • Close up words with the following prefixes (unless part of a proper noun) “micro,” “meta,” “multi,” “pseudo,” “re,” “non,” “sub,” and "co" (e.g., “multiusers,” “pseudoattribute,” “nonprogrammer,” “subprocess,” "coauthor"). Exceptions are noted in the word list (e.g., "re-create," "re-identification").

  • Avoid using the possessive case for singular nouns ending in “s,” if possible. So, it’s “the Windows Start menu,” not “Windows’s Start menu.”

  • Avoid wholesale changes to the author’s voice—for example, changing the first-person plural (the royal “we”) to the first-person singular or the second person. However, do try to maintain a consistency within sentences or paragraphs, where appropriate.

  • We advise using a conversational, user-friendly tone that assumes the reader is intelligent but doesn’t have this particular knowledge yet—like an experienced colleague onboarding a new hire. First-person pronouns, contractions, and active verbs are all encouraged.(Copyeditors: please check with your production editor if you wish to suggest global changes to tone.)

  • Companies are always singular. So, for example, “Apple emphasizes the value of aesthetics in its product line. Consequently, it dominates the digital-music market” is correct. “Apple emphasize the value of aesthetics in their product line. They dominate the digital-music market” is not. (Also applies to generic terms “organization,” “team,” “group,” etc.)

  • When referring to software elements or labels, always capitalize words that are capitalized on screen. Put quotes around any multiword element names that are lowercase on screen and would thus be hard to distinguish from the rest of the text (e.g., Click “Don’t select object until rendered” only if necessary.)

  • Use “between” for two items, “among” for three or more. Use “each other” for two, “one another” for three or more.

  • Use the American spellings of words when they differ.

  • Common foreign terms (such as “en masse”) are roman.

  • Introduce unnumbered code blocks with colons.

  • Do not stack admonitions, sidebars, or headings.

  • Avoid obscenities and slurs, and obscure if included (grawlix, a two-em dash, etc.)

back to top

Typography and Font Conventions

The following shows the basic font conventions used in O’Reilly books. Follow these links for detailed instructions for applying these styles in Asciidoc, DocBook, and Word.

If you want to use a font convention that is slightly different for one of the following items, check with your editor first—some things can change; some can’t. For example, URLs will not be anything but italic, but you might come up with a different font convention for function names or menu items. If you have a “new” element, please consult with your editor about which font to use.

Type of element Final result

Filenames, file extensions (such as .jpeg), directory paths, and libraries.

Body font italic

URLs, URIs, email addresses, domain names

Body font italic

Emphasized words (shouting!).

Please use italics rather than bold for emphasis.

Body font italic

First instance of a technical term

Body font italic

Code blocks

Constant width

Registry keys

Constant width

Language and script elements: class names, types, namespaces, attributes, methods, variables, keywords, functions, modules, commands, properties, parameters, values, objects, events, XML and HTML tags, and similar elements. Some examples include: System.Web.UI, a while loop, the Socket class, the grep command, and the Obsolete attribute.

Constant width

SQL commands (SELECT, INSERT, ALTER TABLE, CREATE INDEX, etc.)

CONSTANT WIDTH CAPS

Replaceable items (placeholder items in syntax); “username” in the following example is a placeholder: login: username

Constant width italic

Commands or text to be typed by the user

Constant width bold

Line annotations

Body font italic (but smaller)

Placeholders in paths, directories, URLs, or other text that would be italic anyway

http://www.<yourname>.com

Keyboard accelerators (Ctrl, Shift, etc.), menu titles, menu options, menu buttons

Body text

These font conventions may vary slightly for each project; please consult your editor, the production editor, or the freelance coordinator if you have any questions. Please note: Word authors should refer to the Word Template Quickstart Guide; DocBook authors should refer to our DocBook Authoring Guidelines (username: guest; leave the password blank).

It’s very important to follow tagging conventions for terms. The method for applying conventions will vary depending on the format: Word/OpenOffice, DocBook XML, or InDesign. Please consult with your editor or toolsreq@oreilly.com for instructions specific to each environment.

For Word copyediting, please do the following before submitting files for conversion: replace any tabs in code with the appropriate number of spaces (see earlier section, Code); convert any remaining Word comments to tagged Comment paragraphs highlighted in blue; search for any manual linebreaks (^l) and delete or replace with paragraph breaks as appropriate; and accept all changes and make sure filenames adhere to house style.

back to top

O'Reilly Cover Style

Use Chicago Manual of Style, 17th Edition for anything not mentioned here.

Bulleted lists on the back cover should begin with a capitalized word and end with no punctuation. Even if the list item is a complete sentence, it will not take a period.

back to top

O’Reilly Word List

Alphabetical Word List: Default spellings

A | B | C | D | E | F | G | H | I | J | K | L | M | N | O | P | Q | R | S | T | U | V | W | X | Y | Z |

A

  • acknowledgments
  • ActionScript
  • ActiveX control
  • Addison-Wesley
  • ad hoc
  • ADO.NET
  • Agile (cap when referring to Agile software development or when used on its own as a noun)
  • AI (no need to expand acronym to artificial intelligence)
  • Ajax
  • a.k.a. or aka (be consistent)
  • a.m. or A.M.
  • Alt key
  • Alt-N
  • anonymous FTP
  • antipattern
  • API (no need to expand acronym to application programming interface)
  • appendixes
  • applet (or Java applet)
  • AppleScript
  • AppleScript Studio (ASS)
  • ARPAnet
  • ASCII
  • ASP.NET
  • at sign
  • autogenerate
  • awk

back to top

B

  • backend
  • background processes
  • backpressure
  • backquote
  • backslash
  • Backspace key
  • backtick
  • backup (n); back up (v)
  • backward
  • backward compatible
  • bash (avoid starting sentence with this word, but if unavoidable, cap as Bash)
  • BeOS
  • Berkeley Software Distribution (BSD)
  • Berkeley Unix (older books may have UNIX)
  • BHOs
  • big data
  • Big Design Up Front (BDUF)
  • bioinformatics
  • Bitcoin (capitalize the concept/network/currency in general; lowercase specific units of currency)
  • bitmap
  • bit mask
  • Bitnet
  • bit plane
  • bitwise operators
  • BlackBerry
  • –black-box/white-box testing s/b avoided (alternatives: behavioral/structural testing, closed/open testing, opaque/clear testing)
  • –black hat/white hat s/b avoided (alternatives: unethical/ethical, malicious/preventative)
  • –blacklist/whitelist s/b avoided (alternatives: block list/allow list, deny/permit, excluded/included)
  • Boolean (unless referring to a datatype in code, in which case s/b lowercase)
  • Bourne-again shell (bash)
  • Bourne shell
  • braces or curly braces
  • brackets or square brackets
  • browsable
  • _build->measure->learn_ cycle
  • built-in (a, n)
  • button bar

back to top

C

  • CacheStorage
  • call-to-action
  • Caps Lock key
  • caret or circumflex
  • CAT-5
  • CD-ROM
  • C language (n)
  • C-language (a)
  • checkbox
  • checkmark
  • check-in (n)
  • classpath
  • CLI (no need to expand acronym to command-line interface)
  • click-through (a)
  • client/server
  • client side (n)
  • client-side (a)
  • cloud native (n or a)
  • co-class
  • coauthor
  • codebase
  • code set
  • colorcell
  • colormap
  • Command key (Mac)
  • command line (n)
  • command-line (a)
  • Common Object Request
  • Broker Architecture (CORBA)
  • compact disc
  • compile time (n)
  • compile-time (a)
  • CompuServe
  • Control key (Mac)
  • copyleft
  • copyright
  • coworker
  • CPU (no need to expand to central processing unit)
  • –crazy s/b avoided (alternatives: foolish, bizarre, etc.)
  • criterion (s), criteria (p)
  • cross-reference
  • C shell
  • <CR><LF>
  • Ctrl key (Windows)
  • curly braces or braces
  • cybersecurity

back to top

D

  • data block
  • datacenter or data center (be consistent)
  • Data Encryption Standard (DES)
  • datafile
  • datatype or data type (be consistent)
  • data is
  • dataset or data set (be consistent)
  • DB-9
  • Debian GNU/Linux
  • decision making (n)
  • decision-making (a)
  • deep learning (n and a, no hyphen)
  • de-identification (hyphenate)
  • DevOps
  • dial-up (a)
  • dial up (v)
  • disk
  • disk-imaging software
  • Delete key
  • design time (n)
  • design-time (a)
  • DNS
  • DocBook
  • Document Object Model (DOM)
  • Domain Name System
  • dot
  • dot-com
  • double-click
  • double-precision (a)
  • double quotes
  • down arrow
  • downlevel (a)
  • drag-and-drop (n)
  • drag and drop (v)
  • drop-down (a)
  • –dummy s/b avoided (alternatives include: placeholder)

back to top

E

  • ebook
  • ebusiness
  • ecommerce
  • eBay
  • Emacs
  • email
  • empty-element tag
  • end-of-file (EOF)
  • end-tag
  • end user (n); end-user (a)
  • Engines of Groth
  • Enter key
  • equals sign
  • ereader
  • Escape key (or Esc key)
  • et al.
  • Ethernet
  • exclamation mark
  • Exim

back to top

F

  • failback
  • failover
  • fax
  • file manager
  • filename
  • filepath
  • file server
  • filesystem
  • file type
  • FireWire
  • foreground
  • FORTRAN
  • Fortran 90
  • forward (adv)
  • frame type
  • FreeBSD
  • Free Documentation License (FDL)
  • Free Software Foundation (FSF)
  • frontend
  • _ftp_ (Unix command)
  • FTP (protocol)
  • FTP site
  • full stack (Full Stack in headings), no hyphen, even if adjective

back to top

G

  • gateway
  • Gb (gigabit)
  • GB (gigabyte)
  • GBps (gigabytes per second)
  • GHz
  • gid
  • GIMP
  • Git
  • GitHub
  • GNOME
  • GNU Emacs
  • GNU Public License (GPL)
  • GNUstep
  • Google PageRank
  • grayscale
  • greater-than sign or >
  • greenlight (v)
  • GUI, GUIs

back to top

H

  • handcode
  • handoff (n)
  • hardcode (v)
  • hardcore
  • hardcopy
  • hard link
  • hardware-in-the-loop
  • hash sign or sharp sign
  • high-level (a)
  • home page
  • hostname
  • hotspot
  • HTML
  • HTTP
  • hypertext

back to top

I

  • IDs
  • IDE
  • IndexedDB
  • infrastructure as a service (IaaS)
  • inline
  • inode
  • interclient
  • internet, the internet
  • Internet of Things (IoT)
  • internetwork
  • intranet
  • Intrinsics
  • I/O
  • IP (Internet Protocol)
  • IPsec
  • ISO
  • ISP

back to top

J

  • Jabber
  • Jabber client
  • Jabber server
  • Jabber applet
  • JAR archive
  • JAR file
  • JavaScript
  • JPEG

back to top

K

  • K Desktop Environment (KDE)
  • Kb (kilobit)
  • KB (kilobyte) (denotes file size or disk space)
  • Kbps (kilobits per second)
  • Kerberos
  • keepalive (n or a)
  • keyclick
  • keycode
  • keymaps
  • keypad
  • keystroke
  • keysym
  • keywords
  • key performance indicators (KPIs)
  • kHz (kilohertz)
  • –kill s/b avoided (alternatives: end, exit, cancel)
  • Korn shell

back to top

L

  • lambda (lc unless referring to a product)
  • Lean (capitalize noun or adjective when referring to Lean business methodology)
  • local area network or LAN
  • left angle bracket or <
  • lefthand (a)
  • leftmost
  • less-than sign or <
  • leveled (not levelled)
  • life cycle or lifecycle (be consistent)
  • line-feed (a)
  • line feed (n)
  • Linux
  • LinuxPPC
  • listbox
  • logfile
  • login, logout, or logon (n or a)
  • log in, log out, or log on (v)
  • lower-level (a)
  • lower-right (a)
  • Linux Professional Institute (LPI)

back to top

M

  • Mac (or MacBook)
  • macOS (replaces Mac OS X)
  • machine learning (n and a, no hyphen)
  • mail-handling (adjective)
  • – man hours s/b avoided (alternatives: work hours, employee hours)
  • manpage
  • markup
  • –master/slave (n, a) s/b avoided (alternatives: parent/child, leader/follower, primary/secondary)
  • Mb (megabit)
  • MB (megabyte)
  • MBps (megabytes per second)
  • McGraw-Hill
  • menu bar
  • metacharacter
  • Meta key
  • Meta-N
  • MHz (megahertz)
  • mice or mouses (be consistent)
  • microservices
  • Microsoft Windows
  • Microsoft Windows Me
  • Microsoft Windows NT
  • Microsoft Windows XP
  • Microsoft Windows 2000
  • –middleman s/b avoided (alternatives: go-between, link, etc.)
  • MIDlet
  • MKS Toolkit
  • model-in-the-loop
  • MS-DOS
  • multiline
  • Multi-Touch (when referring to Apple's trademark)
  • My Services
  • MySpace

back to top

N

  • nameserver
  • name service
  • namespace
  • the Net
  • .NET
  • NetBIOS
  • NetBSD
  • NetInfo
  • newline
  • newsgroups
  • NeXTSTEP
  • NGINX (company), nginx (server)
  • NOOP
  • nonlocal
  • NoSQL
  • Novell NetWare
  • the New York Times

back to top

O

  • Objective-C
  • object linking and embedding (OLE)
  • object-oriented programming (OOP)
  • object request broker (ORB)
  • OK
  • offline
  • offload
  • online
  • on premises (prep. phrase) on-premises (modifier); may be abbreviated to on prem/on-prem
  • open source (n or a, rewrite to avoid using in a verb form)
  • open source software (OSS)
  • OpenBSD
  • OpenMotif
  • OpenStep
  • OpenWindows
  • Option key (Mac)
  • Oracle7
  • Oracle8
  • Oracle 8.0
  • Oracle 8i (italic “i”)
  • Oracle 9i (italic “i”)
  • Oracle Parallel Query Option
  • O’Reilly Media, Inc.
    • O’Reilly’s platform s/b "the O’Reilly platform" or "the O’Reilly learning platform" and then "O’Reilly" on subsequent mentions.
  • OS/2
  • OSA
  • OSF/Motif
  • OS X

back to top

P

  • packet switch networks
  • Paint Shop Pro
  • pagefile
  • page rank (but Google PageRank)
  • parentheses (p)
  • parenthesis (s)
  • Pascal
  • pathname
  • pattern-matching (a)
  • peer-to-peer (or P2P)
  • % (not percent)
  • performant (Oracle)
  • period
  • Perl
  • Perl DBI
  • plain text (n)
  • plain-text (a)
  • platform as a service (PaaS)
  • Plug and Play (PnP)
  • plug in (v)
  • plug-in (a, n)
  • p.m. or P.M.
  • Point-to-Point Protocol (PPP)
  • pop up (v)
  • pop-up (n, a)
  • POP-3
  • Portable Document Format (PDF)
  • Portable Network Graphics (PNG)
  • Portable Operating
    • System Interface (POSIX)
  • POSIX-compliant
  • Post Office Protocol (POP)
  • postprocess
  • PostScript
  • Prentice Hall
  • process ID
  • progress bar
  • pseudoattribute
  • pseudo-tty
  • public key (n)
  • public-key (a)
  • publish/subscribe or pub/sub
  • pull-down (a)

back to top

Q

  • qmail
  • Qt
  • QuarkXPress
  • Quartz
  • Quartz Extreme
  • QuickTime
  • quotation marks (spell out first time; it can be “quotes” thereafter)

back to top

R

  • random-access (a)
  • RCS
  • read-only (a)
  • read/write
  • real time (n)
  • real-time (a)
  • re-create
  • Red Hat Linux
  • Red Hat Package Manager (RPM)
  • redirection
  • reference page or manpage
  • re-identification (hyphenate)
  • remote-access server
  • Rendezvous (Mac OS X zeroconf networking)
  • Return (key)
  • RFC 822
  • rich text (n)
  • rich-text (a)
  • right angle bracket or greater-than sign (>)
  • right-click
  • righthand (a)
  • rmail
  • road map or roadmap (be consistent)
  • rollback (n); roll back (v)
  • rollout (n); roll out (v)
  • rootkit
  • Rubout key
  • rulebase
  • ruleset
  • runtime (n, a)

back to top

S

  • Samba
  • saveset
  • screen dump
  • screenful
  • screensaver
  • scroll bar
  • securelevel (in Linux)
  • Secure Shell (SSH)
  • Secure Sockets Layer (SSL)
  • sed scripts
  • server-dependent
  • server side (n)
  • server-side (a)
  • service worker
  • servlet
  • set up (v)
  • setup (n)
  • SGML
  • sharp sign or hash sign
  • shell (lowercase even in shell name: Bourne shell)
  • shell scripts
  • Shift key
  • Simple API for XML (SAX)
  • single-precision (a)
  • single quote
  • site map
  • –slave/master (n, a) s/b avoided (alternatives: child/parent, follower/leader, secondary/primary)
  • Smalltalk
  • SMP (a, n)
  • SOAP
  • Social Security number (SSN)
  • software as a service (SaaS)
  • software-in-the-loop
  • source code
  • space bar
  • spam (not SPAM)
  • spellcheck
  • spellchecker
  • split screen
  • square brackets or brackets
  • standalone
  • standard input (stdin)
  • standard output (stdout)
  • start tag
  • startup file
  • stateful
  • stateless
  • status bar
  • stylesheet
  • subprocess
  • SUSE Linux
  • swapfile
  • swapspace
  • sync
  • system administrator
  • system-wide

back to top

T

  • 10-baseT
  • T1
  • t-shirt
  • Tab key
  • TAR file
  • TCP/IP
  • Telnet (the protocol)
  • telnet (v)
  • terabyte
  • TEX
  • texinfo
  • text box
  • text-input mode
  • thread pooling (n)
  • timeout (in tech/computing contexts)
  • time-sharing processes
  • timestamp
  • time zone
  • title bar
  • Token Ring
  • toolbar
  • toolchain
  • toolkit
  • tool tip
  • top-level (a)
  • toward
  • trade-off
  • – tribe s/b avoided (alternatives: company, institution, network, community)
  • tweet, retweet, live-tweet v, n (avoid “tweet out”)
  • Twitter user (preferred to "tweeter")
  • Twitterstorm, tweetstorm

back to top

U

  • UI (no need to expand to user interface)
  • UK (for United Kingdom)
  • Ultrix
  • Universal Serial Bus (USB)
  • Unix (UNIX in many books, esp. older ones)
  • up arrow
  • upper- and lowercase
  • uppercase
  • upper-left corner
  • UPSs
  • up-to-date
  • URLs
  • US (for United States)
  • Usenet
  • user ID (n)
  • user-ID (a)
  • username
  • UX (no need to expand to user experience)

back to top

V

  • v2 or version 2
  • VAX/VMS
  • VB.NET
  • versus (avoid vs.)
  • vice versa
  • VoiceXML
  • Visual Basic .NET
  • Visual Basic 6 or VB 6
  • Visual C++ .NET
  • Visual Studio .NET
  • VS.NET
  • Volume One

back to top

W

  • the Wall Street Journal
  • the web (n)
  • web (a)
  • web client
  • webmaster
  • web page
  • web server
  • web services (unless preceded by a proper noun, as in Microsoft Web Services)
  • website
  • –white-box testing s/b avoided (alternatives: structural/behavioral testing open/closed testing, clear/opaque testing)
  • –white hat/black hat s/b avoided (alternatives: ethical/unethical, preventative/malicious)
  • white pages
  • –whitelist/blacklist s/b avoided (alternatives: allow list/block list, permit/deny, included/excluded)
  • whitepaper (I printed my whitepaper on white paper.)
  • whitespace
  • wide area network or WAN
  • WiFi
  • wiki
  • wildcard
  • Windows 95
  • Windows 98
  • Windows 2000
  • Windows NT
  • Windows Vista
  • Windows XP
  • Wizard (proper noun)
  • wizard (a, n)
  • workaround
  • workbench
  • workgroup
  • workstation
  • World Wide Web (WWW)
  • wraparound
  • writable
  • write-only (a)
  • WYSIWYG

back to top

X

  • (x,y) (no space)
  • x-axis
  • Xbox
  • X client
  • x coordinate
  • X protocol
  • X server
  • X Toolkit
  • XView
  • X Window series
  • X Window System
  • x86
  • xFree86
  • XHTML
  • XLink
  • XML
  • XML Query Language (XQuery)
  • XML-RPC
  • XPath
  • XPointer
  • XSL
  • XSLT

back to top

Y

  • Yahoo!
  • y-axis
  • y coordinate

back to top

Z

  • Zeroconf (short for “Zero Configuration”)
  • zeros
  • zip code
  • zip (v)
  • ZIP file

back to top

================================================ FILE: docs/ORM_style_guide_files/main.css ================================================ body { font-family: "Helvetica Neue", Helvetica, Arial, Sans-Serif; font-size: 14px; color: #333333; margin: 60px auto; width: 70%; } .hyperlink { word-break: break-all; } nav ul, footer ul { font-family:'Helvetica', 'Arial', 'Sans-Serif'; padding: 0; list-style: none; font-weight: bold; } nav ul li, footer ul li { display: inline; margin-right: 20px; } a { text-decoration: none; color: #003399; } a:hover { text-decoration: underline; } h1 { font-size: 24pt; } p { line-height: 1.4em; color: #333; } footer { border-top: 1px solid #d5d5d5; font-size: .8em; } ul.posts { margin: 20px auto 40px; font-size: 1.5em; } ul.posts li, .no-bullets { list-style: none; font-size: 14px; margin-left: 0; padding-left: 0; } /* Lists */ dt { padding: 1em 0 .5em .75em; } ul { list-style-type: square; } ul, ol { line-height: 1.4em; } /* Tables */ table { margin: 12.5pt 0; border-collapse: collapse; max-width: 100%; hyphens: none; } table caption { font-style: italic; margin-bottom: 4pt; display: table-caption; text-align: left; } td, th { padding: 0.2em 0.4em 0.2em 0.4em; vertical-align: top; text-align: left; word-spacing: 0.3pt; } thead tr th { font-weight: 600; } /*lists within tables*/ table ul li, table ol li{ margin: 0 0 0 8pt; } table, th, td { border: 1px solid black; } th { background-color: #C8C8C8; color: black; } div[data-type="warning"] { background: #ffe9ec; } div[data-type="note"], div[data-type="warning"], div[data-type="tip"] { border: solid 1pt black; padding-left: 2em; padding-right: 2em; } /* Headers */ h2 { font-size: 20pt; text-align: center; border-bottom: 1px solid #009999; color: #009999; } h3 { font-size: 16pt; color: #009999; padding-top: 10px; } h4 { font-size: 12pt; color: #009999; } /* ORM Word Template Quickstart Guide */ .no-bullets li { background-image: url('../illustrations/download.svg'); background-repeat: no-repeat; display: block; min-height: 40px; background-size: 30px; padding: 0 0 0 50px; margin-right: 50px; margin-top: 20px; } /* Styled Admonitions */ div[data-type="note"] { border: 1px solid #86aac8; font-size: 10pt; padding: 1.5em; color: #86aac8; padding-bottom: 5px; } div[data-type="warning"] { border: 1px solid #ac2e3d; font-size: 12pt; padding: 1.5em; color: #ac2e3d; padding-bottom: 5px; } div[data-type="note"] h6 { background-image: url('../illustrations/note.png'); background-size:40px; padding-top: .5em; padding-bottom: .5em; background-repeat: no-repeat; display: block; min-height: 20px; font-size: 1.5em; padding-left: 50px; margin: 0 0 0 0; } div[data-type="warning"] h6 { background-image: url('../illustrations/warning.png'); background-size:40px; padding-top: .5em; padding-bottom: .5em; background-repeat: no-repeat; display: block; min-height: 20px; font-size: 1.5em; padding-left: 60px; margin: 0 0 0 0; } /* Responsive */ @media only screen and (min-width: 800px) { body { font-size: 14pt; } /* Headers */ h2 { font-size: 24pt; text-align: center; border-bottom: 1px solid #009999; color: #009999; } h3 { font-size: 20pt; color: #009999; padding-top: 10px; } h4 { font-size: 16pt; color: #009999; } .no-bullets li { float: left; margin: 0 0 35px 3px; padding: 15px 20px 0 35px; } #table-of-contents-word { clear: left; } .no-bullets { padding: 0; margin: 0; } div[data-type="note"], div[data-type="warning"] { font-size: 12pt; } ul.no-bullets { display: inline-block; } .center { width: 100%; text-align: center; } } @media only screen and (min-width: 950px) { .no-bullets li { float: left; margin: 0 0 35px 5px; padding: 15px 50px 0px 35px; } } @media only screen and (min-width: 1000px) { #table-of-contents-word+ul { width: 375px; margin: 0 auto; } } ================================================ FILE: docs/asciidoc-cheatsheet.html ================================================ AsciiDoc cheatsheet
POWERMAN
"In each of us sleeps a genius...
and his sleep gets deeper everyday."

Abstract

This is a cheatsheet for AsciiDoc - “Text based document generation” script. The cheatsheet available for different AsciiDoc versions (because of some markup syntax changes) and using different css styles. Here is list with all available cheatsheets for different AsciiDoc version and using different css styles.

This cheatsheet is for AsciiDoc 8.6.7, using default css.

Document header

Main Header
===========
Optional Author Name <optional@author.email>
Optional version, optional date
:Author:    AlternativeWayToSetOptional Author Name
:Email:     <AlternativeWayToSetOptional@author.email>
:Date:      AlternativeWayToSetOptional date
:Revision:  AlternativeWayToSetOptional version

Attributes

There a lot of predefined attributes in AsciiDoc, plus you can add your own. To get attribute value use {attributename} syntax.

Author is {author}

Version is {revision}

Author is Alex Efros

Version is 2.2.2

:My name: Alex Efros
My name is {myname}

My name is Alex Efros

Line
with bad attribute {qwe} will be
deleted

Line deleted

Escaped: \{qwe} and +++{qwe}+++

Escaped: {qwe} and {qwe}

Headers

Level 1
-------
Text.

Level 2
~~~~~~~
Text.

Level 3
^^^^^^^
Text.

Level 4
+++++++
Text.

Level 1

Text.

Level 2

Text.

Level 3

Text.

Level 4

Text.

== Level 1
Text.

=== Level 2
Text.

==== Level 3
Text.

===== Level 4
Text.

Level 1

Text.

Level 2

Text.

Level 3

Text.

Level 4

Text.

Paragraphs

.Optional Title

Usual
paragraph.
Optional Title

Usual paragraph.

.Optional Title

 Literal paragraph.
  Must be indented.
Optional Title
Literal paragraph.
 Must be indented.
.Optional Title

[source,perl]
die 'connect: '.$dbh->errstr;

Not a code in next paragraph.
Optional Title
die 'connect: '.$dbh->errstr;

Not a code in next paragraph.

.Optional Title
NOTE: This is an example
      single-paragraph note.
Note
Optional Title
This is an example single-paragraph note.		 
.Optional Title
[NOTE]
This is an example
single-paragraph note.
Note
Optional Title
This is an example single-paragraph note.		 
TIP: Tip.
Tip Tip. 
IMPORTANT: Important.
Important Important. 
WARNING: Warning.
Warning Warning. 
CAUTION: Caution.
Caution Caution.

Blocks

.Optional Title
----
*Listing* Block

Use: code or file listings
----
Optional Title
*Listing* Block

Use: code or file listings
.Optional Title
[source,perl]
----
# *Source* block
# Use: highlight code listings
# (require `source-highlight` or `pygmentize`)
use DBI;
my $dbh = DBI->connect('...',$u,$p)
    or die "connect: $dbh->errstr";
----
Optional Title
# *Source* block
# Use: highlight code listings
# (require `source-highlight` or `pygmentize`)
use DBI;
my $dbh = DBI->connect('...',$u,$p)
    or die "connect: $dbh->errstr";

.Optional Title
****
*Sidebar* Block

Use: sidebar notes :)
****
Optional Title

Sidebar Block

Use: sidebar notes :)

.Optional Title
==========================
*Example* Block

Use: examples :)

Default caption "Example:"
can be changed using

 [caption="Custom: "]

before example block.
==========================
Example 1. Optional Title

Example Block

Use: examples :)

Default caption "Example:" can be changed using

[caption="Custom: "]

before example block.

.Optional Title
[NOTE]
===============================
*NOTE* Block

Use: multi-paragraph notes.
===============================
Note
Optional Title

NOTE Block

Use: multi-paragraph notes.

 
////
*Comment* block

Use: hide comments
////
 
++++
*Passthrough* Block
<p>
Use: backend-specific markup like
<table border="1">
<tr><td>1<td>2
</table>
++++
 *Passthrough* Block

Use: backend-specific markup like

12 
 .Optional Title
 ....
 *Literal* Block

 Use: workaround when literal
 paragraph (indented) like
   1. First.
   2. Second.
 incorrectly processed as list.
 ....
Optional Title
*Literal* Block

Use: workaround when literal
paragraph (indented) like
  1. First.
  2. Second.
incorrectly processed as list.
.Optional Title
[quote, cite author, cite source]
____
*Quote* Block

Use: cite somebody
____
Optional Title

Quote Block

Use: cite somebody

cite source
— cite author

Text

forced +
line break

forced
line break

normal, _italic_, *bold*, +mono+.

``double quoted'', `single quoted'.

normal, ^super^, ~sub~.

normal, italic, bold, mono.

“double quoted”, ‘single quoted’.

normal, super, sub.

Command: `ls -al`

+mono *bold*+

`passthru *bold*`

Command: ls -al

mono bold

passthru *bold*

Path: '/some/filez.txt', '.b'

Path: /some/filez.txt, .b

[red]#red text# [yellow-background]#on yellow#
[big]#large# [red yellow-background big]*all bold*

red text on yellow large all bold

Chars: n__i__**b**++m++[red]##r##

Chars: nibmr

// Comment
 
(C) (R) (TM) -- ... -> <- => <= &#182;

© ® ™ — … → ← ⇒ ⇐ ¶

''''
 
Escaped:
\_italic_, +++_italic_+++,
t\__e__st, +++t__e__st+++,
+++<b>bold</b>+++, $$<b>normal</b>$$
\&#182;
\`not single quoted'
\`\`not double quoted''

Escaped: _italic_, _italic_, t__e__st, t__e__st, bold, <b>normal</b> &#182; `not single quoted' ``not double quoted''

Macros: links, images & include

If you’ll need to use space in url/path you should replace it with %20.

[[anchor-1]]
Paragraph or block 1.

anchor:anchor-2[]
Paragraph or block 2.

<<anchor-1>>,
<<anchor-1,First anchor>>,
xref:anchor-2[],
xref:anchor-2[Second anchor].

Paragraph or block 1.

Paragraph or block 2.

[anchor-1], First anchor, [anchor-2], Second anchor.

link:asciidoc[This document]
link:asciidoc.html[]
link:/index.html[This site root]

This document asciidoc.html This site root

http://google.com
http://google.com[Google Search]
mailto:root@localhost[email admin]

http://google.com Google Search email admin

First home
image:images/icons/home.png[]
, second home
image:images/icons/home.png[Alt text]
.

.Block image
image::images/icons/home.png[]
image::images/icons/home.png[Alt text]

.Thumbnail linked to full image
image:/images/font/640-screen2.gif[
"My screenshot",width=128,
link="/images/font/640-screen2.gif"]

First home images/icons/home.png , second home Alt text .

images/icons/home.png
Figure 1. Block image
Alt text
Thumbnail linked to full image

My screenshot 

This is example how files
can be included.
It's commented because
there no such files. :)

// include::footer.txt[]

// [source,perl]
// ----
// include::script.pl[]
// ----

This is example how files can be included. It’s commented because there no such files. :)

Lists

.Bulleted
* bullet
* bullet
  - bullet
  - bullet
* bullet
** bullet
** bullet
*** bullet
*** bullet
**** bullet
**** bullet
***** bullet
***** bullet
**** bullet
*** bullet
** bullet
* bullet
Bulleted

  • bullet

  • bullet

    • bullet

    • bullet

  • bullet

    • bullet

    • bullet

      • bullet

      • bullet

        • bullet

        • bullet

          • bullet

          • bullet

        • bullet

      • bullet

    • bullet

  • bullet 

.Bulleted 2
- bullet
  * bullet
Bulleted 2

  • bullet

    • bullet 

.Ordered
. number
. number
  .. letter
  .. letter
. number
.. loweralpha
.. loweralpha
... lowerroman
... lowerroman
.... upperalpha
.... upperalpha
..... upperroman
..... upperroman
.... upperalpha
... lowerroman
.. loweralpha
. number
Ordered

  1. number

  2. number

    1. letter

    2. letter

  3. number

    1. loweralpha

    2. loweralpha

      1. lowerroman

      2. lowerroman

        1. upperalpha

        2. upperalpha

          1. upperroman

          2. upperroman

        3. upperalpha

      3. lowerroman

    3. loweralpha

  4. number 

.Ordered 2
a. letter
b. letter
   .. letter2
   .. letter2
       .  number
       .  number
           1. number2
           2. number2
           3. number2
           4. number2
       .  number
   .. letter2
c. letter
Ordered 2

  1. letter

  2. letter

    1. letter2

    2. letter2

      1. number

      2. number

        1. number2

        2. number2

        3. number2

        4. number2

      3. number

    3. letter2

  3. letter 

.Labeled
Term 1::
    Definition 1
Term 2::
    Definition 2
    Term 2.1;;
        Definition 2.1
    Term 2.2;;
        Definition 2.2
Term 3::
    Definition 3
Term 4:: Definition 4
Term 4.1::: Definition 4.1
Term 4.2::: Definition 4.2
Term 4.2.1:::: Definition 4.2.1
Term 4.2.2:::: Definition 4.2.2
Term 4.3::: Definition 4.3
Term 5:: Definition 5
Labeled
Term 1

Definition 1

Term 2

Definition 2

Term 2.1

Definition 2.1

Term 2.2

Definition 2.2

Term 3

Definition 3

Term 4

Definition 4

Term 4.1

Definition 4.1

Term 4.2

Definition 4.2

Term 4.2.1

Definition 4.2.1

Term 4.2.2

Definition 4.2.2

Term 4.3

Definition 4.3

Term 5

Definition 5 

.Labeled 2
Term 1;;
    Definition 1
    Term 1.1::
        Definition 1.1
Labeled 2
Term 1

Definition 1

Term 1.1

Definition 1.1 

[horizontal]
.Labeled horizontal
Term 1:: Definition 1
Term 2:: Definition 2
[horizontal]
    Term 2.1;;
        Definition 2.1
    Term 2.2;;
        Definition 2.2
Term 3::
    Definition 3
Term 4:: Definition 4
[horizontal]
Term 4.1::: Definition 4.1
Term 4.2::: Definition 4.2
[horizontal]
Term 4.2.1:::: Definition 4.2.1
Term 4.2.2:::: Definition 4.2.2
Term 4.3::: Definition 4.3
Term 5:: Definition 5
Labeled horizontal
Term 1

Definition 1

Term 2

Definition 2

Term 2.1

Definition 2.1

Term 2.2

Definition 2.2

Term 3

Definition 3

Term 4

Definition 4

Term 4.1

Definition 4.1

Term 4.2

Definition 4.2

Term 4.2.1

Definition 4.2.1

Term 4.2.2

Definition 4.2.2

Term 4.3

Definition 4.3

Term 5

Definition 5 

[qanda]
.Q&A
Question 1::
    Answer 1
Question 2:: Answer 2
Q&A

  1. Question 1

    Answer 1

  2. Question 2

    Answer 2 

.Indent is optional
- bullet
    * another bullet
        1. number
        .  again number
            a. letter
            .. again letter

.. letter
. number

* bullet
- bullet
Indent is optional

  • bullet

    • another bullet

      1. number

        1. again number

          1. letter

            1. again letter

            2. letter

        2. number

    • bullet

  • bullet 

.Break two lists
. number
. number

Independent paragraph break list.

. number

.Header break list too
. number

--
. List block define list boundary too
. number
. number
--

--
. number
. number
--
Break two lists

  1. number

  2. number

Independent paragraph break list.

  1. number

Header break list too

  1. number

  1. List block define list boundary too

  2. number

  3. number

  1. number

  2. number 

.Continuation
- bullet
continuation
. number
  continuation
* bullet

  literal continuation

.. letter
+
Non-literal continuation.
+
----
any block can be

included in list
----
+
Last continuation.
Continuation

  • bullet continuation

    1. number continuation

      • bullet 

        literal continuation

        1. letter

          Non-literal continuation.

          any block can be

included in list

          Last continuation.

.List block allow sublist inclusion
- bullet
  * bullet
+
--
    - bullet
      * bullet
--
  * bullet
- bullet
  . number
    .. letter
+
--
      . number
        .. letter
--
    .. letter
  . number
List block allow sublist inclusion

  • bullet

    • bullet

      • bullet

      • bullet

    • bullet

  • bullet

    1. number

      1. letter

        1. number

        1. letter

      2. letter

    2. number

Tables

You can fill table from CSV file using include:: macros inside table.

.An example table
[options="header,footer"]
|=======================
|Col 1|Col 2      |Col 3
|1    |Item 1     |a
|2    |Item 2     |b
|3    |Item 3     |c
|6    |Three items|d
|=======================
Table 1. An example table
Col 1 Col 2 Col 3

6

Three items

d

1

Item 1

a

2

Item 2

b

3

Item 3

c

 
.CSV data, 15% each column
[format="csv",width="60%",cols="4"]
[frame="topbot",grid="none"]
|======
1,2,3,4
a,b,c,d
A,B,C,D
|======
Table 2. CSV data, 15% each column

1

2

3

4

a

b

c

d

A

B

C

D

 
[grid="rows",format="csv"]
[options="header",cols="^,<,<s,<,>m"]
|===========================
ID,FName,LName,Address,Phone
1,Vasya,Pupkin,London,+123
2,X,Y,"A,B",45678
|===========================
ID FName LName Address Phone

1

Vasya

Pupkin

London

+123

2

X

Y

A,B

45678

 
.Multiline cells, row/col span
|====
|Date |Duration |Avg HR |Notes

|22-Aug-08 .2+^.^|10:24 | 157 |
Worked out MSHR (max sustainable
heart rate) by going hard
for this interval.

|22-Aug-08 | 152 |
Back-to-back with previous interval.

|24-Aug-08 3+^|none

|====
Table 3. Multiline cells, row/col span

Date

Duration

Avg HR

Notes

22-Aug-08

10:24

157

Worked out MSHR (max sustainable heart rate) by going hard for this interval.

22-Aug-08

152

Back-to-back with previous interval.

24-Aug-08

none
================================================ FILE: docs/asciidoc-cheatsheet_files/Content.css ================================================ /* ShareMeNot is licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php Copyright (c) 2012 University of Washington Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* * Every property is !important to prevent any styles declared on the web page * from overriding ours. */ .sharemenotReplacementButton { border: none !important; cursor: pointer !important; height: auto !important; width: auto !important; } .sharemenotOriginalButton { border: none !important; height: 1.5em !important; } ================================================ FILE: docs/asciidoc-cheatsheet_files/asciidoc.asc ================================================ Not found: /doc/javascripts/867/asciidoc.js ================================================ FILE: docs/asciidoc-cheatsheet_files/asciidoc.css ================================================ /* Shared CSS for AsciiDoc xhtml11 and html5 backends */ /* Default font. */ body { font-family: Georgia,serif; } /* Title font. */ h1, h2, h3, h4, h5, h6, div.title, caption.title, thead, p.table.header, #toctitle, #author, #revnumber, #revdate, #revremark, #footer { font-family: Arial,Helvetica,sans-serif; } body { margin: 1em 5% 1em 5%; } a { color: blue; text-decoration: underline; } a:visited { color: fuchsia; } em { font-style: italic; color: navy; } strong { font-weight: bold; color: #083194; } h1, h2, h3, h4, h5, h6 { color: #527bbd; margin-top: 1.2em; margin-bottom: 0.5em; line-height: 1.3; } h1, h2, h3 { border-bottom: 2px solid silver; } h2 { padding-top: 0.5em; } h3 { float: left; } h3 + * { clear: left; } h5 { font-size: 1.0em; } div.sectionbody { margin-left: 0; } hr { border: 1px solid silver; } p { margin-top: 0.5em; margin-bottom: 0.5em; } ul, ol, li > p { margin-top: 0; } ul > li { color: #aaa; } ul > li > * { color: black; } pre { padding: 0; margin: 0; } #author { color: #527bbd; font-weight: bold; font-size: 1.1em; } #email { } #revnumber, #revdate, #revremark { } #footer { font-size: small; border-top: 2px solid silver; padding-top: 0.5em; margin-top: 4.0em; } #footer-text { float: left; padding-bottom: 0.5em; } #footer-badges { float: right; padding-bottom: 0.5em; } #preamble { margin-top: 1.5em; margin-bottom: 1.5em; } div.imageblock, div.exampleblock, div.verseblock, div.quoteblock, div.literalblock, div.listingblock, div.sidebarblock, div.admonitionblock { margin-top: 1.0em; margin-bottom: 1.5em; } div.admonitionblock { margin-top: 2.0em; margin-bottom: 2.0em; margin-right: 10%; color: #606060; } div.content { /* Block element content. */ padding: 0; } /* Block element titles. */ div.title, caption.title { color: #527bbd; font-weight: bold; text-align: left; margin-top: 1.0em; margin-bottom: 0.5em; } div.title + * { margin-top: 0; } td div.title:first-child { margin-top: 0.0em; } div.content div.title:first-child { margin-top: 0.0em; } div.content + div.title { margin-top: 0.0em; } div.sidebarblock > div.content { background: #ffffee; border: 1px solid #dddddd; border-left: 4px solid #f0f0f0; padding: 0.5em; } div.listingblock > div.content { border: 1px solid #dddddd; border-left: 5px solid #f0f0f0; background: #f8f8f8; padding: 0.5em; } div.quoteblock, div.verseblock { padding-left: 1.0em; margin-left: 1.0em; margin-right: 10%; border-left: 5px solid #f0f0f0; color: #888; } div.quoteblock > div.attribution { padding-top: 0.5em; text-align: right; } div.verseblock > pre.content { font-family: inherit; font-size: inherit; } div.verseblock > div.attribution { padding-top: 0.75em; text-align: left; } /* DEPRECATED: Pre version 8.2.7 verse style literal block. */ div.verseblock + div.attribution { text-align: left; } div.admonitionblock .icon { vertical-align: top; font-size: 1.1em; font-weight: bold; text-decoration: underline; color: #527bbd; padding-right: 0.5em; } div.admonitionblock td.content { padding-left: 0.5em; border-left: 3px solid #dddddd; } div.exampleblock > div.content { border-left: 3px solid #dddddd; padding-left: 0.5em; } div.imageblock div.content { padding-left: 0; } span.image img { border-style: none; } a.image:visited { color: white; } dl { margin-top: 0.8em; margin-bottom: 0.8em; } dt { margin-top: 0.5em; margin-bottom: 0; font-style: normal; color: navy; } dd > *:first-child { margin-top: 0.1em; } ul, ol { list-style-position: outside; } ol.arabic { list-style-type: decimal; } ol.loweralpha { list-style-type: lower-alpha; } ol.upperalpha { list-style-type: upper-alpha; } ol.lowerroman { list-style-type: lower-roman; } ol.upperroman { list-style-type: upper-roman; } div.compact ul, div.compact ol, div.compact p, div.compact p, div.compact div, div.compact div { margin-top: 0.1em; margin-bottom: 0.1em; } tfoot { font-weight: bold; } td > div.verse { white-space: pre; } div.hdlist { margin-top: 0.8em; margin-bottom: 0.8em; } div.hdlist tr { padding-bottom: 15px; } dt.hdlist1.strong, td.hdlist1.strong { font-weight: bold; } td.hdlist1 { vertical-align: top; font-style: normal; padding-right: 0.8em; color: navy; } td.hdlist2 { vertical-align: top; } div.hdlist.compact tr { margin: 0; padding-bottom: 0; } .comment { background: yellow; } .footnote, .footnoteref { font-size: 0.8em; } span.footnote, span.footnoteref { vertical-align: super; } #footnotes { margin: 20px 0 20px 0; padding: 7px 0 0 0; } #footnotes div.footnote { margin: 0 0 5px 0; } #footnotes hr { border: none; border-top: 1px solid silver; height: 1px; text-align: left; margin-left: 0; width: 20%; min-width: 100px; } div.colist td { padding-right: 0.5em; padding-bottom: 0.3em; vertical-align: top; } div.colist td img { margin-top: 0.3em; } @media print { #footer-badges { display: none; } } #toc { margin-bottom: 2.5em; } #toctitle { color: #527bbd; font-size: 1.1em; font-weight: bold; margin-top: 1.0em; margin-bottom: 0.1em; } div.toclevel0, div.toclevel1, div.toclevel2, div.toclevel3, div.toclevel4 { margin-top: 0; margin-bottom: 0; } div.toclevel2 { margin-left: 2em; font-size: 0.9em; } div.toclevel3 { margin-left: 4em; font-size: 0.9em; } div.toclevel4 { margin-left: 6em; font-size: 0.9em; } span.aqua { color: aqua; } span.black { color: black; } span.blue { color: blue; } span.fuchsia { color: fuchsia; } span.gray { color: gray; } span.green { color: green; } span.lime { color: lime; } span.maroon { color: maroon; } span.navy { color: navy; } span.olive { color: olive; } span.purple { color: purple; } span.red { color: red; } span.silver { color: silver; } span.teal { color: teal; } span.white { color: white; } span.yellow { color: yellow; } span.aqua-background { background: aqua; } span.black-background { background: black; } span.blue-background { background: blue; } span.fuchsia-background { background: fuchsia; } span.gray-background { background: gray; } span.green-background { background: green; } span.lime-background { background: lime; } span.maroon-background { background: maroon; } span.navy-background { background: navy; } span.olive-background { background: olive; } span.purple-background { background: purple; } span.red-background { background: red; } span.silver-background { background: silver; } span.teal-background { background: teal; } span.white-background { background: white; } span.yellow-background { background: yellow; } span.big { font-size: 2em; } span.small { font-size: 0.6em; } span.underline { text-decoration: underline; } span.overline { text-decoration: overline; } span.line-through { text-decoration: line-through; } div.unbreakable { page-break-inside: avoid; } /* * xhtml11 specific * * */ tt { font-family: "Courier New", Courier, monospace; font-size: inherit; color: navy; } div.tableblock { margin-top: 1.0em; margin-bottom: 1.5em; } div.tableblock > table { border: 3px solid #527bbd; } thead, p.table.header { font-weight: bold; color: #527bbd; } p.table { margin-top: 0; } /* Because the table frame attribute is overriden by CSS in most browsers. */ div.tableblock > table[frame="void"] { border-style: none; } div.tableblock > table[frame="hsides"] { border-left-style: none; border-right-style: none; } div.tableblock > table[frame="vsides"] { border-top-style: none; border-bottom-style: none; } /* * html5 specific * * */ .monospaced { font-family: "Courier New", Courier, monospace; font-size: inherit; color: navy; } table.tableblock { margin-top: 1.0em; margin-bottom: 1.5em; } thead, p.tableblock.header { font-weight: bold; color: #527bbd; } p.tableblock { margin-top: 0; } table.tableblock { border-width: 3px; border-spacing: 0px; border-style: solid; border-color: #527bbd; border-collapse: collapse; } th.tableblock, td.tableblock { border-width: 1px; padding: 4px; border-style: solid; border-color: #527bbd; } table.tableblock.frame-topbot { border-left-style: hidden; border-right-style: hidden; } table.tableblock.frame-sides { border-top-style: hidden; border-bottom-style: hidden; } table.tableblock.frame-none { border-style: hidden; } th.tableblock.halign-left, td.tableblock.halign-left { text-align: left; } th.tableblock.halign-center, td.tableblock.halign-center { text-align: center; } th.tableblock.halign-right, td.tableblock.halign-right { text-align: right; } th.tableblock.valign-top, td.tableblock.valign-top { vertical-align: top; } th.tableblock.valign-middle, td.tableblock.valign-middle { vertical-align: middle; } th.tableblock.valign-bottom, td.tableblock.valign-bottom { vertical-align: bottom; } /* * manpage specific * * */ body.manpage h1 { padding-top: 0.5em; padding-bottom: 0.5em; border-top: 2px solid silver; border-bottom: 2px solid silver; } body.manpage h2 { border-style: none; } body.manpage div.sectionbody { margin-left: 3em; } @media print { body.manpage div#toc { display: none; } } ================================================ FILE: docs/asciidoc-cheatsheet_files/asciidoc.js ================================================ var asciidoc = { // Namespace. ///////////////////////////////////////////////////////////////////// // Table Of Contents generator ///////////////////////////////////////////////////////////////////// /* Author: Mihai Bazon, September 2002 * http://students.infoiasi.ro/~mishoo * * Table Of Content generator * Version: 0.4 * * Feel free to use this script under the terms of the GNU General Public * License, as long as you do not remove or alter this notice. */ /* modified by Troy D. Hanson, September 2006. License: GPL */ /* modified by Stuart Rackham, 2006, 2009. License: GPL */ // toclevels = 1..4. toc: function (toclevels) { function getText(el) { var text = ""; for (var i = el.firstChild; i != null; i = i.nextSibling) { if (i.nodeType == 3 /* Node.TEXT_NODE */) // IE doesn't speak constants. text += i.data; else if (i.firstChild != null) text += getText(i); } return text; } function TocEntry(el, text, toclevel) { this.element = el; this.text = text; this.toclevel = toclevel; } function tocEntries(el, toclevels) { var result = new Array; var re = new RegExp('[hH]([1-'+(toclevels+1)+'])'); // Function that scans the DOM tree for header elements (the DOM2 // nodeIterator API would be a better technique but not supported by all // browsers). var iterate = function (el) { for (var i = el.firstChild; i != null; i = i.nextSibling) { if (i.nodeType == 1 /* Node.ELEMENT_NODE */) { var mo = re.exec(i.tagName); if (mo && (i.getAttribute("class") || i.getAttribute("className")) != "float") { result[result.length] = new TocEntry(i, getText(i), mo[1]-1); } iterate(i); } } } iterate(el); return result; } var toc = document.getElementById("toc"); if (!toc) { return; } // Delete existing TOC entries in case we're reloading the TOC. var tocEntriesToRemove = []; var i; for (i = 0; i < toc.childNodes.length; i++) { var entry = toc.childNodes[i]; if (entry.nodeName.toLowerCase() == 'div' && entry.getAttribute("class") && entry.getAttribute("class").match(/^toclevel/)) tocEntriesToRemove.push(entry); } for (i = 0; i < tocEntriesToRemove.length; i++) { toc.removeChild(tocEntriesToRemove[i]); } // Rebuild TOC entries. var entries = tocEntries(document.getElementById("content"), toclevels); for (var i = 0; i < entries.length; ++i) { var entry = entries[i]; if (entry.element.id == "") entry.element.id = "_toc_" + i; var a = document.createElement("a"); a.href = "#" + entry.element.id; a.appendChild(document.createTextNode(entry.text)); var div = document.createElement("div"); div.appendChild(a); div.className = "toclevel" + entry.toclevel; toc.appendChild(div); } if (entries.length == 0) toc.parentNode.removeChild(toc); }, ///////////////////////////////////////////////////////////////////// // Footnotes generator ///////////////////////////////////////////////////////////////////// /* Based on footnote generation code from: * http://www.brandspankingnew.net/archive/2005/07/format_footnote.html */ footnotes: function () { // Delete existing footnote entries in case we're reloading the footnodes. var i; var noteholder = document.getElementById("footnotes"); if (!noteholder) { return; } var entriesToRemove = []; for (i = 0; i < noteholder.childNodes.length; i++) { var entry = noteholder.childNodes[i]; if (entry.nodeName.toLowerCase() == 'div' && entry.getAttribute("class") == "footnote") entriesToRemove.push(entry); } for (i = 0; i < entriesToRemove.length; i++) { noteholder.removeChild(entriesToRemove[i]); } // Rebuild footnote entries. var cont = document.getElementById("content"); var spans = cont.getElementsByTagName("span"); var refs = {}; var n = 0; for (i=0; i" + n + "]"; spans[i].setAttribute("data-note", note); } noteholder.innerHTML += "
" + "" + n + ". " + note + "
"; var id =spans[i].getAttribute("id"); if (id != null) refs["#"+id] = n; } } if (n == 0) noteholder.parentNode.removeChild(noteholder); else { // Process footnoterefs. for (i=0; i" + n + "]"; } } } }, install: function(toclevels) { var timerId; function reinstall() { asciidoc.footnotes(); if (toclevels) { asciidoc.toc(toclevels); } } function reinstallAndRemoveTimer() { clearInterval(timerId); reinstall(); } timerId = setInterval(reinstall, 500); if (document.addEventListener) document.addEventListener("DOMContentLoaded", reinstallAndRemoveTimer, false); else window.onload = reinstallAndRemoveTimer; } } ================================================ FILE: docs/asciidoc-cheatsheet_files/jquery-1.js ================================================ /* * jQuery 1.2 - New Wave Javascript * * Copyright (c) 2007 John Resig (jquery.com) * Dual licensed under the MIT (MIT-LICENSE.txt) * and GPL (GPL-LICENSE.txt) licenses. * * $Date: 2007-09-10 15:45:49 -0400 (Mon, 10 Sep 2007) $ * $Rev: 3219 $ */ (function(){if(typeof jQuery!="undefined")var _jQuery=jQuery;var jQuery=window.jQuery=function(a,c){if(window==this||!this.init)return new jQuery(a,c);return this.init(a,c);};if(typeof $!="undefined")var _$=$;window.$=jQuery;var quickExpr=/^[^<]*(<(.|\s)+>)[^>]*$|^#(\w+)$/;jQuery.fn=jQuery.prototype={init:function(a,c){a=a||document;if(typeof a=="string"){var m=quickExpr.exec(a);if(m&&(m[1]||!c)){if(m[1])a=jQuery.clean([m[1]],c);else{var tmp=document.getElementById(m[3]);if(tmp)if(tmp.id!=m[3])return jQuery().find(a);else{this[0]=tmp;this.length=1;return this;}else a=[];}}else return new jQuery(c).find(a);}else if(jQuery.isFunction(a))return new jQuery(document)[jQuery.fn.ready?"ready":"load"](a);return this.setArray(a.constructor==Array&&a||(a.jquery||a.length&&a!=window&&!a.nodeType&&a[0]!=undefined&&a[0].nodeType)&&jQuery.makeArray(a)||[a]);},jquery:"1.2",size:function(){return this.length;},length:0,get:function(num){return num==undefined?jQuery.makeArray(this):this[num];},pushStack:function(a){var ret=jQuery(a);ret.prevObject=this;return ret;},setArray:function(a){this.length=0;Array.prototype.push.apply(this,a);return this;},each:function(fn,args){return jQuery.each(this,fn,args);},index:function(obj){var pos=-1;this.each(function(i){if(this==obj)pos=i;});return pos;},attr:function(key,value,type){var obj=key;if(key.constructor==String)if(value==undefined)return this.length&&jQuery[type||"attr"](this[0],key)||undefined;else{obj={};obj[key]=value;}return this.each(function(index){for(var prop in obj)jQuery.attr(type?this.style:this,prop,jQuery.prop(this,obj[prop],type,index,prop));});},css:function(key,value){return this.attr(key,value,"curCSS");},text:function(e){if(typeof e!="object"&&e!=null)return this.empty().append(document.createTextNode(e));var t="";jQuery.each(e||this,function(){jQuery.each(this.childNodes,function(){if(this.nodeType!=8)t+=this.nodeType!=1?this.nodeValue:jQuery.fn.text([this]);});});return t;},wrapAll:function(html){if(this[0])jQuery(html,this[0].ownerDocument).clone().insertBefore(this[0]).map(function(){var elem=this;while(elem.firstChild)elem=elem.firstChild;return elem;}).append(this);return this;},wrapInner:function(html){return this.each(function(){jQuery(this).contents().wrapAll(html);});},wrap:function(html){return this.each(function(){jQuery(this).wrapAll(html);});},append:function(){return this.domManip(arguments,true,1,function(a){this.appendChild(a);});},prepend:function(){return this.domManip(arguments,true,-1,function(a){this.insertBefore(a,this.firstChild);});},before:function(){return this.domManip(arguments,false,1,function(a){this.parentNode.insertBefore(a,this);});},after:function(){return this.domManip(arguments,false,-1,function(a){this.parentNode.insertBefore(a,this.nextSibling);});},end:function(){return this.prevObject||jQuery([]);},find:function(t){var data=jQuery.map(this,function(a){return jQuery.find(t,a);});return this.pushStack(/[^+>] [^+>]/.test(t)||t.indexOf("..")>-1?jQuery.unique(data):data);},clone:function(events){var ret=this.map(function(){return this.outerHTML?jQuery(this.outerHTML)[0]:this.cloneNode(true);});if(events===true){var clone=ret.find("*").andSelf();this.find("*").andSelf().each(function(i){var events=jQuery.data(this,"events");for(var type in events)for(var handler in events[type])jQuery.event.add(clone[i],type,events[type][handler],events[type][handler].data);});}return ret;},filter:function(t){return this.pushStack(jQuery.isFunction(t)&&jQuery.grep(this,function(el,index){return t.apply(el,[index]);})||jQuery.multiFilter(t,this));},not:function(t){return this.pushStack(t.constructor==String&&jQuery.multiFilter(t,this,true)||jQuery.grep(this,function(a){return(t.constructor==Array||t.jquery)?jQuery.inArray(a,t)<0:a!=t;}));},add:function(t){return this.pushStack(jQuery.merge(this.get(),t.constructor==String?jQuery(t).get():t.length!=undefined&&(!t.nodeName||t.nodeName=="FORM")?t:[t]));},is:function(expr){return expr?jQuery.multiFilter(expr,this).length>0:false;},hasClass:function(expr){return this.is("."+expr);},val:function(val){if(val==undefined){if(this.length){var elem=this[0];if(jQuery.nodeName(elem,"select")){var index=elem.selectedIndex,a=[],options=elem.options,one=elem.type=="select-one";if(index<0)return null;for(var i=one?index:0,max=one?index+1:options.length;i=0||jQuery.inArray(this.name,val)>=0);else if(jQuery.nodeName(this,"select")){var tmp=val.constructor==Array?val:[val];jQuery("option",this).each(function(){this.selected=(jQuery.inArray(this.value,tmp)>=0||jQuery.inArray(this.text,tmp)>=0);});if(!tmp.length)this.selectedIndex=-1;}else this.value=val;});},html:function(val){return val==undefined?(this.length?this[0].innerHTML:null):this.empty().append(val);},replaceWith:function(val){return this.after(val).remove();},slice:function(){return this.pushStack(Array.prototype.slice.apply(this,arguments));},map:function(fn){return this.pushStack(jQuery.map(this,function(elem,i){return fn.call(elem,i,elem);}));},andSelf:function(){return this.add(this.prevObject);},domManip:function(args,table,dir,fn){var clone=this.length>1,a;return this.each(function(){if(!a){a=jQuery.clean(args,this.ownerDocument);if(dir<0)a.reverse();}var obj=this;if(table&&jQuery.nodeName(this,"table")&&jQuery.nodeName(a[0],"tr"))obj=this.getElementsByTagName("tbody")[0]||this.appendChild(document.createElement("tbody"));jQuery.each(a,function(){if(jQuery.nodeName(this,"script")){if(this.src)jQuery.ajax({url:this.src,async:false,dataType:"script"});else jQuery.globalEval(this.text||this.textContent||this.innerHTML||"");}else fn.apply(obj,[clone?this.cloneNode(true):this]);});});}};jQuery.extend=jQuery.fn.extend=function(){var target=arguments[0]||{},a=1,al=arguments.length,deep=false;if(target.constructor==Boolean){deep=target;target=arguments[1]||{};}if(al==1){target=this;a=0;}var prop;for(;a-1;}},swap:function(e,o,f){for(var i in o){e.style["old"+i]=e.style[i];e.style[i]=o[i];}f.apply(e,[]);for(var i in o)e.style[i]=e.style["old"+i];},css:function(e,p){if(p=="height"||p=="width"){var old={},oHeight,oWidth,d=["Top","Bottom","Right","Left"];jQuery.each(d,function(){old["padding"+this]=0;old["border"+this+"Width"]=0;});jQuery.swap(e,old,function(){if(jQuery(e).is(':visible')){oHeight=e.offsetHeight;oWidth=e.offsetWidth;}else{e=jQuery(e.cloneNode(true)).find(":radio").removeAttr("checked").end().css({visibility:"hidden",position:"absolute",display:"block",right:"0",left:"0"}).appendTo(e.parentNode)[0];var parPos=jQuery.css(e.parentNode,"position")||"static";if(parPos=="static")e.parentNode.style.position="relative";oHeight=e.clientHeight;oWidth=e.clientWidth;if(parPos=="static")e.parentNode.style.position="static";e.parentNode.removeChild(e);}});return p=="height"?oHeight:oWidth;}return jQuery.curCSS(e,p);},curCSS:function(elem,prop,force){var ret,stack=[],swap=[];function color(a){if(!jQuery.browser.safari)return false;var ret=document.defaultView.getComputedStyle(a,null);return!ret||ret.getPropertyValue("color")=="";}if(prop=="opacity"&&jQuery.browser.msie){ret=jQuery.attr(elem.style,"opacity");return ret==""?"1":ret;}if(prop.match(/float/i))prop=styleFloat;if(!force&&elem.style[prop])ret=elem.style[prop];else if(document.defaultView&&document.defaultView.getComputedStyle){if(prop.match(/float/i))prop="float";prop=prop.replace(/([A-Z])/g,"-$1").toLowerCase();var cur=document.defaultView.getComputedStyle(elem,null);if(cur&&!color(elem))ret=cur.getPropertyValue(prop);else{for(var a=elem;a&&color(a);a=a.parentNode)stack.unshift(a);for(a=0;a]*?)\/>/g,function(m,all,tag){return tag.match(/^(abbr|br|col|img|input|link|meta|param|hr|area)$/i)?m:all+">";});var s=jQuery.trim(arg).toLowerCase(),div=doc.createElement("div"),tb=[];var wrap=!s.indexOf("",""]||!s.indexOf("",""]||s.match(/^<(thead|tbody|tfoot|colg|cap)/)&&[1,"","
"]||!s.indexOf("",""]||(!s.indexOf("",""]||!s.indexOf("",""]||jQuery.browser.msie&&[1,"div
","
"]||[0,"",""];div.innerHTML=wrap[1]+arg+wrap[2];while(wrap[0]--)div=div.lastChild;if(jQuery.browser.msie){if(!s.indexOf(""&&s.indexOf("=0;--n)if(jQuery.nodeName(tb[n],"tbody")&&!tb[n].childNodes.length)tb[n].parentNode.removeChild(tb[n]);if(/^\s/.test(arg))div.insertBefore(doc.createTextNode(arg.match(/^\s*/)[0]),div.firstChild);}arg=jQuery.makeArray(div.childNodes);}if(0===arg.length&&(!jQuery.nodeName(arg,"form")&&!jQuery.nodeName(arg,"select")))return;if(arg[0]==undefined||jQuery.nodeName(arg,"form")||arg.options)r.push(arg);else r=jQuery.merge(r,arg);});return r;},attr:function(elem,name,value){var fix=jQuery.isXMLDoc(elem)?{}:jQuery.props;if(name=="selected"&&jQuery.browser.safari)elem.parentNode.selectedIndex;if(fix[name]){if(value!=undefined)elem[fix[name]]=value;return elem[fix[name]];}else if(jQuery.browser.msie&&name=="style")return jQuery.attr(elem.style,"cssText",value);else if(value==undefined&&jQuery.browser.msie&&jQuery.nodeName(elem,"form")&&(name=="action"||name=="method"))return elem.getAttributeNode(name).nodeValue;else if(elem.tagName){if(value!=undefined){if(name=="type"&&jQuery.nodeName(elem,"input")&&elem.parentNode)throw"type property can't be changed";elem.setAttribute(name,value);}if(jQuery.browser.msie&&/href|src/.test(name)&&!jQuery.isXMLDoc(elem))return elem.getAttribute(name,2);return elem.getAttribute(name);}else{if(name=="opacity"&&jQuery.browser.msie){if(value!=undefined){elem.zoom=1;elem.filter=(elem.filter||"").replace(/alpha\([^)]*\)/,"")+(parseFloat(value).toString()=="NaN"?"":"alpha(opacity="+value*100+")");}return elem.filter?(parseFloat(elem.filter.match(/opacity=([^)]*)/)[1])/100).toString():"";}name=name.replace(/-([a-z])/ig,function(z,b){return b.toUpperCase();});if(value!=undefined)elem[name]=value;return elem[name];}},trim:function(t){return(t||"").replace(/^\s+|\s+$/g,"");},makeArray:function(a){var r=[];if(typeof a!="array")for(var i=0,al=a.length;i\\s*("+chars+"+)"),quickID=new RegExp("^("+chars+"+)(#)("+chars+"+)"),quickClass=new RegExp("^([#.]?)("+chars+"*)");jQuery.extend({expr:{"":"m[2]=='*'||jQuery.nodeName(a,m[2])","#":"a.getAttribute('id')==m[2]",":":{lt:"im[3]-0",nth:"m[3]-0==i",eq:"m[3]-0==i",first:"i==0",last:"i==r.length-1",even:"i%2==0",odd:"i%2","first-child":"a.parentNode.getElementsByTagName('*')[0]==a","last-child":"jQuery.nth(a.parentNode.lastChild,1,'previousSibling')==a","only-child":"!jQuery.nth(a.parentNode.lastChild,2,'previousSibling')",parent:"a.firstChild",empty:"!a.firstChild",contains:"(a.textContent||a.innerText||'').indexOf(m[3])>=0",visible:'"hidden"!=a.type&&jQuery.css(a,"display")!="none"&&jQuery.css(a,"visibility")!="hidden"',hidden:'"hidden"==a.type||jQuery.css(a,"display")=="none"||jQuery.css(a,"visibility")=="hidden"',enabled:"!a.disabled",disabled:"a.disabled",checked:"a.checked",selected:"a.selected||jQuery.attr(a,'selected')",text:"'text'==a.type",radio:"'radio'==a.type",checkbox:"'checkbox'==a.type",file:"'file'==a.type",password:"'password'==a.type",submit:"'submit'==a.type",image:"'image'==a.type",reset:"'reset'==a.type",button:'"button"==a.type||jQuery.nodeName(a,"button")',input:"/input|select|textarea|button/i.test(a.nodeName)",has:"jQuery.find(m[3],a).length",header:"/h\\d/i.test(a.nodeName)",animated:"jQuery.grep(jQuery.timers,function(fn){return a==fn.elem;}).length"}},parse:[/^(\[) *@?([\w-]+) *([!*$^~=]*) *('?"?)(.*?)\4 *\]/,/^(:)([\w-]+)\("?'?(.*?(\(.*?\))?[^(]*?)"?'?\)/,new RegExp("^([:.#]*)("+chars+"+)")],multiFilter:function(expr,elems,not){var old,cur=[];while(expr&&expr!=old){old=expr;var f=jQuery.filter(expr,elems,not);expr=f.t.replace(/^\s*,\s*/,"");cur=not?elems=f.r:jQuery.merge(cur,f.r);}return cur;},find:function(t,context){if(typeof t!="string")return[t];if(context&&!context.nodeType)context=null;context=context||document;var ret=[context],done=[],last;while(t&&last!=t){var r=[];last=t;t=jQuery.trim(t);var foundToken=false;var re=quickChild;var m=re.exec(t);if(m){var nodeName=m[1].toUpperCase();for(var i=0;ret[i];i++)for(var c=ret[i].firstChild;c;c=c.nextSibling)if(c.nodeType==1&&(nodeName=="*"||c.nodeName.toUpperCase()==nodeName.toUpperCase()))r.push(c);ret=r;t=t.replace(re,"");if(t.indexOf(" ")==0)continue;foundToken=true;}else{re=/^([>+~])\s*(\w*)/i;if((m=re.exec(t))!=null){r=[];var nodeName=m[2],merge={};m=m[1];for(var j=0,rl=ret.length;j=0;if(!not&&pass||not&&!pass)tmp.push(r[i]);}return tmp;},filter:function(t,r,not){var last;while(t&&t!=last){last=t;var p=jQuery.parse,m;for(var i=0;p[i];i++){m=p[i].exec(t);if(m){t=t.substring(m[0].length);m[2]=m[2].replace(/\\/g,"");break;}}if(!m)break;if(m[1]==":"&&m[2]=="not")r=jQuery.filter(m[3],r,true).r;else if(m[1]==".")r=jQuery.classFilter(r,m[2],not);else if(m[1]=="["){var tmp=[],type=m[3];for(var i=0,rl=r.length;i=0)^not)tmp.push(a);}r=tmp;}else if(m[1]==":"&&m[2]=="nth-child"){var merge={},tmp=[],test=/(\d*)n\+?(\d*)/.exec(m[3]=="even"&&"2n"||m[3]=="odd"&&"2n+1"||!/\D/.test(m[3])&&"n+"+m[3]||m[3]),first=(test[1]||1)-0,last=test[2]-0;for(var i=0,rl=r.length;i<\/script>");var script=document.getElementById("__ie_init");if(script)script.onreadystatechange=function(){if(this.readyState!="complete")return;jQuery.ready();};script=null;}else if(jQuery.browser.safari)jQuery.safariTimer=setInterval(function(){if(document.readyState=="loaded"||document.readyState=="complete"){clearInterval(jQuery.safariTimer);jQuery.safariTimer=null;jQuery.ready();}},10);jQuery.event.add(window,"load",jQuery.ready);}jQuery.fn.extend({load:function(url,params,callback){if(jQuery.isFunction(url))return this.bind("load",url);var off=url.indexOf(" ");if(off>=0){var selector=url.slice(off,url.length);url=url.slice(0,off);}callback=callback||function(){};var type="GET";if(params)if(jQuery.isFunction(params)){callback=params;params=null;}else{params=jQuery.param(params);type="POST";}var self=this;jQuery.ajax({url:url,type:type,data:params,complete:function(res,status){if(status=="success"||status=="notmodified")self.html(selector?jQuery("
").append(res.responseText.replace(//g,"")).find(selector):res.responseText);setTimeout(function(){self.each(callback,[res.responseText,status,res]);},13);}});return this;},serialize:function(){return jQuery.param(this.serializeArray());},serializeArray:function(){return this.map(function(){return jQuery.nodeName(this,"form")?jQuery.makeArray(this.elements):this;}).filter(function(){return this.name&&!this.disabled&&(this.checked||/select|textarea/i.test(this.nodeName)||/text|hidden|password/i.test(this.type));}).map(function(i,elem){var val=jQuery(this).val();return val==null?null:val.constructor==Array?jQuery.map(val,function(i,val){return{name:elem.name,value:val};}):{name:elem.name,value:val};}).get();}});jQuery.each("ajaxStart,ajaxStop,ajaxComplete,ajaxError,ajaxSuccess,ajaxSend".split(","),function(i,o){jQuery.fn[o]=function(f){return this.bind(o,f);};});var jsc=(new Date).getTime();jQuery.extend({get:function(url,data,callback,type){if(jQuery.isFunction(data)){callback=data;data=null;}return jQuery.ajax({type:"GET",url:url,data:data,success:callback,dataType:type});},getScript:function(url,callback){return jQuery.get(url,null,callback,"script");},getJSON:function(url,data,callback){return jQuery.get(url,data,callback,"json");},post:function(url,data,callback,type){if(jQuery.isFunction(data)){callback=data;data={};}return jQuery.ajax({type:"POST",url:url,data:data,success:callback,dataType:type});},ajaxSetup:function(settings){jQuery.extend(jQuery.ajaxSettings,settings);},ajaxSettings:{global:true,type:"GET",timeout:0,contentType:"application/x-www-form-urlencoded",processData:true,async:true,data:null},lastModified:{},ajax:function(s){var jsonp,jsre=/=(\?|%3F)/g,status,data;s=jQuery.extend(true,s,jQuery.extend(true,{},jQuery.ajaxSettings,s));if(s.data&&s.processData&&typeof s.data!="string")s.data=jQuery.param(s.data);var q=s.url.indexOf("?");if(q>-1){s.data=(s.data?s.data+"&":"")+s.url.slice(q+1);s.url=s.url.slice(0,q);}if(s.dataType=="jsonp"){if(!s.data||!s.data.match(jsre))s.data=(s.data?s.data+"&":"")+(s.jsonp||"callback")+"=?";s.dataType="json";}if(s.dataType=="json"&&s.data&&s.data.match(jsre)){jsonp="jsonp"+jsc++;s.data=s.data.replace(jsre,"="+jsonp);s.dataType="script";window[jsonp]=function(tmp){data=tmp;success();window[jsonp]=undefined;try{delete window[jsonp];}catch(e){}};}if(s.dataType=="script"&&s.cache==null)s.cache=false;if(s.cache===false&&s.type.toLowerCase()=="get")s.data=(s.data?s.data+"&":"")+"_="+(new Date()).getTime();if(s.data&&s.type.toLowerCase()=="get"){s.url+="?"+s.data;s.data=null;}if(s.global&&!jQuery.active++)jQuery.event.trigger("ajaxStart");if(!s.url.indexOf("http")&&s.dataType=="script"){var head=document.getElementsByTagName("head")[0];var script=document.createElement("script");script.src=s.url;if(!jsonp&&(s.success||s.complete)){var done=false;script.onload=script.onreadystatechange=function(){if(!done&&(!this.readyState||this.readyState=="loaded"||this.readyState=="complete")){done=true;success();complete();head.removeChild(script);}};}head.appendChild(script);return;}var requestDone=false;var xml=window.ActiveXObject?new ActiveXObject("Microsoft.XMLHTTP"):new XMLHttpRequest();xml.open(s.type,s.url,s.async);if(s.data)xml.setRequestHeader("Content-Type",s.contentType);if(s.ifModified)xml.setRequestHeader("If-Modified-Since",jQuery.lastModified[s.url]||"Thu, 01 Jan 1970 00:00:00 GMT");xml.setRequestHeader("X-Requested-With","XMLHttpRequest");if(s.beforeSend)s.beforeSend(xml);if(s.global)jQuery.event.trigger("ajaxSend",[xml,s]);var onreadystatechange=function(isTimeout){if(!requestDone&&xml&&(xml.readyState==4||isTimeout=="timeout")){requestDone=true;if(ival){clearInterval(ival);ival=null;}status=isTimeout=="timeout"&&"timeout"||!jQuery.httpSuccess(xml)&&"error"||s.ifModified&&jQuery.httpNotModified(xml,s.url)&&"notmodified"||"success";if(status=="success"){try{data=jQuery.httpData(xml,s.dataType);}catch(e){status="parsererror";}}if(status=="success"){var modRes;try{modRes=xml.getResponseHeader("Last-Modified");}catch(e){}if(s.ifModified&&modRes)jQuery.lastModified[s.url]=modRes;if(!jsonp)success();}else jQuery.handleError(s,xml,status);complete();if(s.async)xml=null;}};if(s.async){var ival=setInterval(onreadystatechange,13);if(s.timeout>0)setTimeout(function(){if(xml){xml.abort();if(!requestDone)onreadystatechange("timeout");}},s.timeout);}try{xml.send(s.data);}catch(e){jQuery.handleError(s,xml,null,e);}if(!s.async)onreadystatechange();return xml;function success(){if(s.success)s.success(data,status);if(s.global)jQuery.event.trigger("ajaxSuccess",[xml,s]);}function complete(){if(s.complete)s.complete(xml,status);if(s.global)jQuery.event.trigger("ajaxComplete",[xml,s]);if(s.global&&!--jQuery.active)jQuery.event.trigger("ajaxStop");}},handleError:function(s,xml,status,e){if(s.error)s.error(xml,status,e);if(s.global)jQuery.event.trigger("ajaxError",[xml,s,e]);},active:0,httpSuccess:function(r){try{return!r.status&&location.protocol=="file:"||(r.status>=200&&r.status<300)||r.status==304||jQuery.browser.safari&&r.status==undefined;}catch(e){}return false;},httpNotModified:function(xml,url){try{var xmlRes=xml.getResponseHeader("Last-Modified");return xml.status==304||xmlRes==jQuery.lastModified[url]||jQuery.browser.safari&&xml.status==undefined;}catch(e){}return false;},httpData:function(r,type){var ct=r.getResponseHeader("content-type");var xml=type=="xml"||!type&&ct&&ct.indexOf("xml")>=0;var data=xml?r.responseXML:r.responseText;if(xml&&data.documentElement.tagName=="parsererror")throw"parsererror";if(type=="script")jQuery.globalEval(data);if(type=="json")data=eval("("+data+")");return data;},param:function(a){var s=[];if(a.constructor==Array||a.jquery)jQuery.each(a,function(){s.push(encodeURIComponent(this.name)+"="+encodeURIComponent(this.value));});else for(var j in a)if(a[j]&&a[j].constructor==Array)jQuery.each(a[j],function(){s.push(encodeURIComponent(j)+"="+encodeURIComponent(this));});else s.push(encodeURIComponent(j)+"="+encodeURIComponent(a[j]));return s.join("&").replace(/%20/g,"+");}});jQuery.fn.extend({show:function(speed,callback){return speed?this.animate({height:"show",width:"show",opacity:"show"},speed,callback):this.filter(":hidden").each(function(){this.style.display=this.oldblock?this.oldblock:"";if(jQuery.css(this,"display")=="none")this.style.display="block";}).end();},hide:function(speed,callback){return speed?this.animate({height:"hide",width:"hide",opacity:"hide"},speed,callback):this.filter(":visible").each(function(){this.oldblock=this.oldblock||jQuery.css(this,"display");if(this.oldblock=="none")this.oldblock="block";this.style.display="none";}).end();},_toggle:jQuery.fn.toggle,toggle:function(fn,fn2){return jQuery.isFunction(fn)&&jQuery.isFunction(fn2)?this._toggle(fn,fn2):fn?this.animate({height:"toggle",width:"toggle",opacity:"toggle"},fn,fn2):this.each(function(){jQuery(this)[jQuery(this).is(":hidden")?"show":"hide"]();});},slideDown:function(speed,callback){return this.animate({height:"show"},speed,callback);},slideUp:function(speed,callback){return this.animate({height:"hide"},speed,callback);},slideToggle:function(speed,callback){return this.animate({height:"toggle"},speed,callback);},fadeIn:function(speed,callback){return this.animate({opacity:"show"},speed,callback);},fadeOut:function(speed,callback){return this.animate({opacity:"hide"},speed,callback);},fadeTo:function(speed,to,callback){return this.animate({opacity:to},speed,callback);},animate:function(prop,speed,easing,callback){var opt=jQuery.speed(speed,easing,callback);return this[opt.queue===false?"each":"queue"](function(){opt=jQuery.extend({},opt);var hidden=jQuery(this).is(":hidden"),self=this;for(var p in prop){if(prop[p]=="hide"&&hidden||prop[p]=="show"&&!hidden)return jQuery.isFunction(opt.complete)&&opt.complete.apply(this);if(p=="height"||p=="width"){opt.display=jQuery.css(this,"display");opt.overflow=this.style.overflow;}}if(opt.overflow!=null)this.style.overflow="hidden";opt.curAnim=jQuery.extend({},prop);jQuery.each(prop,function(name,val){var e=new jQuery.fx(self,opt,name);if(/toggle|show|hide/.test(val))e[val=="toggle"?hidden?"show":"hide":val](prop);else{var parts=val.toString().match(/^([+-]?)([\d.]+)(.*)$/),start=e.cur(true)||0;if(parts){end=parseFloat(parts[2]),unit=parts[3]||"px";if(unit!="px"){self.style[name]=end+unit;start=(end/e.cur(true))*start;self.style[name]=start+unit;}if(parts[1])end=((parts[1]=="-"?-1:1)*end)+start;e.custom(start,end,unit);}else e.custom(start,val,"");}});return true;});},queue:function(type,fn){if(!fn){fn=type;type="fx";}if(!arguments.length)return queue(this[0],type);return this.each(function(){if(fn.constructor==Array)queue(this,type,fn);else{queue(this,type).push(fn);if(queue(this,type).length==1)fn.apply(this);}});},stop:function(){var timers=jQuery.timers;return this.each(function(){for(var i=0;i-10000?r:parseFloat(jQuery.css(this.elem,this.prop))||0;},custom:function(from,to,unit){this.startTime=(new Date()).getTime();this.start=from;this.end=to;this.unit=unit||this.unit||"px";this.now=this.start;this.pos=this.state=0;this.update();var self=this;function t(){return self.step();}t.elem=this.elem;jQuery.timers.push(t);if(jQuery.timers.length==1){var timer=setInterval(function(){var timers=jQuery.timers;for(var i=0;ithis.options.duration+this.startTime){this.now=this.end;this.pos=this.state=1;this.update();this.options.curAnim[this.prop]=true;var done=true;for(var i in this.options.curAnim)if(this.options.curAnim[i]!==true)done=false;if(done){if(this.options.display!=null){this.elem.style.overflow=this.options.overflow;this.elem.style.display=this.options.display;if(jQuery.css(this.elem,"display")=="none")this.elem.style.display="block";}if(this.options.hide)this.elem.style.display="none";if(this.options.hide||this.options.show)for(var p in this.options.curAnim)jQuery.attr(this.elem.style,p,this.options.orig[p]);}if(done&&jQuery.isFunction(this.options.complete))this.options.complete.apply(this.elem);return false;}else{var n=t-this.startTime;this.state=n/this.options.duration;this.pos=jQuery.easing[this.options.easing||(jQuery.easing.swing?"swing":"linear")](this.state,n,0,1,this.options.duration);this.now=this.start+((this.end-this.start)*this.pos);this.update();}return true;}};jQuery.fx.step={scrollLeft:function(fx){fx.elem.scrollLeft=fx.now;},scrollTop:function(fx){fx.elem.scrollTop=fx.now;},opacity:function(fx){jQuery.attr(fx.elem.style,"opacity",fx.now);},_default:function(fx){fx.elem.style[fx.prop]=fx.now+fx.unit;}};jQuery.fn.offset=function(){var left=0,top=0,elem=this[0],results;if(elem)with(jQuery.browser){var absolute=jQuery.css(elem,"position")=="absolute",parent=elem.parentNode,offsetParent=elem.offsetParent,doc=elem.ownerDocument,safari2=safari&&!absolute&&parseInt(version)<522;if(elem.getBoundingClientRect){box=elem.getBoundingClientRect();add(box.left+Math.max(doc.documentElement.scrollLeft,doc.body.scrollLeft),box.top+Math.max(doc.documentElement.scrollTop,doc.body.scrollTop));if(msie){var border=jQuery("html").css("borderWidth");border=(border=="medium"||jQuery.boxModel&&parseInt(version)>=7)&&2||border;add(-border,-border);}}else{add(elem.offsetLeft,elem.offsetTop);while(offsetParent){add(offsetParent.offsetLeft,offsetParent.offsetTop);if(mozilla&&/^t[d|h]$/i.test(parent.tagName)||!safari2)border(offsetParent);if(safari2&&!absolute&&jQuery.css(offsetParent,"position")=="absolute")absolute=true;offsetParent=offsetParent.offsetParent;}while(parent.tagName&&/^body|html$/i.test(parent.tagName)){if(/^inline|table-row.*$/i.test(jQuery.css(parent,"display")))add(-parent.scrollLeft,-parent.scrollTop);if(mozilla&&jQuery.css(parent,"overflow")!="visible")border(parent);parent=parent.parentNode;}if(safari&&absolute)add(-doc.body.offsetLeft,-doc.body.offsetTop);}results={top:top,left:left};}return results;function border(elem){add(jQuery.css(elem,"borderLeftWidth"),jQuery.css(elem,"borderTopWidth"));}function add(l,t){left+=parseInt(l)||0;top+=parseInt(t)||0;}};})(); ================================================ FILE: docs/asciidoc-cheatsheet_files/pygments.css ================================================ .highlight { background: #f4f4f4; } .highlight .hll { background-color: #ffffcc } .highlight .c { color: #408080; font-style: italic } /* Comment */ .highlight .err { border: 1px solid #FF0000 } /* Error */ .highlight .k { color: #008000; font-weight: bold } /* Keyword */ .highlight .o { color: #666666 } /* Operator */ .highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */ .highlight .cp { color: #BC7A00 } /* Comment.Preproc */ .highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */ .highlight .cs { color: #408080; font-style: italic } /* Comment.Special */ .highlight .gd { color: #A00000 } /* Generic.Deleted */ .highlight .ge { font-style: italic } /* Generic.Emph */ .highlight .gr { color: #FF0000 } /* Generic.Error */ .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ .highlight .gi { color: #00A000 } /* Generic.Inserted */ .highlight .go { color: #808080 } /* Generic.Output */ .highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */ .highlight .gs { font-weight: bold } /* Generic.Strong */ .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ .highlight .gt { color: #0040D0 } /* Generic.Traceback */ .highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */ .highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ .highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ .highlight .kp { color: #008000 } /* Keyword.Pseudo */ .highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ .highlight .kt { color: #B00040 } /* Keyword.Type */ .highlight .m { color: #666666 } /* Literal.Number */ .highlight .s { color: #BA2121 } /* Literal.String */ .highlight .na { color: #7D9029 } /* Name.Attribute */ .highlight .nb { color: #008000 } /* Name.Builtin */ .highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */ .highlight .no { color: #880000 } /* Name.Constant */ .highlight .nd { color: #AA22FF } /* Name.Decorator */ .highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */ .highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */ .highlight .nf { color: #0000FF } /* Name.Function */ .highlight .nl { color: #A0A000 } /* Name.Label */ .highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ .highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */ .highlight .nv { color: #19177C } /* Name.Variable */ .highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ .highlight .w { color: #bbbbbb } /* Text.Whitespace */ .highlight .mf { color: #666666 } /* Literal.Number.Float */ .highlight .mh { color: #666666 } /* Literal.Number.Hex */ .highlight .mi { color: #666666 } /* Literal.Number.Integer */ .highlight .mo { color: #666666 } /* Literal.Number.Oct */ .highlight .sb { color: #BA2121 } /* Literal.String.Backtick */ .highlight .sc { color: #BA2121 } /* Literal.String.Char */ .highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ .highlight .s2 { color: #BA2121 } /* Literal.String.Double */ .highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */ .highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */ .highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */ .highlight .sx { color: #008000 } /* Literal.String.Other */ .highlight .sr { color: #BB6688 } /* Literal.String.Regex */ .highlight .s1 { color: #BA2121 } /* Literal.String.Single */ .highlight .ss { color: #19177C } /* Literal.String.Symbol */ .highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */ .highlight .vc { color: #19177C } /* Name.Variable.Class */ .highlight .vg { color: #19177C } /* Name.Variable.Global */ .highlight .vi { color: #19177C } /* Name.Variable.Instance */ .highlight .il { color: #666666 } /* Literal.Number.Integer.Long */ ================================================ FILE: docs/asciidoc-userguide.html ================================================ AsciiDoc User Guide
AsciiDoc
Text based document generation

AsciiDoc is a text document format for writing notes, documentation, articles, books, ebooks, slideshows, web pages, blogs and UNIX man pages. AsciiDoc files can be translated to many formats including HTML, PDF, EPUB, man page. AsciiDoc is highly configurable: both the AsciiDoc source file syntax and the backend output markups (which can be almost any type of SGML/XML markup) can be customized and extended by the user.

This document

This is an overly large document, it probably needs to be refactored into a Tutorial, Quick Reference and Formal Reference.

If you’re new to AsciiDoc read this section and the Getting Started section and take a look at the example AsciiDoc (*.txt) source files in the distribution doc directory.

1. Introduction

AsciiDoc is a plain text human readable/writable document format that can be translated to DocBook or HTML using the asciidoc(1) command. You can then either use asciidoc(1) generated HTML directly or run asciidoc(1) DocBook output through your favorite DocBook toolchain or use the AsciiDoc a2x(1) toolchain wrapper to produce PDF, EPUB, DVI, LaTeX, PostScript, man page, HTML and text formats.

The AsciiDoc format is a useful presentation format in its own right: AsciiDoc markup is simple, intuitive and as such is easily proofed and edited.

AsciiDoc is light weight: it consists of a single Python script and a bunch of configuration files. Apart from asciidoc(1) and a Python interpreter, no other programs are required to convert AsciiDoc text files to DocBook or HTML. See Example AsciiDoc Documents below.

Text markup conventions tend to be a matter of (often strong) personal preference: if the default syntax is not to your liking you can define your own by editing the text based asciidoc(1) configuration files. You can also create configuration files to translate AsciiDoc documents to almost any SGML/XML markup.

asciidoc(1) comes with a set of configuration files to translate AsciiDoc articles, books and man pages to HTML or DocBook backend formats.

My AsciiDoc Itch

DocBook has emerged as the de facto standard Open Source documentation format. But DocBook is a complex language, the markup is difficult to read and even more difficult to write directly — I found I was spending more time typing markup tags, consulting reference manuals and fixing syntax errors, than I was writing the documentation.

2. Getting Started

2.1. Installing AsciiDoc

See the README and INSTALL files for install prerequisites and procedures. Packagers take a look at Packager Notes.

2.2. Example AsciiDoc Documents

The best way to quickly get a feel for AsciiDoc is to view the AsciiDoc web site and/or distributed examples:

  • Take a look at the linked examples on the AsciiDoc web site home page http://www.methods.co.nz/asciidoc/. Press the Page Source sidebar menu item to view corresponding AsciiDoc source.

  • Read the *.txt source files in the distribution ./doc directory along with the corresponding HTML and DocBook XML files.

3. AsciiDoc Document Types

There are three types of AsciiDoc documents: article, book and manpage. All document types share the same AsciiDoc format with some minor variations. If you are familiar with DocBook you will have noticed that AsciiDoc document types correspond to the same-named DocBook document types.

Use the asciidoc(1) -d (--doctype) option to specify the AsciiDoc document type — the default document type is article.

By convention the .txt file extension is used for AsciiDoc document source files.

3.1. article

Used for short documents, articles and general documentation. See the AsciiDoc distribution ./doc/article.txt example.

AsciiDoc defines standard DocBook article frontmatter and backmatter section markup templates (appendix, abstract, bibliography, glossary, index).

3.2. book

Books share the same format as articles, with the following differences:

  • The part titles in multi-part books are top level titles (same level as book title).

  • Some sections are book specific e.g. preface and colophon.

Book documents will normally be used to produce DocBook output since DocBook processors can automatically generate footnotes, table of contents, list of tables, list of figures, list of examples and indexes.

AsciiDoc defines standard DocBook book frontmatter and backmatter section markup templates (appendix, dedication, preface, bibliography, glossary, index, colophon).

Example book documents
Book

The ./doc/book.txt file in the AsciiDoc distribution.

Multi-part book

The ./doc/book-multi.txt file in the AsciiDoc distribution.

3.3. manpage

Used to generate roff format UNIX manual pages. AsciiDoc manpage documents observe special header title and section naming conventions — see the Manpage Documents section for details.

AsciiDoc defines the synopsis section markup template to generate the DocBook refsynopsisdiv section.

See also the asciidoc(1) man page source (./doc/asciidoc.1.txt) from the AsciiDoc distribution.

4. AsciiDoc Backends

The asciidoc(1) command translates an AsciiDoc formatted file to the backend format specified by the -b (--backend) command-line option. asciidoc(1) itself has little intrinsic knowledge of backend formats, all translation rules are contained in customizable cascading configuration files. Backend specific attributes are listed in the Backend Attributes section.

docbook45

Outputs DocBook XML 4.5 markup.

html4

This backend generates plain HTML 4.01 Transitional markup.

xhtml11

This backend generates XHTML 1.1 markup styled with CSS2. Output files have an .html extension.

html5

This backend generates HTML 5 markup, apart from the inclusion of audio and video block macros it is functionally identical to the xhtml11 backend.

slidy

Use this backend to generate self-contained Slidy HTML slideshows for your web browser from AsciiDoc documents. The Slidy backend is documented in the distribution doc/slidy.txt file and online.

wordpress

A minor variant of the html4 backend to support blogpost.

latex

Experimental LaTeX backend.

4.1. Backend Aliases

Backend aliases are alternative names for AsciiDoc backends. AsciiDoc comes with two backend aliases: html (aliased to xhtml11) and docbook (aliased to docbook45).

You can assign (or reassign) backend aliases by setting an AsciiDoc attribute named like backend-alias-<alias> to an AsciiDoc backend name. For example, the following backend alias attribute definitions appear in the [attributes] section of the global asciidoc.conf configuration file:

backend-alias-html=xhtml11
backend-alias-docbook=docbook45

4.2. Backend Plugins

The asciidoc(1) --backend option is also used to install and manage backend plugins.

  • A backend plugin is used just like the built-in backends.

  • Backend plugins take precedence over built-in backends with the same name.

  • You can use the {asciidoc-confdir} intrinsic attribute to refer to the built-in backend configuration file location from backend plugin configuration files.

  • You can use the {backend-confdir} intrinsic attribute to refer to the backend plugin configuration file location.

  • By default backends plugins are installed in $HOME/.asciidoc/backends/<backend> where <backend> is the backend name.

5. DocBook

AsciiDoc generates article, book and refentry DocBook documents (corresponding to the AsciiDoc article, book and manpage document types).

Most Linux distributions come with conversion tools (collectively called a toolchain) for converting DocBook files to presentation formats such as Postscript, HTML, PDF, EPUB, DVI, PostScript, LaTeX, roff (the native man page format), HTMLHelp, JavaHelp and text. There are also programs that allow you to view DocBook files directly, for example Yelp (the GNOME help viewer).

5.1. Converting DocBook to other file formats

DocBook files are validated, parsed and translated various presentation file formats using a combination of applications collectively called a DocBook tool chain. The function of a tool chain is to read the DocBook markup (produced by AsciiDoc) and transform it to a presentation format (for example HTML, PDF, HTML Help, EPUB, DVI, PostScript, LaTeX).

A wide range of user output format requirements coupled with a choice of available tools and stylesheets results in many valid tool chain combinations.

5.2. a2x Toolchain Wrapper

One of the biggest hurdles for new users is installing, configuring and using a DocBook XML toolchain. a2x(1) can help — it’s a toolchain wrapper command that will generate XHTML (chunked and unchunked), PDF, EPUB, DVI, PS, LaTeX, man page, HTML Help and text file outputs from an AsciiDoc text file. a2x(1) does all the grunt work associated with generating and sequencing the toolchain commands and managing intermediate and output files. a2x(1) also optionally deploys admonition and navigation icons and a CSS stylesheet. See the a2x(1) man page for more details. In addition to asciidoc(1) you also need xsltproc(1), DocBook XSL Stylesheets and optionally: dblatex or FOP (to generate PDF); w3m(1) or lynx(1) (to generate text).

The following examples generate doc/source-highlight-filter.pdf from the AsciiDoc doc/source-highlight-filter.txt source file. The first example uses dblatex(1) (the default PDF generator) the second example forces FOP to be used:

$ a2x -f pdf doc/source-highlight-filter.txt
$ a2x -f pdf --fop doc/source-highlight-filter.txt

See the a2x(1) man page for details.

Tip Use the --verbose command-line option to view executed toolchain commands.

5.3. HTML generation

AsciiDoc produces nicely styled HTML directly without requiring a DocBook toolchain but there are also advantages in going the DocBook route:

  • HTML from DocBook can optionally include automatically generated indexes, tables of contents, footnotes, lists of figures and tables.

  • DocBook toolchains can also (optionally) generate separate (chunked) linked HTML pages for each document section.

  • Toolchain processing performs link and document validity checks.

  • If the DocBook lang attribute is set then things like table of contents, figure and table captions and admonition captions will be output in the specified language (setting the AsciiDoc lang attribute sets the DocBook lang attribute).

On the other hand, HTML output directly from AsciiDoc is much faster, is easily customized and can be used in situations where there is no suitable DocBook toolchain (for example, see the AsciiDoc website).

5.4. PDF generation

There are two commonly used tools to generate PDFs from DocBook, dblatex and FOP.

dblatex or FOP?

  • dblatex is easier to install, there’s zero configuration required and no Java VM to install — it just works out of the box.

  • dblatex source code highlighting and numbering is superb.

  • dblatex is easier to use as it converts DocBook directly to PDF whereas before using FOP you have to convert DocBook to XML-FO using DocBook XSL Stylesheets.

  • FOP is more feature complete (for example, callouts are processed inside literal layouts) and arguably produces nicer looking output.

5.5. HTML Help generation

  1. Convert DocBook XML documents to HTML Help compiler source files using DocBook XSL Stylesheets and xsltproc(1).

  2. Convert the HTML Help source (.hhp and .html) files to HTML Help (.chm) files using the Microsoft HTML Help Compiler.

5.6. Toolchain components summary

AsciiDoc

Converts AsciiDoc (.txt) files to DocBook XML (.xml) files.

DocBook XSL Stylesheets

These are a set of XSL stylesheets containing rules for converting DocBook XML documents to HTML, XSL-FO, manpage and HTML Help files. The stylesheets are used in conjunction with an XML parser such as xsltproc(1).

xsltproc

An XML parser for applying XSLT stylesheets (in our case the DocBook XSL Stylesheets) to XML documents.

dblatex

Generates PDF, DVI, PostScript and LaTeX formats directly from DocBook source via the intermediate LaTeX typesetting language —  uses DocBook XSL Stylesheets, xsltproc(1) and latex(1).

FOP

The Apache Formatting Objects Processor converts XSL-FO (.fo) files to PDF files. The XSL-FO files are generated from DocBook source files using DocBook XSL Stylesheets and xsltproc(1).

Microsoft Help Compiler

The Microsoft HTML Help Compiler (hhc.exe) is a command-line tool that converts HTML Help source files to a single HTML Help (.chm) file. It runs on MS Windows platforms and can be downloaded from http://www.microsoft.com.

5.7. AsciiDoc dblatex configuration files

The AsciiDoc distribution ./dblatex directory contains asciidoc-dblatex.xsl (customized XSL parameter settings) and asciidoc-dblatex.sty (customized LaTeX settings). These are examples of optional dblatex output customization and are used by a2x(1).

5.8. AsciiDoc DocBook XSL Stylesheets drivers

You will have noticed that the distributed HTML and HTML Help documentation files (for example ./doc/asciidoc.html) are not the plain outputs produced using the default DocBook XSL Stylesheets configuration. This is because they have been processed using customized DocBook XSL Stylesheets along with (in the case of HTML outputs) the custom ./stylesheets/docbook-xsl.css CSS stylesheet.

You’ll find the customized DocBook XSL drivers along with additional documentation in the distribution ./docbook-xsl directory. The examples that follow are executed from the distribution documentation (./doc) directory. These drivers are also used by a2x(1).

common.xsl

Shared driver parameters. This file is not used directly but is included in all the following drivers.

chunked.xsl

Generate chunked XHTML (separate HTML pages for each document section) in the ./doc/chunked directory. For example: 

$ python ../asciidoc.py -b docbook asciidoc.txt
$ xsltproc --nonet ../docbook-xsl/chunked.xsl asciidoc.xml
epub.xsl

Used by a2x(1) to generate EPUB formatted documents.

fo.xsl

Generate XSL Formatting Object (.fo) files for subsequent PDF file generation using FOP. For example: 

$ python ../asciidoc.py -b docbook article.txt
$ xsltproc --nonet ../docbook-xsl/fo.xsl article.xml > article.fo
$ fop article.fo article.pdf
htmlhelp.xsl

Generate Microsoft HTML Help source files for the MS HTML Help Compiler in the ./doc/htmlhelp directory. This example is run on MS Windows from a Cygwin shell prompt: 

$ python ../asciidoc.py -b docbook asciidoc.txt
$ xsltproc --nonet ../docbook-xsl/htmlhelp.xsl asciidoc.xml
$ c:/Program\ Files/HTML\ Help\ Workshop/hhc.exe htmlhelp.hhp
manpage.xsl

Generate a roff(1) format UNIX man page from a DocBook XML refentry document. This example generates an asciidoc.1 man page file: 

$ python ../asciidoc.py -d manpage -b docbook asciidoc.1.txt
$ xsltproc --nonet ../docbook-xsl/manpage.xsl asciidoc.1.xml
xhtml.xsl

Convert a DocBook XML file to a single XHTML file. For example: 

$ python ../asciidoc.py -b docbook asciidoc.txt
$ xsltproc --nonet ../docbook-xsl/xhtml.xsl asciidoc.xml > asciidoc.html

If you want to see how the complete documentation set is processed take a look at the A-A-P script ./doc/main.aap.

6. Generating Plain Text Files

AsciiDoc does not have a text backend (for most purposes AsciiDoc source text is fine), however you can convert AsciiDoc text files to formatted text using the AsciiDoc a2x(1) toolchain wrapper utility.

7. HTML5 and XHTML 1.1

The xhtml11 and html5 backends embed or link CSS and JavaScript files in their outputs, there is also a themes plugin framework.

  • If the AsciiDoc linkcss attribute is defined then CSS and JavaScript files are linked to the output document, otherwise they are embedded (the default behavior).

  • The default locations for CSS and JavaScript files can be changed by setting the AsciiDoc stylesdir and scriptsdir attributes respectively.

  • The default locations for embedded and linked files differ and are calculated at different times — embedded files are loaded when asciidoc(1) generates the output document, linked files are loaded by the browser when the user views the output document.

  • Embedded files are automatically inserted in the output files but you need to manually copy linked CSS and Javascript files from AsciiDoc configuration directories to the correct location relative to the output document.

Table 1. Stylesheet file locations
stylesdir attribute Linked location (linkcss attribute defined) Embedded location (linkcss attribute undefined)

Undefined (default).

Same directory as the output document.

stylesheets subdirectory in the AsciiDoc configuration directory (the directory containing the backend conf file).

Absolute or relative directory name.

Absolute or relative to the output document.

Absolute or relative to the AsciiDoc configuration directory (the directory containing the backend conf file).
Table 2. JavaScript file locations
scriptsdir attribute Linked location (linkcss attribute defined) Embedded location (linkcss attribute undefined)

Undefined (default).

Same directory as the output document.

javascripts subdirectory in the AsciiDoc configuration directory (the directory containing the backend conf file).

Absolute or relative directory name.

Absolute or relative to the output document.

Absolute or relative to the AsciiDoc configuration directory (the directory containing the backend conf file).

7.1. Themes

The AsciiDoc theme attribute is used to select an alternative CSS stylesheet and to optionally include additional JavaScript code.

  • Theme files reside in an AsciiDoc configuration directory named themes/<theme>/ (where <theme> is the the theme name set by the theme attribute). asciidoc(1) sets the themedir attribute to the theme directory path name.

  • The theme attribute can also be set using the asciidoc(1) --theme option, the --theme option can also be used to manage theme plugins.

  • AsciiDoc ships with two themes: flask and volnitsky.

  • The <theme>.css file replaces the default asciidoc.css CSS file.

  • The <theme>.js file is included in addition to the default asciidoc.js JavaScript file.

  • If the data-uri attribute is defined then icons are loaded from the theme icons sub-directory if it exists (i.e. the iconsdir attribute is set to theme icons sub-directory path).

  • Embedded theme files are automatically inserted in the output files but you need to manually copy linked CSS and Javascript files to the location of the output documents.

  • Linked CSS and JavaScript theme files are linked to the same linked locations as other CSS and JavaScript files.

For example, the command-line option --theme foo (or --attribute theme=foo) will cause asciidoc(1) to search configuration file locations 1, 2 and 3 for a sub-directory called themes/foo containing the stylesheet foo.css and optionally a JavaScript file name foo.js.

8. Document Structure

An AsciiDoc document consists of a series of block elements starting with an optional document Header, followed by an optional Preamble, followed by zero or more document Sections.

Almost any combination of zero or more elements constitutes a valid AsciiDoc document: documents can range from a single sentence to a multi-part book.

8.1. Block Elements

Block elements consist of one or more lines of text and may contain other block elements.

The AsciiDoc block structure can be informally summarized as follows
[This is a rough structural guide, not a rigorous syntax definition]
:

Document      ::= (Header?,Preamble?,Section*)
Header        ::= (Title,(AuthorInfo,RevisionInfo?)?)
AuthorInfo    ::= (FirstName,(MiddleName?,LastName)?,EmailAddress?)
RevisionInfo  ::= (RevisionNumber?,RevisionDate,RevisionRemark?)
Preamble      ::= (SectionBody)
Section       ::= (Title,SectionBody?,(Section)*)
SectionBody   ::= ((BlockTitle?,Block)|BlockMacro)+
Block         ::= (Paragraph|DelimitedBlock|List|Table)
List          ::= (BulletedList|NumberedList|LabeledList|CalloutList)
BulletedList  ::= (ListItem)+
NumberedList  ::= (ListItem)+
CalloutList   ::= (ListItem)+
LabeledList   ::= (ListEntry)+
ListEntry     ::= (ListLabel,ListItem)
ListLabel     ::= (ListTerm+)
ListItem      ::= (ItemText,(List|ListParagraph|ListContinuation)*)

Where:

  • ? implies zero or one occurrence, + implies one or more occurrences, * implies zero or more occurrences.

  • All block elements are separated by line boundaries.

  • BlockId, AttributeEntry and AttributeList block elements (not shown) can occur almost anywhere.

  • There are a number of document type and backend specific restrictions imposed on the block syntax.

  • The following elements cannot contain blank lines: Header, Title, Paragraph, ItemText.

  • A ListParagraph is a Paragraph with its listelement option set.

  • A ListContinuation is a list continuation element.

8.2. Header

The Header contains document meta-data, typically title plus optional authorship and revision information:

  • The Header is optional, but if it is used it must start with a document title.

  • Optional Author and Revision information immediately follows the header title.

  • The document header must be separated from the remainder of the document by one or more blank lines and cannot contain blank lines.

  • The header can include comments.

  • The header can include attribute entries, typically doctype, lang, encoding, icons, data-uri, toc, numbered.

  • Header attributes are overridden by command-line attributes.

  • If the header contains non-UTF-8 characters then the encoding must precede the header (either in the document or on the command-line).

Here’s an example AsciiDoc document header:

Writing Documentation using AsciiDoc
====================================
Joe Bloggs <jbloggs@mymail.com>
v2.0, February 2003:
Rewritten for version 2 release.

The author information line contains the author’s name optionally followed by the author’s email address. The author’s name is formatted like:

firstname[ [middlename ]lastname][ <email>]]

i.e. a first name followed by optional middle and last names followed by an email address in that order. Multi-word first, middle and last names can be entered using the underscore as a word separator. The email address comes last and must be enclosed in angle <> brackets. Here a some examples of author information lines:

Joe Bloggs <jbloggs@mymail.com>
Joe Bloggs
Vincent Willem van_Gogh

If the author line does not match the above specification then the entire author line is treated as the first name.

The optional revision information line follows the author information line. The revision information can be one of two formats:

  1. An optional document revision number followed by an optional revision date followed by an optional revision remark:

    • If the revision number is specified it must be followed by a comma.

    • The revision number must contain at least one numeric character.

    • Any non-numeric characters preceding the first numeric character will be dropped.

    • If a revision remark is specified it must be preceded by a colon. The revision remark extends from the colon up to the next blank line, attribute entry or comment and is subject to normal text substitutions.

    • If a revision number or remark has been set but the revision date has not been set then the revision date is set to the value of the docdate attribute.

    Examples:

    v2.0, February 2003
February 2003
v2.0,
v2.0, February 2003: Rewritten for version 2 release.
February 2003: Rewritten for version 2 release.
v2.0,: Rewritten for version 2 release.
:Rewritten for version 2 release.

  2. The revision information line can also be an RCS/CVS/SVN $Id$ marker:

    • AsciiDoc extracts the revnumber, revdate, and author attributes from the $Id$ revision marker and displays them in the document header.

    • If an $Id$ revision marker is used the header author line can be omitted.

    Example:

    $Id: mydoc.txt,v 1.5 2009/05/17 17:58:44 jbloggs Exp $

You can override or set header parameters by passing revnumber, revremark, revdate, email, author, authorinitials, firstname and lastname attributes using the asciidoc(1) -a (--attribute) command-line option. For example:

$ asciidoc -a revdate=2004/07/27 article.txt

Attribute entries can also be added to the header for substitution in the header template with Attribute Entry elements.

The title element in HTML outputs is set to the AsciiDoc document title, you can set it to a different value by including a title attribute entry in the document header.

8.2.1. Additional document header information

AsciiDoc has two mechanisms for optionally including additional meta-data in the header of the output document:

docinfo configuration file sections

If a configuration file section named docinfo has been loaded then it will be included in the document header. Typically the docinfo section name will be prefixed with a + character so that it is appended to (rather than replace) other docinfo sections.

docinfo files

Two docinfo files are recognized: one named docinfo and a second named like the AsciiDoc source file with a -docinfo suffix. For example, if the source document is called mydoc.txt then the document information files would be docinfo.xml and mydoc-docinfo.xml (for DocBook outputs) and docinfo.html and mydoc-docinfo.html (for HTML outputs). The docinfo attributes control which docinfo files are included in the output files.

The contents docinfo templates and files is dependent on the type of output:

HTML

Valid head child elements. Typically style and script elements for CSS and JavaScript inclusion.

DocBook

Valid articleinfo or bookinfo child elements. DocBook defines numerous elements for document meta-data, for example: copyrights, document history and authorship information. See the DocBook ./doc/article-docinfo.xml example that comes with the AsciiDoc distribution. The rendering of meta-data elements (or not) is DocBook processor dependent.

8.3. Preamble

The Preamble is an optional untitled section body between the document Header and the first Section title.

8.4. Sections

In addition to the document title (level 0), AsciiDoc supports four section levels: 1 (top) to 4 (bottom). Section levels are delimited by section titles. Sections are translated using configuration file section markup templates. AsciiDoc generates the following intrinsic attributes specifically for use in section markup templates:

level

The level attribute is the section level number, it is normally just the title level number (1..4). However, if the leveloffset attribute is defined it will be added to the level attribute. The leveloffset attribute is useful for combining documents.

sectnum

The -n (--section-numbers) command-line option generates the sectnum (section number) attribute. The sectnum attribute is used for section numbers in HTML outputs (DocBook section numbering are handled automatically by the DocBook toolchain commands).

8.4.1. Section markup templates

Section markup templates specify output markup and are defined in AsciiDoc configuration files. Section markup template names are derived as follows (in order of precedence):

  1. From the title’s first positional attribute or template attribute. For example, the following three section titles are functionally equivalent: 

    [[terms]]
[glossary]
List of Terms
-------------

["glossary",id="terms"]
List of Terms
-------------

[template="glossary",id="terms"]
List of Terms
-------------

  2. When the title text matches a configuration file [specialsections] entry.

  3. If neither of the above the default sect<level> template is used (where <level> is a number from 1 to 4).

In addition to the normal section template names (sect1, sect2, sect3, sect4) AsciiDoc has the following templates for frontmatter, backmatter and other special sections: abstract, preface, colophon, dedication, glossary, bibliography, synopsis, appendix, index. These special section templates generate the corresponding Docbook elements; for HTML outputs they default to the sect1 section template.

8.4.2. Section IDs

If no explicit section ID is specified an ID will be synthesised from the section title. The primary purpose of this feature is to ensure persistence of table of contents links (permalinks): the missing section IDs are generated dynamically by the JavaScript TOC generator after the page is loaded. If you link to a dynamically generated TOC address the page will load but the browser will ignore the (as yet ungenerated) section ID.

The IDs are generated by the following algorithm:

  • Replace all non-alphanumeric title characters with underscores.

  • Strip leading or trailing underscores.

  • Convert to lowercase.

  • Prepend the idprefix attribute (so there’s no possibility of name clashes with existing document IDs). Prepend an underscore if the idprefix attribute is not defined.

  • A numbered suffix (_2, _3 …) is added if a same named auto-generated section ID exists.

  • If the ascii-ids attribute is defined then non-ASCII characters are replaced with ASCII equivalents. This attribute may be deprecated in future releases and should be avoided, it’s sole purpose is to accommodate deficient downstream applications that cannot process non-ASCII ID attributes.

Example: the title Jim’s House would generate the ID _jim_s_house.

Section ID synthesis can be disabled by undefining the sectids attribute.

8.4.3. Special Section Titles

AsciiDoc has a mechanism for mapping predefined section titles auto-magically to specific markup templates. For example a title Appendix A: Code Reference will automatically use the appendix section markup template. The mappings from title to template name are specified in [specialsections] sections in the Asciidoc language configuration files (lang-*.conf). Section entries are formatted like:

<title>=<template>

<title> is a Python regular expression and <template> is the name of a configuration file markup template section. If the <title> matches an AsciiDoc document section title then the backend output is marked up using the <template> markup template (instead of the default sect<level> section template). The {title} attribute value is set to the value of the matched regular expression group named title, if there is no title group {title} defaults to the whole of the AsciiDoc section title. If <template> is blank then any existing entry with the same <title> will be deleted.

Special section titles vs. explicit template names

AsciiDoc has two mechanisms for specifying non-default section markup templates: you can specify the template name explicitly (using the template attribute) or indirectly (using special section titles). Specifying a section template attribute explicitly is preferred. Auto-magical special section titles have the following drawbacks:

  • They are non-obvious, you have to know the exact matching title for each special section on a language by language basis.

  • Section titles are predefined and can only be customised with a configuration change.

  • The implementation is complicated by multiple languages: every special section title has to be defined for each language (in each of the lang-*.conf files).

Specifying special section template names explicitly does add more noise to the source document (the template attribute declaration), but the intention is obvious and the syntax is consistent with other AsciiDoc elements c.f. bibliographic, Q&A and glossary lists.

Special section titles have been deprecated but are retained for backward compatibility.

8.5. Inline Elements

Inline document elements are used to format text and to perform various types of text substitution. Inline elements and inline element syntax is defined in the asciidoc(1) configuration files.

Here is a list of AsciiDoc inline elements in the (default) order in which they are processed:

Special characters

These character sequences escape special characters used by the backend markup (typically <, >, and & characters). See [specialcharacters] configuration file sections.

Quotes

Elements that markup words and phrases; usually for character formatting. See [quotes] configuration file sections.

Special Words

Word or word phrase patterns singled out for markup without the need for further annotation. See [specialwords] configuration file sections.

Replacements

Each replacement defines a word or word phrase pattern to search for along with corresponding replacement text. See [replacements] configuration file sections.

Attribute references

Document attribute names enclosed in braces are replaced by the corresponding attribute value.

Inline Macros

Inline macros are replaced by the contents of parametrized configuration file sections.

9. Document Processing

The AsciiDoc source document is read and processed as follows:

  1. The document Header is parsed, header parameter values are substituted into the configuration file [header] template section which is then written to the output file.

  2. Each document Section is processed and its constituent elements translated to the output file.

  3. The configuration file [footer] template section is substituted and written to the output file.

When a block element is encountered asciidoc(1) determines the type of block by checking in the following order (first to last): (section) Titles, BlockMacros, Lists, DelimitedBlocks, Tables, AttributeEntrys, AttributeLists, BlockTitles, Paragraphs.

The default paragraph definition [paradef-default] is last element to be checked.

Knowing the parsing order will help you devise unambiguous macro, list and block syntax rules.

Inline substitutions within block elements are performed in the following default order:

  1. Special characters

  2. Quotes

  3. Special words

  4. Replacements

  5. Attributes

  6. Inline Macros

  7. Replacements2

The substitutions and substitution order performed on Title, Paragraph and DelimitedBlock elements is determined by configuration file parameters.

10. Text Formatting

10.1. Quoted Text

Words and phrases can be formatted by enclosing inline text with quote characters:

Emphasized text

Word phrases 'enclosed in single quote characters' (acute accents) or _underline characters_ are emphasized.

Strong text

Word phrases *enclosed in asterisk characters* are rendered in a strong font (usually bold).

Monospaced text

Word phrases +enclosed in plus characters+ are rendered in a monospaced font. Word phrases `enclosed in backtick characters` (grave accents) are also rendered in a monospaced font but in this case the enclosed text is rendered literally and is not subject to further expansion (see inline literal passthrough).

‘Single quoted text’

Phrases enclosed with a `single grave accent to the left and a single acute accent to the right' are rendered in single quotation marks.

“Double quoted text”

Phrases enclosed with ``two grave accents to the left and two acute accents to the right'' are rendered in quotation marks.

Unquoted text

Placing #hashes around text# does nothing, it is a mechanism to allow inline attributes to be applied to otherwise unformatted text.

New quote types can be defined by editing asciidoc(1) configuration files. See the Configuration Files section for details.

Quoted text behavior

  • Quoting cannot be overlapped.

  • Different quoting types can be nested.

  • To suppress quoted text formatting place a backslash character immediately in front of the leading quote character(s). In the case of ambiguity between escaped and non-escaped text you will need to escape both leading and trailing quotes, in the case of multi-character quotes you may even need to escape individual characters.

10.1.1. Quoted text attributes

Quoted text can be prefixed with an attribute list. The first positional attribute (role attribute) is translated by AsciiDoc to an HTML span element class attribute or a DocBook phrase element role attribute.

DocBook XSL Stylesheets translate DocBook phrase elements with role attributes to corresponding HTML span elements with the same class attributes; CSS can then be used to style the generated HTML. Thus CSS styling can be applied to both DocBook and AsciiDoc generated HTML outputs. You can also specify multiple class names separated by spaces.

CSS rules for text color, text background color, text size and text decorators are included in the distributed AsciiDoc CSS files and are used in conjunction with AsciiDoc xhtml11, html5 and docbook outputs. The CSS class names are:

  • <color> (text foreground color).

  • <color>-background (text background color).

  • big and small (text size).

  • underline, overline and line-through (strike through) text decorators.

Where <color> can be any of the sixteen HTML color names. Examples:

[red]#Obvious# and [big red yellow-background]*very obvious*.
[underline]#Underline text#, [overline]#overline text# and
[blue line-through]*bold blue and line-through*.

is rendered as:

Obvious and very obvious.

Underline text, overline text and bold blue and line-through.

Note Color and text decorator attributes are rendered for XHTML and HTML 5 outputs using CSS stylesheets. The mechanism to implement color and text decorator attributes is provided for DocBook toolchains via the DocBook phrase element role attribute, but the actual rendering is toolchain specific and is not part of the AsciiDoc distribution.

10.1.2. Constrained and Unconstrained Quotes

There are actually two types of quotes:

Constrained quotes

Quoted must be bounded by white space or commonly adjoining punctuation characters. These are the most commonly used type of quote.

Unconstrained quotes

Unconstrained quotes have no boundary constraints and can be placed anywhere within inline text. For consistency and to make them easier to remember unconstrained quotes are double-ups of the _, *, + and # constrained quotes:

__unconstrained emphasized text__
**unconstrained strong text**
++unconstrained monospaced text++
##unconstrained unquoted text##

The following example emboldens the letter F:

**F**ile Open...

10.2. Superscripts and Subscripts

Put ^carets on either^ side of the text to be superscripted, put ~tildes on either side~ of text to be subscripted. For example, the following line:

e^&#960;i^+1 = 0. H~2~O and x^10^. Some ^super text^
and ~some sub text~

Is rendered like:

eπi+1 = 0. H2O and x10. Some super text and some sub text

Superscripts and subscripts are implemented as unconstrained quotes and they can be escaped with a leading backslash and prefixed with with an attribute list.

10.3. Line Breaks

A plus character preceded by at least one space character at the end of a non-blank line forces a line break. It generates a line break (br) tag for HTML outputs and a custom XML asciidoc-br processing instruction for DocBook outputs. The asciidoc-br processing instruction is handled by a2x(1).

10.4. Page Breaks

A line of three or more less-than (<<<) characters will generate a hard page break in DocBook and printed HTML outputs. It uses the CSS page-break-after property for HTML outputs and a custom XML asciidoc-pagebreak processing instruction for DocBook outputs. The asciidoc-pagebreak processing instruction is handled by a2x(1). Hard page breaks are sometimes handy but as a general rule you should let your page processor generate page breaks for you.

10.5. Rulers

A line of three or more apostrophe characters will generate a ruler line. It generates a ruler (hr) tag for HTML outputs and a custom XML asciidoc-hr processing instruction for DocBook outputs. The asciidoc-hr processing instruction is handled by a2x(1).

10.6. Tabs

By default tab characters input files will translated to 8 spaces. Tab expansion is set with the tabsize entry in the configuration file [miscellaneous] section and can be overridden in included files by setting a tabsize attribute in the include macro’s attribute list. For example:

include::addendum.txt[tabsize=2]

The tab size can also be set using the attribute command-line option, for example --attribute tabsize=4

10.7. Replacements

The following replacements are defined in the default AsciiDoc configuration:

(C) copyright, (TM) trademark, (R) registered trademark,
-- em dash, ... ellipsis, -> right arrow, <- left arrow, => right
double arrow, <= left double arrow.

Which are rendered as:

© copyright, ™ trademark, ® registered trademark, — em dash, … ellipsis, → right arrow, ← left arrow, ⇒ right double arrow, ⇐ left double arrow.

You can also include arbitrary entity references in the AsciiDoc source. Examples:

&#x278a; &#182;

renders:

➊ ¶

To render a replacement literally escape it with a leading back-slash.

The Configuration Files section explains how to configure your own replacements.

10.8. Special Words

Words defined in [specialwords] configuration file sections are automatically marked up without having to be explicitly notated.

The Configuration Files section explains how to add and replace special words.

11. Titles

Document and section titles can be in either of two formats:

11.1. Two line titles

A two line title consists of a title line, starting hard against the left margin, and an underline. Section underlines consist a repeated character pairs spanning the width of the preceding title (give or take up to two characters):

The default title underlines for each of the document levels are:

Level 0 (top level):     ======================
Level 1:                 ----------------------
Level 2:                 ~~~~~~~~~~~~~~~~~~~~~~
Level 3:                 ^^^^^^^^^^^^^^^^^^^^^^
Level 4 (bottom level):  ++++++++++++++++++++++

Examples:

Level One Section Title
-----------------------
Level 2 Subsection Title
~~~~~~~~~~~~~~~~~~~~~~~~

11.2. One line titles

One line titles consist of a single line delimited on either side by one or more equals characters (the number of equals characters corresponds to the section level minus one). Here are some examples:

= Document Title (level 0) =
== Section title (level 1) ==
=== Section title (level 2) ===
==== Section title (level 3) ====
===== Section title (level 4) =====
Note

  • One or more spaces must fall between the title and the delimiters.

  • The trailing title delimiter is optional.

  • The one-line title syntax can be changed by editing the configuration file [titles] section sect0…sect4 entries.

11.3. Floating titles

Setting the title’s first positional attribute or style attribute to float generates a free-floating title. A free-floating title is rendered just like a normal section title but is not formally associated with a text body and is not part of the regular section hierarchy so the normal ordering rules do not apply. Floating titles can also be used in contexts where section titles are illegal: for example sidebar and admonition blocks. Example:

[float]
The second day
~~~~~~~~~~~~~~

Floating titles do not appear in a document’s table of contents.

12. Block Titles

A BlockTitle element is a single line beginning with a period followed by the title text. A BlockTitle is applied to the immediately following Paragraph, DelimitedBlock, List, Table or BlockMacro. For example:

.Notes
- Note 1.
- Note 2.

is rendered as:

Notes

  • Note 1.

  • Note 2.

13. BlockId Element

A BlockId is a single line block element containing a unique identifier enclosed in double square brackets. It is used to assign an identifier to the ensuing block element. For example:

[[chapter-titles]]
Chapter titles can be ...

The preceding example identifies the ensuing paragraph so it can be referenced from other locations, for example with <<chapter-titles,chapter titles>>.

BlockId elements can be applied to Title, Paragraph, List, DelimitedBlock, Table and BlockMacro elements. The BlockId element sets the {id} attribute for substitution in the subsequent block’s markup template. If a second positional argument is supplied it sets the {reftext} attribute which is used to set the DocBook xreflabel attribute.

The BlockId element has the same syntax and serves the same function to the anchor inline macro.

14. AttributeList Element

An AttributeList block element is an attribute list on a line by itself:

  • AttributeList attributes are only applied to the immediately following block element — the attributes are made available to the block’s markup template.

  • Multiple contiguous AttributeList elements are additively combined in the order they appear..

  • The first positional attribute in the list is often used to specify the ensuing element’s style.

14.1. Attribute value substitution

By default, only substitutions that take place inside attribute list values are attribute references, this is because not all attributes are destined to be marked up and rendered as text (for example the table cols attribute). To perform normal inline text substitutions (special characters, quotes, macros, replacements) on an attribute value you need to enclose it in single quotes. In the following quote block the second attribute value in the AttributeList is quoted to ensure the http macro is expanded to a hyperlink.

[quote,'http://en.wikipedia.org/wiki/Samuel_Johnson[Samuel Johnson]']
_____________________________________________________________________
Sir, a woman's preaching is like a dog's walking on his hind legs. It
is not done well; but you are surprised to find it done at all.
_____________________________________________________________________

14.2. Common attributes

Most block elements support the following attributes:

Name Backends Description

id

html4, html5, xhtml11, docbook

Unique identifier typically serve as link targets. Can also be set by the BlockId element.

role

html4, html5, xhtml11, docbook

Role contains a string used to classify or subclassify an element and can be applied to AsciiDoc block elements. The AsciiDoc role attribute is translated to the role attribute in DocBook outputs and is included in the class attribute in HTML outputs, in this respect it behaves like the quoted text role attribute.

DocBook XSL Stylesheets translate DocBook role attributes to HTML class attributes; CSS can then be used to style the generated HTML.

reftext

docbook

reftext is used to set the DocBook xreflabel attribute. The reftext attribute can an also be set by the BlockId element.

15. Paragraphs

Paragraphs are blocks of text terminated by a blank line, the end of file, or the start of a delimited block or a list. There are three paragraph syntaxes: normal, indented (literal) and admonition which are rendered, by default, with the corresponding paragraph style.

Each syntax has a default style, but you can explicitly apply any paragraph style to any paragraph syntax. You can also apply delimited block styles to single paragraphs.

The built-in paragraph styles are: normal, literal, verse, quote, listing, TIP, NOTE, IMPORTANT, WARNING, CAUTION, abstract, partintro, comment, example, sidebar, source, music, latex, graphviz.

15.1. normal paragraph syntax

Normal paragraph syntax consists of one or more non-blank lines of text. The first line must start hard against the left margin (no intervening white space). The default processing expectation is that of a normal paragraph of text.

15.2. literal paragraph syntax

Literal paragraphs are rendered verbatim in a monospaced font without any distinguishing background or border. By default there is no text formatting or substitutions within Literal paragraphs apart from Special Characters and Callouts.

The literal style is applied implicitly to indented paragraphs i.e. where the first line of the paragraph is indented by one or more space or tab characters. For example:

  Consul *necessitatibus* per id,
  consetetur, eu pro everti postulant
  homero verear ea mea, qui.

Renders:

Consul *necessitatibus* per id,
consetetur, eu pro everti postulant
homero verear ea mea, qui.
Note Because lists can be indented it’s possible for your indented paragraph to be misinterpreted as a list — in situations like this apply the literal style to a normal paragraph.

Instead of using a paragraph indent you could apply the literal style explicitly, for example:

[literal]
Consul *necessitatibus* per id,
consetetur, eu pro everti postulant
homero verear ea mea, qui.

Renders:

Consul *necessitatibus* per id,
consetetur, eu pro everti postulant
homero verear ea mea, qui.

15.3. quote and verse paragraph styles

The optional attribution and citetitle attributes (positional attributes 2 and 3) specify the author and source respectively.

The verse style retains the line breaks, for example:

[verse, William Blake, from Auguries of Innocence]
To see a world in a grain of sand,
And a heaven in a wild flower,
Hold infinity in the palm of your hand,
And eternity in an hour.

Which is rendered as:

To see a world in a grain of sand,
And a heaven in a wild flower,
Hold infinity in the palm of your hand,
And eternity in an hour.
from Auguries of Innocence
— William Blake

The quote style flows the text at left and right margins, for example:

[quote, Bertrand Russell, The World of Mathematics (1956)]
A good notation has subtlety and suggestiveness which at times makes
it almost seem like a live teacher.

Which is rendered as:

A good notation has subtlety and suggestiveness which at times makes it almost seem like a live teacher.
The World of Mathematics (1956)
— Bertrand Russell

15.4. Admonition Paragraphs

TIP, NOTE, IMPORTANT, WARNING and CAUTION admonishment paragraph styles are generated by placing NOTE:, TIP:, IMPORTANT:, WARNING: or CAUTION: as the first word of the paragraph. For example:

NOTE: This is an example note.

Alternatively, you can specify the paragraph admonition style explicitly using an AttributeList element. For example:

[NOTE]
This is an example note.

Renders:

Note This is an example note.
Tip If your admonition requires more than a single paragraph use an admonition block instead.

15.4.1. Admonition Icons and Captions

Note Admonition customization with icons, iconsdir, icon and caption attributes does not apply when generating DocBook output. If you are going the DocBook route then the a2x(1) --no-icons and --icons-dir options can be used to set the appropriate XSL Stylesheets parameters.

By default the asciidoc(1) HTML backends generate text captions instead of admonition icon image links. To generate links to icon images define the icons attribute, for example using the -a icons command-line option.

The iconsdir attribute sets the location of linked icon images.

You can override the default icon image using the icon attribute to specify the path of the linked image. For example:

[icon="./images/icons/wink.png"]
NOTE: What lovely war.

Use the caption attribute to customize the admonition captions (not applicable to docbook backend). The following example suppresses the icon image and customizes the caption of a NOTE admonition (undefining the icons attribute with icons=None is only necessary if admonition icons have been enabled):

[icons=None, caption="My Special Note"]
NOTE: This is my special note.

This subsection also applies to Admonition Blocks.

16. Delimited Blocks

Delimited blocks are blocks of text enveloped by leading and trailing delimiter lines (normally a series of four or more repeated characters). The behavior of Delimited Blocks is specified by entries in configuration file [blockdef-*] sections.

16.1. Predefined Delimited Blocks

AsciiDoc ships with a number of predefined DelimitedBlocks (see the asciidoc.conf configuration file in the asciidoc(1) program directory):

Predefined delimited block underlines:

CommentBlock:     //////////////////////////
PassthroughBlock: ++++++++++++++++++++++++++
ListingBlock:     --------------------------
LiteralBlock:     ..........................
SidebarBlock:     **************************
QuoteBlock:       __________________________
ExampleBlock:     ==========================
OpenBlock:        --
Table 3. Default DelimitedBlock substitutions
Attributes Callouts Macros Quotes Replacements Special chars Special words

PassthroughBlock

Yes

No

Yes

No

No

No

No

ListingBlock

No

Yes

No

No

No

Yes

No

LiteralBlock

No

Yes

No

No

No

Yes

No

SidebarBlock

Yes

No

Yes

Yes

Yes

Yes

Yes

QuoteBlock

Yes

No

Yes

Yes

Yes

Yes

Yes

ExampleBlock

Yes

No

Yes

Yes

Yes

Yes

Yes

OpenBlock

Yes

No

Yes

Yes

Yes

Yes

Yes

16.2. Listing Blocks

ListingBlocks are rendered verbatim in a monospaced font, they retain line and whitespace formatting and are often distinguished by a background or border. There is no text formatting or substitutions within Listing blocks apart from Special Characters and Callouts. Listing blocks are often used for computer output and file listings.

Here’s an example:

--------------------------------------
#include <stdio.h>

int main() {
   printf("Hello World!\n");
   exit(0);
}
--------------------------------------

Which will be rendered like:

#include <stdio.h>

int main() {
    printf("Hello World!\n");
    exit(0);
}

By convention filter blocks use the listing block syntax and are implemented as distinct listing block styles.

16.3. Literal Blocks

LiteralBlocks are rendered just like literal paragraphs. Example:

...................................
Consul *necessitatibus* per id,
consetetur, eu pro everti postulant
homero verear ea mea, qui.
...................................

Renders:

Consul *necessitatibus* per id,
consetetur, eu pro everti postulant
homero verear ea mea, qui.

If the listing style is applied to a LiteralBlock it will be rendered as a ListingBlock (this is handy if you have a listing containing a ListingBlock).

16.4. Sidebar Blocks

A sidebar is a short piece of text presented outside the narrative flow of the main text. The sidebar is normally presented inside a bordered box to set it apart from the main text.

The sidebar body is treated like a normal section body.

Here’s an example:

.An Example Sidebar
************************************************
Any AsciiDoc SectionBody element (apart from
SidebarBlocks) can be placed inside a sidebar.
************************************************

Which will be rendered like:

An Example Sidebar

Any AsciiDoc SectionBody element (apart from SidebarBlocks) can be placed inside a sidebar.

16.5. Comment Blocks

The contents of CommentBlocks are not processed; they are useful for annotations and for excluding new or outdated content that you don’t want displayed. CommentBlocks are never written to output files. Example:

//////////////////////////////////////////
CommentBlock contents are not processed by
asciidoc(1).
//////////////////////////////////////////

See also Comment Lines.

Note System macros are executed inside comment blocks.

16.6. Passthrough Blocks

By default the block contents is subject only to attributes and macros substitutions (use an explicit subs attribute to apply different substitutions). PassthroughBlock content will often be backend specific. Here’s an example:

[subs="quotes"]
++++++++++++++++++++++++++++++++++++++
<table border="1"><tr>
  <td>*Cell 1*</td>
  <td>*Cell 2*</td>
</tr></table>
++++++++++++++++++++++++++++++++++++++

The following styles can be applied to passthrough blocks:

pass

No substitutions are performed. This is equivalent to subs="none".

asciimath, latexmath

By default no substitutions are performed, the contents are rendered as mathematical formulas.

16.7. Quote Blocks

QuoteBlocks are used for quoted passages of text. There are two styles: quote and verse. The style behavior is identical to quote and verse paragraphs except that blocks can contain multiple paragraphs and, in the case of the quote style, other section elements. The first positional attribute sets the style, if no attributes are specified the quote style is used. The optional attribution and citetitle attributes (positional attributes 2 and 3) specify the quote’s author and source. For example:

[quote, Sir Arthur Conan Doyle, The Adventures of Sherlock Holmes]
____________________________________________________________________
As he spoke there was the sharp sound of horses' hoofs and
grating wheels against the curb, followed by a sharp pull at the
bell. Holmes whistled.

"A pair, by the sound," said he. "Yes," he continued, glancing
out of the window. "A nice little brougham and a pair of
beauties. A hundred and fifty guineas apiece. There's money in
this case, Watson, if there is nothing else."
____________________________________________________________________

Which is rendered as:

As he spoke there was the sharp sound of horses' hoofs and grating wheels against the curb, followed by a sharp pull at the bell. Holmes whistled.

"A pair, by the sound," said he. "Yes," he continued, glancing out of the window. "A nice little brougham and a pair of beauties. A hundred and fifty guineas apiece. There’s money in this case, Watson, if there is nothing else."

The Adventures of Sherlock Holmes
— Sir Arthur Conan Doyle

16.8. Example Blocks

ExampleBlocks encapsulate the DocBook Example element and are used for, well, examples. Example blocks can be titled by preceding them with a BlockTitle. DocBook toolchains will normally automatically number examples and generate a List of Examples backmatter section.

Example blocks are delimited by lines of equals characters and can contain any block elements apart from Titles, BlockTitles and Sidebars) inside an example block. For example:

.An example
=====================================================================
Qui in magna commodo, est labitur dolorum an. Est ne magna primis
adolescens.
=====================================================================

Renders:

Example 1. An example

Qui in magna commodo, est labitur dolorum an. Est ne magna primis adolescens.

A title prefix that can be inserted with the caption attribute (HTML backends). For example:

[caption="Example 1: "]
.An example with a custom caption
=====================================================================
Qui in magna commodo, est labitur dolorum an. Est ne magna primis
adolescens.
=====================================================================

16.9. Admonition Blocks

The ExampleBlock definition includes a set of admonition styles (NOTE, TIP, IMPORTANT, WARNING, CAUTION) for generating admonition blocks (admonitions containing more than a single paragraph). Just precede the ExampleBlock with an attribute list specifying the admonition style name. For example:

[NOTE]
.A NOTE admonition block
=====================================================================
Qui in magna commodo, est labitur dolorum an. Est ne magna primis
adolescens.

. Fusce euismod commodo velit.
. Vivamus fringilla mi eu lacus.
  .. Fusce euismod commodo velit.
  .. Vivamus fringilla mi eu lacus.
. Donec eget arcu bibendum
  nunc consequat lobortis.
=====================================================================

Renders:

Note
A NOTE admonition block

Qui in magna commodo, est labitur dolorum an. Est ne magna primis adolescens.

  1. Fusce euismod commodo velit.

  2. Vivamus fringilla mi eu lacus.

    1. Fusce euismod commodo velit.

    2. Vivamus fringilla mi eu lacus.

  3. Donec eget arcu bibendum nunc consequat lobortis.

See also Admonition Icons and Captions.

16.10. Open Blocks

Open blocks are special:

  • The open block delimiter is line containing two hyphen characters (instead of four or more repeated characters).

  • They can be used to group block elements for List item continuation.

  • Open blocks can be styled to behave like any other type of delimited block. The following built-in styles can be applied to open blocks: literal, verse, quote, listing, TIP, NOTE, IMPORTANT, WARNING, CAUTION, abstract, partintro, comment, example, sidebar, source, music, latex, graphviz. For example, the following open block and listing block are functionally identical: 

    [listing]
--
Lorum ipsum ...
--
    ---------------
Lorum ipsum ...
---------------

  • An unstyled open block groups section elements but otherwise does nothing.

Open blocks are used to generate document abstracts and book part introductions:

  • Apply the abstract style to generate an abstract, for example: 

    [abstract]
--
In this paper we will ...
--

    1. Apply the partintro style to generate a book part introduction for a multi-part book, for example: 

      [partintro]
.Optional part introduction title
--
Optional part introduction goes here.
--

17. Lists

List types

  • Bulleted lists. Also known as itemized or unordered lists.

  • Numbered lists. Also called ordered lists.

  • Labeled lists. Sometimes called variable or definition lists.

  • Callout lists (a list of callout annotations).

List behavior

  • List item indentation is optional and does not determine nesting, indentation does however make the source more readable.

  • Another list or a literal paragraph immediately following a list item will be implicitly included in the list item; use list item continuation to explicitly append other block elements to a list item.

  • A comment block or a comment line block macro element will terminate a list — use inline comment lines to put comments inside lists.

  • The listindex intrinsic attribute is the current list item index (1..). If this attribute is used outside a list then it’s value is the number of items in the most recently closed list. Useful for displaying the number of items in a list.

17.1. Bulleted Lists

Bulleted list items start with a single dash or one to five asterisks followed by some white space then some text. Bulleted list syntaxes are:

- List item.
* List item.
** List item.
*** List item.
**** List item.
***** List item.

17.2. Numbered Lists

List item numbers are explicit or implicit.

Explicit numbering

List items begin with a number followed by some white space then the item text. The numbers can be decimal (arabic), roman (upper or lower case) or alpha (upper or lower case). Decimal and alpha numbers are terminated with a period, roman numbers are terminated with a closing parenthesis. The different terminators are necessary to ensure i, v and x roman numbers are are distinguishable from x, v and x alpha numbers. Examples:

1.   Arabic (decimal) numbered list item.
a.   Lower case alpha (letter) numbered list item.
F.   Upper case alpha (letter) numbered list item.
iii) Lower case roman numbered list item.
IX)  Upper case roman numbered list item.
Implicit numbering

List items begin one to five period characters, followed by some white space then the item text. Examples:

. Arabic (decimal) numbered list item.
.. Lower case alpha (letter) numbered list item.
... Lower case roman numbered list item.
.... Upper case alpha (letter) numbered list item.
..... Upper case roman numbered list item.

You can use the style attribute (also the first positional attribute) to specify an alternative numbering style. The numbered list style can be one of the following values: arabic, loweralpha, upperalpha, lowerroman, upperroman.

Here are some examples of bulleted and numbered lists:

- Praesent eget purus quis magna eleifend eleifend.
  1. Fusce euismod commodo velit.
    a. Fusce euismod commodo velit.
    b. Vivamus fringilla mi eu lacus.
    c. Donec eget arcu bibendum nunc consequat lobortis.
  2. Vivamus fringilla mi eu lacus.
    i)  Fusce euismod commodo velit.
    ii) Vivamus fringilla mi eu lacus.
  3. Donec eget arcu bibendum nunc consequat lobortis.
  4. Nam fermentum mattis ante.
- Lorem ipsum dolor sit amet, consectetuer adipiscing elit.
  * Fusce euismod commodo velit.
  ** Qui in magna commodo, est labitur dolorum an. Est ne magna primis
     adolescens. Sit munere ponderum dignissim et. Minim luptatum et
     vel.
  ** Vivamus fringilla mi eu lacus.
  * Donec eget arcu bibendum nunc consequat lobortis.
- Nulla porttitor vulputate libero.
  . Fusce euismod commodo velit.
  . Vivamus fringilla mi eu lacus.
[upperroman]
    .. Fusce euismod commodo velit.
    .. Vivamus fringilla mi eu lacus.
  . Donec eget arcu bibendum nunc consequat lobortis.

Which render as:

  • Praesent eget purus quis magna eleifend eleifend.

    1. Fusce euismod commodo velit.

      1. Fusce euismod commodo velit.

      2. Vivamus fringilla mi eu lacus.

      3. Donec eget arcu bibendum nunc consequat lobortis.

    2. Vivamus fringilla mi eu lacus.

      1. Fusce euismod commodo velit.

      2. Vivamus fringilla mi eu lacus.

    3. Donec eget arcu bibendum nunc consequat lobortis.

    4. Nam fermentum mattis ante.

  • Lorem ipsum dolor sit amet, consectetuer adipiscing elit.

    • Fusce euismod commodo velit.

      • Qui in magna commodo, est labitur dolorum an. Est ne magna primis adolescens. Sit munere ponderum dignissim et. Minim luptatum et vel.

      • Vivamus fringilla mi eu lacus.

    • Donec eget arcu bibendum nunc consequat lobortis.

  • Nulla porttitor vulputate libero.

    1. Fusce euismod commodo velit.

    2. Vivamus fringilla mi eu lacus.

      1. Fusce euismod commodo velit.

      2. Vivamus fringilla mi eu lacus.

    3. Donec eget arcu bibendum nunc consequat lobortis.

A predefined compact option is available to bulleted and numbered lists — this translates to the DocBook spacing="compact" lists attribute which may or may not be processed by the DocBook toolchain. Example:

[options="compact"]
- Compact list item.
- Another compact list item.
Tip To apply the compact option globally define a document-wide compact-option attribute, e.g. using the -a compact-option command-line option.

You can set the list start number using the start attribute (works for HTML outputs and DocBook outputs processed by DocBook XSL Stylesheets). Example:

[start=7]
. List item 7.
. List item 8.

17.3. Labeled Lists

Labeled list items consist of one or more text labels followed by the text of the list item.

An item label begins a line with an alphanumeric character hard against the left margin and ends with two, three or four colons or two semi-colons. A list item can have multiple labels, one per line.

The list item text consists of one or more lines of text starting after the last label (either on the same line or a new line) and can be followed by nested List or ListParagraph elements. Item text can be optionally indented.

Here are some examples:

In::
Lorem::
  Fusce euismod commodo velit.

  Fusce euismod commodo velit.

Ipsum:: Vivamus fringilla mi eu lacus.
  * Vivamus fringilla mi eu lacus.
  * Donec eget arcu bibendum nunc consequat lobortis.
Dolor::
  Donec eget arcu bibendum nunc consequat lobortis.
  Suspendisse;;
    A massa id sem aliquam auctor.
  Morbi;;
    Pretium nulla vel lorem.
  In;;
    Dictum mauris in urna.
    Vivamus::: Fringilla mi eu lacus.
    Donec:::   Eget arcu bibendum nunc consequat lobortis.

Which render as:

In
Lorem

Fusce euismod commodo velit. 

Fusce euismod commodo velit.
Ipsum

Vivamus fringilla mi eu lacus.

  • Vivamus fringilla mi eu lacus.

  • Donec eget arcu bibendum nunc consequat lobortis.

Dolor

Donec eget arcu bibendum nunc consequat lobortis.

Suspendisse

A massa id sem aliquam auctor.

Morbi

Pretium nulla vel lorem.

In

Dictum mauris in urna.

Vivamus

Fringilla mi eu lacus.

Donec

Eget arcu bibendum nunc consequat lobortis.

17.3.1. Horizontal labeled list style

The horizontal labeled list style (also the first positional attribute) places the list text side-by-side with the label instead of under the label. Here is an example:

[horizontal]
*Lorem*:: Fusce euismod commodo velit.  Qui in magna commodo, est
labitur dolorum an. Est ne magna primis adolescens.

  Fusce euismod commodo velit.

*Ipsum*:: Vivamus fringilla mi eu lacus.
- Vivamus fringilla mi eu lacus.
- Donec eget arcu bibendum nunc consequat lobortis.

*Dolor*::
  - Vivamus fringilla mi eu lacus.
  - Donec eget arcu bibendum nunc consequat lobortis.

Which render as:

Lorem

Fusce euismod commodo velit. Qui in magna commodo, est labitur dolorum an. Est ne magna primis adolescens. 

Fusce euismod commodo velit.
Ipsum

Vivamus fringilla mi eu lacus.

  • Vivamus fringilla mi eu lacus.

  • Donec eget arcu bibendum nunc consequat lobortis.

Dolor

  • Vivamus fringilla mi eu lacus.

  • Donec eget arcu bibendum nunc consequat lobortis.

Note

  • Current PDF toolchains do not make a good job of determining the relative column widths for horizontal labeled lists.

  • Nested horizontal labeled lists will generate DocBook validation errors because the DocBook XML V4.2 DTD does not permit nested informal tables (although DocBook XSL Stylesheets and dblatex process them correctly).

  • The label width can be set as a percentage of the total width by setting the width attribute e.g. width="10%"

17.4. Question and Answer Lists

AsciiDoc comes pre-configured with a qanda style labeled list for generating DocBook question and answer (Q&A) lists. Example:

[qanda]
Question one::
        Answer one.
Question two::
        Answer two.

Renders:

  1. Question one

    Answer one.

  2. Question two

    Answer two.

17.5. Glossary Lists

AsciiDoc comes pre-configured with a glossary style labeled list for generating DocBook glossary lists. Example:

[glossary]
A glossary term::
    The corresponding definition.
A second glossary term::
    The corresponding definition.

For working examples see the article.txt and book.txt documents in the AsciiDoc ./doc distribution directory.

Note To generate valid DocBook output glossary lists must be located in a section that uses the glossary section markup template.

17.6. Bibliography Lists

AsciiDoc comes with a predefined bibliography bulleted list style generating DocBook bibliography entries. Example:

[bibliography]
.Optional list title
- [[[taoup]]] Eric Steven Raymond. 'The Art of UNIX
  Programming'. Addison-Wesley. ISBN 0-13-142901-9.
- [[[walsh-muellner]]] Norman Walsh & Leonard Muellner.
  'DocBook - The Definitive Guide'. O'Reilly & Associates.
  1999. ISBN 1-56592-580-7.

The [[[<reference>]]] syntax is a bibliography entry anchor, it generates an anchor named <reference> and additionally displays [<reference>] at the anchor position. For example [[[taoup]]] generates an anchor named taoup that displays [taoup] at the anchor position. Cite the reference from elsewhere your document using <<taoup>>, this displays a hyperlink ([taoup]) to the corresponding bibliography entry anchor.

For working examples see the article.txt and book.txt documents in the AsciiDoc ./doc distribution directory.

Note To generate valid DocBook output bibliography lists must be located in a bibliography section.

17.7. List Item Continuation

Another list or a literal paragraph immediately following a list item is implicitly appended to the list item; to append other block elements to a list item you need to explicitly join them to the list item with a list continuation (a separator line containing a single plus character). Multiple block elements can be appended to a list item using list continuations (provided they are legal list item children in the backend markup).

Here are some examples of list item continuations: list item one contains multiple continuations; list item two is continued with an OpenBlock containing multiple elements:

1. List item one.
+
List item one continued with a second paragraph followed by an
Indented block.
+
.................
$ ls *.sh
$ mv *.sh ~/tmp
.................
+
List item continued with a third paragraph.

2. List item two continued with an open block.
+
--
This paragraph is part of the preceding list item.

a. This list is nested and does not require explicit item continuation.
+
This paragraph is part of the preceding list item.

b. List item b.

This paragraph belongs to item two of the outer list.
--

Renders:

  1. List item one.

    List item one continued with a second paragraph followed by an Indented block.

    $ ls *.sh
$ mv *.sh ~/tmp

    List item continued with a third paragraph.

  2. List item two continued with an open block.

    This paragraph is part of the preceding list item.

    1. This list is nested and does not require explicit item continuation.

      This paragraph is part of the preceding list item.

    2. List item b.

    This paragraph belongs to item two of the outer list.

18. Footnotes

The shipped AsciiDoc configuration includes three footnote inline macros:

footnote:[<text>]

Generates a footnote with text <text>.

footnoteref:[<id>,<text>]

Generates a footnote with a reference ID <id> and text <text>.

footnoteref:[<id>]

Generates a reference to the footnote with ID <id>.

The footnote text can span multiple lines.

The xhtml11 and html5 backends render footnotes dynamically using JavaScript; html4 outputs do not use JavaScript and leave the footnotes inline; docbook footnotes are processed by the downstream DocBook toolchain.

Example footnotes:

A footnote footnote:[An example footnote.];
a second footnote with a reference ID footnoteref:[note2,Second footnote.];
finally a reference to the second footnote footnoteref:[note2].

Renders:

A footnote
[An example footnote.]
; a second footnote with a reference ID
[Second footnote.]
; finally a reference to the second footnote
[note2]
.

19. Indexes

The shipped AsciiDoc configuration includes the inline macros for generating DocBook index entries.

indexterm:[<primary>,<secondary>,<tertiary>]
(((<primary>,<secondary>,<tertiary>)))

This inline macro generates an index term (the <secondary> and <tertiary> positional attributes are optional). Example: indexterm:[Tigers,Big cats] (or, using the alternative syntax (((Tigers,Big cats))). Index terms that have secondary and tertiary entries also generate separate index terms for the secondary and tertiary entries. The index terms appear in the index, not the primary text flow.

indexterm2:[<primary>]
((<primary>))

This inline macro generates an index term that appears in both the index and the primary text flow. The <primary> should not be padded to the left or right with white space characters.

For working examples see the article.txt and book.txt documents in the AsciiDoc ./doc distribution directory.

Note Index entries only really make sense if you are generating DocBook markup — DocBook conversion programs automatically generate an index at the point an Index section appears in source document.

20. Callouts

Callouts are a mechanism for annotating verbatim text (for example: source code, computer output and user input). Callout markers are placed inside the annotated text while the actual annotations are presented in a callout list after the annotated text. Here’s an example:

 .MS-DOS directory listing
 -----------------------------------------------------
 10/17/97   9:04         <DIR>    bin
 10/16/97  14:11         <DIR>    DOS            <1>
 10/16/97  14:40         <DIR>    Program Files
 10/16/97  14:46         <DIR>    TEMP
 10/17/97   9:04         <DIR>    tmp
 10/16/97  14:37         <DIR>    WINNT
 10/16/97  14:25             119  AUTOEXEC.BAT   <2>
  2/13/94   6:21          54,619  COMMAND.COM    <2>
 10/16/97  14:25             115  CONFIG.SYS     <2>
 11/16/97  17:17      61,865,984  pagefile.sys
  2/13/94   6:21           9,349  WINA20.386     <3>
 -----------------------------------------------------

 <1> This directory holds MS-DOS.
 <2> System startup code for DOS.
 <3> Some sort of Windows 3.1 hack.

Which renders:

MS-DOS directory listing
10/17/97   9:04         <DIR>    bin
10/16/97  14:11         <DIR>    DOS            1
10/16/97  14:40         <DIR>    Program Files
10/16/97  14:46         <DIR>    TEMP
10/17/97   9:04         <DIR>    tmp
10/16/97  14:37         <DIR>    WINNT
10/16/97  14:25             119  AUTOEXEC.BAT   2
 2/13/94   6:21          54,619  COMMAND.COM    2
10/16/97  14:25             115  CONFIG.SYS     2
11/16/97  17:17      61,865,984  pagefile.sys
 2/13/94   6:21           9,349  WINA20.386     3
1 This directory holds MS-DOS.
2 System startup code for DOS.
3 Some sort of Windows 3.1 hack.
Explanation

  • The callout marks are whole numbers enclosed in angle brackets —  they refer to the correspondingly numbered item in the following callout list.

  • By default callout marks are confined to LiteralParagraphs, LiteralBlocks and ListingBlocks (although this is a configuration file option and can be changed).

  • Callout list item numbering is fairly relaxed — list items can start with <n>, n> or > where n is the optional list item number (in the latter case list items starting with a single > character are implicitly numbered starting at one).

  • Callout lists should not be nested.

  • Callout lists start list items hard against the left margin.

  • If you want to present a number inside angle brackets you’ll need to escape it with a backslash to prevent it being interpreted as a callout mark.

Note Define the AsciiDoc icons attribute (for example using the -a icons command-line option) to display callout icons.

20.1. Implementation Notes

Callout marks are generated by the callout inline macro while callout lists are generated using the callout list definition. The callout macro and callout list are special in that they work together. The callout inline macro is not enabled by the normal macros substitutions option, instead it has its own callouts substitution option.

The following attributes are available during inline callout macro substitution:

{index}

The callout list item index inside the angle brackets.

{coid}

An identifier formatted like CO<listnumber>-<index> that uniquely identifies the callout mark. For example CO2-4 identifies the fourth callout mark in the second set of callout marks.

The {coids} attribute can be used during callout list item substitution — it is a space delimited list of callout IDs that refer to the explanatory list item.

20.2. Including callouts in included code

You can annotate working code examples with callouts — just remember to put the callouts inside source code comments. This example displays the test.py source file (containing a single callout) using the source (code highlighter) filter:

AsciiDoc source
 [source,python]
 -------------------------------------------
 \include::test.py[]
 -------------------------------------------

 <1> Print statement.
Included test.py source
print 'Hello World!'   # <1>

21. Macros

Macros are a mechanism for substituting parametrized text into output documents.

Macros have a name, a single target argument and an attribute list. The usual syntax is <name>:<target>[<attrlist>] (for inline macros) and <name>::<target>[<attrlist>] (for block macros). Here are some examples:

http://www.docbook.org/[DocBook.org]
include::chapt1.txt[tabsize=2]
mailto:srackham@gmail.com[]
Macro behavior

  • <name> is the macro name. It can only contain letters, digits or dash characters and cannot start with a dash.

  • The optional <target> cannot contain white space characters.

  • <attrlist> is a list of attributes enclosed in square brackets.

  • ] characters inside attribute lists must be escaped with a backslash.

  • Expansion of macro references can normally be escaped by prefixing a backslash character (see the AsciiDoc FAQ for examples of exceptions to this rule).

  • Attribute references in block macros are expanded.

  • The substitutions performed prior to Inline macro macro expansion are determined by the inline context.

  • Macros are processed in the order they appear in the configuration file(s).

  • Calls to inline macros can be nested inside different inline macros (an inline macro call cannot contain a nested call to itself).

  • In addition to <name>, <target> and <attrlist> the <passtext> and <subslist> named groups are available to passthrough macros. A macro is a passthrough macro if the definition includes a <passtext> named group.

21.1. Inline Macros

Inline Macros occur in an inline element context. Predefined Inline macros include URLs, image and link macros.

21.1.1. URLs

http, https, ftp, file, mailto and callto URLs are rendered using predefined inline macros.

  • If you don’t need a custom link caption you can enter the http, https, ftp, file URLs and email addresses without any special macro syntax.

  • If the <attrlist> is empty the URL is displayed.

Here are some examples:

http://www.docbook.org/[DocBook.org]
http://www.docbook.org/
mailto:joe.bloggs@foobar.com[email Joe Bloggs]
joe.bloggs@foobar.com

Which are rendered:

DocBook.org

http://www.docbook.org/

email Joe Bloggs

joe.bloggs@foobar.com

If the <target> necessitates space characters use %20, for example large%20image.png.

21.1.2. Internal Cross References

Two AsciiDoc inline macros are provided for creating hypertext links within an AsciiDoc document. You can use either the standard macro syntax or the (preferred) alternative.

anchor

Used to specify hypertext link targets:

[[<id>,<xreflabel>]]
anchor:<id>[<xreflabel>]

The <id> is a unique string that conforms to the output markup’s anchor syntax. The optional <xreflabel> is the text to be displayed by captionless xref macros that refer to this anchor. The optional <xreflabel> is only really useful when generating DocBook output. Example anchor:

[[X1]]

You may have noticed that the syntax of this inline element is the same as that of the BlockId block element, this is no coincidence since they are functionally equivalent.

xref

Creates a hypertext link to a document anchor.

<<<id>,<caption>>>
xref:<id>[<caption>]

The <id> refers to an anchor ID. The optional <caption> is the link’s displayed text. Example:

<<X21,attribute lists>>

If <caption> is not specified then the displayed text is auto-generated:

  • The AsciiDoc xhtml11 and html5 backends display the <id> enclosed in square brackets.

  • If DocBook is produced the DocBook toolchain is responsible for the displayed text which will normally be the referenced figure, table or section title number followed by the element’s title text.

Here is an example:

[[tiger_image]]
.Tyger tyger
image::tiger.png[]

This can be seen in <<tiger_image>>.

21.1.3. Linking to Local Documents

Hypertext links to files on the local file system are specified using the link inline macro.

link:<target>[<caption>]

The link macro generates relative URLs. The link macro <target> is the target file name (relative to the file system location of the referring document). The optional <caption> is the link’s displayed text. If <caption> is not specified then <target> is displayed. Example:

link:downloads/foo.zip[download foo.zip]

You can use the <filename>#<id> syntax to refer to an anchor within a target document but this usually only makes sense when targeting HTML documents.

21.1.4. Images

Inline images are inserted into the output document using the image macro. The inline syntax is:

image:<target>[<attributes>]

The contents of the image file <target> is displayed. To display the image its file format must be supported by the target backend application. HTML and DocBook applications normally support PNG or JPG files.

<target> file name paths are relative to the location of the referring document.

Image macro attributes

  • The optional alt attribute is also the first positional attribute, it specifies alternative text which is displayed if the output application is unable to display the image file (see also Use of ALT texts in IMGs). For example: 

    image:images/logo.png[Company Logo]

  • The optional title attribute provides a title for the image. The block image macro renders the title alongside the image. The inline image macro displays the title as a popup “tooltip” in visual browsers (AsciiDoc HTML outputs only).

  • The optional width and height attributes scale the image size and can be used in any combination. The units are pixels. The following example scales the previous example to a height of 32 pixels: 

    image:images/logo.png["Company Logo",height=32]

  • The optional link attribute is used to link the image to an external document. The following example links a screenshot thumbnail to a full size version: 

    image:screen-thumbnail.png[height=32,link="screen.png"]

  • The optional scaledwidth attribute is only used in DocBook block images (specifically for PDF documents). The following example scales the images to 75% of the available print width: 

    image::images/logo.png[scaledwidth="75%",alt="Company Logo"]

  • The image scale attribute sets the DocBook imagedata element scale attribute.

  • The optional align attribute is used for horizontal image alignment. Allowed values are center, left and right. For example: 

    image::images/tiger.png["Tiger image",align="left"]

  • The optional float attribute floats the image left or right on the page (works with HTML outputs only, has no effect on DocBook outputs). float and align attributes are mutually exclusive. Use the unfloat::[] block macro to stop floating.

21.1.5. Comment Lines

See comment block macro.

21.2. Block Macros

A Block macro reference must be contained in a single line separated either side by a blank line or a block delimiter.

Block macros behave just like Inline macros, with the following differences:

  • They occur in a block context.

  • The default syntax is <name>::<target>[<attrlist>] (two colons, not one).

  • Markup template section names end in -blockmacro instead of -inlinemacro.

21.2.1. Block Identifier

The Block Identifier macro sets the id attribute and has the same syntax as the anchor inline macro since it performs essentially the same function — block templates use the id attribute as a block element ID. For example:

[[X30]]

This is equivalent to the [id="X30"] AttributeList element).

21.2.2. Images

The image block macro is used to display images in a block context. The syntax is:

image::<target>[<attributes>]

The block image macro has the same macro attributes as it’s inline image macro counterpart.

Block images can be titled by preceding the image macro with a BlockTitle. DocBook toolchains normally number titled block images and optionally list them in an automatically generated List of Figures backmatter section.

This example:

.Main circuit board
image::images/layout.png[J14P main circuit board]

is equivalent to:

image::images/layout.png["J14P main circuit board",
                          title="Main circuit board"]

A title prefix that can be inserted with the caption attribute (HTML backends). For example:

.Main circuit board
[caption="Figure 2: "]
image::images/layout.png[J14P main circuit board]
Embedding images in XHTML documents

If you define the data-uri attribute then images will be embedded in XHTML outputs using the data URI scheme. You can use the data-uri attribute with the xhtml11 and html5 backends to produce single-file XHTML documents with embedded images and CSS, for example:

$ asciidoc -a data-uri mydocument.txt
Note

  • All current popular browsers support data URIs, although versions of Internet Explorer prior to version 8 do not.

  • Some browsers limit the size of data URIs.

21.2.3. Comment Lines

Single lines starting with two forward slashes hard up against the left margin are treated as comments. Comment lines do not appear in the output unless the showcomments attribute is defined. Comment lines have been implemented as both block and inline macros so a comment line can appear as a stand-alone block or within block elements that support inline macro expansion. Example comment line:

// This is a comment.

If the showcomments attribute is defined comment lines are written to the output:

  • In DocBook the comment lines are enclosed by the remark element (which may or may not be rendered by your toolchain).

  • The showcomments attribute does not expose Comment Blocks. Comment Blocks are never passed to the output.

21.3. System Macros

System macros are block macros that perform a predefined task and are hardwired into the asciidoc(1) program.

  • You can escape system macros with a leading backslash character (as you can with other macros).

  • The syntax and tasks performed by system macros is built into asciidoc(1) so they don’t appear in configuration files. You can however customize the syntax by adding entries to a configuration file [macros] section.

21.3.1. Include Macros

The include and include1 system macros to include the contents of a named file into the source document.

The include macro includes a file as if it were part of the parent document — tabs are expanded and system macros processed. The contents of include1 files are not subject to tab expansion or system macro processing nor are attribute or lower priority substitutions performed. The include1 macro’s intended use is to include verbatim embedded CSS or scripts into configuration file headers. Example:

include::chapter1.txt[tabsize=4]
Include macro behavior

  • If the included file name is specified with a relative path then the path is relative to the location of the referring document.

  • Include macros can appear inside configuration files.

  • Files included from within DelimitedBlocks are read to completion to avoid false end-of-block underline termination.

  • Attribute references are expanded inside the include target; if an attribute is undefined then the included file is silently skipped.

  • The tabsize macro attribute sets the number of space characters to be used for tab expansion in the included file (not applicable to include1 macro).

  • The depth macro attribute sets the maximum permitted number of subsequent nested includes (not applicable to include1 macro which does not process nested includes). Setting depth to 1 disables nesting inside the included file. By default, nesting is limited to a depth of ten.

  • If the he warnings attribute is set to False (or any other Python literal that evaluates to boolean false) then no warning message is printed if the included file does not exist. By default warnings are enabled.

  • Internally the include1 macro is translated to the include1 system attribute which means it must be evaluated in a region where attribute substitution is enabled. To inhibit nested substitution in included files it is preferable to use the include macro and set the attribute depth=1.

21.3.2. Conditional Inclusion Macros

Lines of text in the source document can be selectively included or excluded from processing based on the existence (or not) of a document attribute.

Document text between the ifdef and endif macros is included if a document attribute is defined:

ifdef::<attribute>[]
:
endif::<attribute>[]

Document text between the ifndef and endif macros is not included if a document attribute is defined:

ifndef::<attribute>[]
:
endif::<attribute>[]

<attribute> is an attribute name which is optional in the trailing endif macro.

If you only want to process a single line of text then the text can be put inside the square brackets and the endif macro omitted, for example:

ifdef::revnumber[Version number 42]

Is equivalent to:

ifdef::revnumber[]
Version number 42
endif::revnumber[]

ifdef and ifndef macros also accept multiple attribute names:

  • Multiple , separated attribute names evaluate to defined if one or more of the attributes is defined, otherwise it’s value is undefined.

  • Multiple + separated attribute names evaluate to defined if all of the attributes is defined, otherwise it’s value is undefined.

Document text between the ifeval and endif macros is included if the Python expression inside the square brackets is true. Example:

ifeval::[{rs458}==2]
:
endif::[]

  • Document attribute references are expanded before the expression is evaluated.

  • If an attribute reference is undefined then the expression is considered false.

Take a look at the *.conf configuration files in the AsciiDoc distribution for examples of conditional inclusion macro usage.

21.3.3. Executable system macros

The eval, sys and sys2 block macros exhibit the same behavior as their same named system attribute references. The difference is that system macros occur in a block macro context whereas system attributes are confined to inline contexts where attribute substitution is enabled.

The following example displays a long directory listing inside a literal block:

------------------
sys::[ls -l *.txt]
------------------
Note There are no block macro versions of the eval3 and sys3 system attributes.

21.3.4. Template System Macro

The template block macro allows the inclusion of one configuration file template section within another. The following example includes the [admonitionblock] section in the [admonitionparagraph] section:

[admonitionparagraph]
template::[admonitionblock]
Template macro behavior

  • The template::[] macro is useful for factoring configuration file markup.

  • template::[] macros cannot be nested.

  • template::[] macro expansion is applied after all configuration files have been read.

21.4. Passthrough macros

Passthrough macros are analogous to passthrough blocks and are used to pass text directly to the output. The substitution performed on the text is determined by the macro definition but can be overridden by the <subslist>. The usual syntax is <name>:<subslist>[<passtext>] (for inline macros) and <name>::<subslist>[<passtext>] (for block macros). Passthroughs, by definition, take precedence over all other text substitutions.

pass

Inline and block. Passes text unmodified (apart from explicitly specified substitutions). Examples: 

pass:[<q>To be or not to be</q>]
pass:attributes,quotes[<u>the '{author}'</u>]
asciimath, latexmath

Inline and block. Passes text unmodified. Used for mathematical formulas.

+++

Inline and block. The triple-plus passthrough is functionally identical to the pass macro but you don’t have to escape ] characters and you can prefix with quoted attributes in the inline version. Example: 

Red [red]+++`sum_(i=1)\^n i=(n(n+1))/2`$+++ AsciiMathML formula
$$

Inline and block. The double-dollar passthrough is functionally identical to the triple-plus passthrough with one exception: special characters are escaped. Example: 

$$`[[a,b],[c,d]]((n),(k))`$$
`

Text quoted with single backtick characters constitutes an inline literal passthrough. The enclosed text is rendered in a monospaced font and is only subject to special character substitution. This makes sense since monospace text is usually intended to be rendered literally and often contains characters that would otherwise have to be escaped. If you need monospaced text containing inline substitutions use a plus character instead of a backtick.

21.5. Macro Definitions

Each entry in the configuration [macros] section is a macro definition which can take one of the following forms:

<pattern>=<name>[<subslist]

Inline macro definition.

<pattern>=#<name>[<subslist]

Block macro definition.

<pattern>=+<name>[<subslist]

System macro definition.

<pattern>

Delete the existing macro with this <pattern>.

<pattern> is a Python regular expression and <name> is the name of a markup template. If <name> is omitted then it is the value of the regular expression match group named name. The optional [<subslist] is a comma-separated list of substitution names enclosed in [] brackets, it sets the default substitutions for passthrough text, if omitted then no passthrough substitutions are performed.

Pattern named groups

The following named groups can be used in macro <pattern> regular expressions and are available as markup template attributes:

name

The macro name.

target

The macro target.

attrlist

The macro attribute list.

passtext

Contents of this group are passed unmodified to the output subject only to subslist substitutions.

subslist

Processed as a comma-separated list of substitution names for passtext substitution, overrides the the macro definition subslist.

Here’s what happens during macro substitution

  • Each contextually relevant macro pattern from the [macros] section is matched against the input source line.

  • If a match is found the text to be substituted is loaded from a configuration markup template section named like <name>-inlinemacro or <name>-blockmacro (depending on the macro type).

  • Global and macro attribute list attributes are substituted in the macro’s markup template.

  • The substituted template replaces the macro reference in the output document.

22. HTML 5 audio and video block macros

The html5 backend audio and video block macros generate the HTML 5 audio and video elements respectively. They follow the usual AsciiDoc block macro syntax <name>::<target>[<attrlist>] where:

<name>

audio or video.

<target>

The URL or file name of the video or audio file.

<attrlist>

A list of named attributes (see below).

Table 4. Audio macro attributes
Name Value

options

A comma separated list of one or more of the following items: autoplay, loop which correspond to the same-named HTML 5 audio element boolean attributes. By default the player controls are enabled, include the nocontrols option value to hide them.
Table 5. Video macro attributes
Name Value

height

The height of the player in pixels.

width

The width of the player in pixels.

poster

The URL or file name of an image representing the video.

options

A comma separated list of one or more of the following items: autoplay, loop and nocontrols. The autoplay and loop options correspond to the same-named HTML 5 video element boolean attributes. By default the player controls are enabled, include the nocontrols option value to hide them.

Examples:

audio::images/example.ogg[]

video::gizmo.ogv[width=200,options="nocontrols,autoplay"]

.Example video
video::gizmo.ogv[]

video::http://www.808.dk/pics/video/gizmo.ogv[]

If your needs are more complex put raw HTML 5 in a markup block, for example (from http://www.808.dk/?code-html-5-video):

++++
<video poster="pics/video/gizmo.jpg" id="video" style="cursor: pointer;" >
  <source src="pics/video/gizmo.mp4" />
  <source src="pics/video/gizmo.webm" type="video/webm" />
  <source src="pics/video/gizmo.ogv" type="video/ogg" />
  Video not playing? <a href="pics/video/gizmo.mp4">Download file</a> instead.
</video>

<script type="text/javascript">
  var video = document.getElementById('video');
  video.addEventListener('click',function(){
    video.play();
  },false);
</script>
++++

23. Tables

The AsciiDoc table syntax looks and behaves like other delimited block types and supports standard block configuration entries. Formatting is easy to read and, just as importantly, easy to enter.

  • Cells and columns can be formatted using built-in customizable styles.

  • Horizontal and vertical cell alignment can be set on columns and cell.

  • Horizontal and vertical cell spanning is supported.

Use tables sparingly

When technical users first start creating documents, tables (complete with column spanning and table nesting) are often considered very important. The reality is that tables are seldom used, even in technical documentation.

Try this exercise: thumb through your library of technical books, you’ll be surprised just how seldom tables are actually used, even less seldom are tables containing block elements (such as paragraphs or lists) or spanned cells. This is no accident, like figures, tables are outside the normal document flow — tables are for consulting not for reading.

Tables are designed for, and should normally only be used for, displaying column oriented tabular data.

23.1. Example tables

Table 6. Simple table

1

2

A

3

4

B

5

6

C
AsciiDoc source
[width="15%"]
|=======
|1 |2 |A
|3 |4 |B
|5 |6 |C
|=======
Table 7. Columns formatted with strong, monospaced and emphasis styles
Columns 2 and 3

footer 1

footer 2

footer 3

1

Item 1

Item 1

2

Item 2

Item 2

3

Item 3

Item 3

4

Item 4

Item 4
AsciiDoc source
.An example table
[width="50%",cols=">s,^m,e",frame="topbot",options="header,footer"]
|==========================
|      2+|Columns 2 and 3
|1       |Item 1  |Item 1
|2       |Item 2  |Item 2
|3       |Item 3  |Item 3
|4       |Item 4  |Item 4
|footer 1|footer 2|footer 3
|==========================
Table 8. Horizontal and vertical source data
Date Duration Avg HR Notes

22-Aug-08

10:24

157

Worked out MSHR (max sustainable heart rate) by going hard for this interval.

22-Aug-08

23:03

152

Back-to-back with previous interval.

24-Aug-08

40:00

145

Moderately hard interspersed with 3x 3min intervals (2min hard + 1min really hard taking the HR up to 160).

Short cells can be entered horizontally, longer cells vertically. The default behavior is to strip leading and trailing blank lines within a cell. These characteristics aid readability and data entry.

AsciiDoc source
.Windtrainer workouts
[width="80%",cols="3,^2,^2,10",options="header"]
|=========================================================
|Date |Duration |Avg HR |Notes

|22-Aug-08 |10:24 | 157 |
Worked out MSHR (max sustainable heart rate) by going hard
for this interval.

|22-Aug-08 |23:03 | 152 |
Back-to-back with previous interval.

|24-Aug-08 |40:00 | 145 |
Moderately hard interspersed with 3x 3min intervals (2min
hard + 1min really hard taking the HR up to 160).

|=========================================================
Table 9. A table with externally sourced CSV data
ID Customer Name Contact Name Customer Address Phone

AROUT

Around the Horn

Thomas Hardy

120 Hanover Sq. London

(171) 555-7788

BERGS

Berglunds snabbkop

Christina Berglund

Berguvsvagen 8 Lulea

0921-12 34 65

BLAUS

Blauer See Delikatessen

Hanna Moos

Forsterstr. 57 Mannheim

0621-08460

BLONP

Blondel pere et fils

Frederique Citeaux

24, place Kleber Strasbourg

88.60.15.31

BOLID

Bolido Comidas preparadas

Martin Sommer

C/ Araquil, 67 Madrid

(91) 555 22 82

BONAP

Bon app'

Laurence Lebihan

12, rue des Bouchers Marseille

91.24.45.40

BOTTM

Bottom-Dollar Markets

Elizabeth Lincoln

23 Tsawassen Blvd. Tsawassen

(604) 555-4729

BSBEV

B’s Beverages

Victoria Ashworth

Fauntleroy Circus London

(171) 555-1212

CACTU

Cactus Comidas para llevar

Patricio Simpson

Cerrito 333 Buenos Aires

(1) 135-5555
AsciiDoc source
[format="csv",cols="^1,4*2",options="header"]
|===================================================
ID,Customer Name,Contact Name,Customer Address,Phone
include::customers.csv[]
|===================================================
Table 10. Cell spans, alignments and styles

1

2

3

4

5

6

7

8

9

10
AsciiDoc source
[cols="e,m,^,>s",width="25%"]
|============================
|1 >s|2 |3 |4
^|5 2.2+^.^|6 .3+<.>m|7
^|8
|9 2+>|10
|============================

23.2. Table input data formats

AsciiDoc table data can be psv, dsv or csv formatted. The default table format is psv.

AsciiDoc psv (Prefix Separated Values) and dsv (Delimiter Separated Values) formats are cell oriented — the table is treated as a sequence of cells — there are no explicit row separators.

  • psv prefixes each cell with a separator whereas dsv delimits cells with a separator.

  • psv and dsv separators are Python regular expressions.

  • The default psv separator contains cell specifier related named regular expression groups.

  • The default dsv separator is :|\n (a colon or a new line character).

  • psv and dsv cell separators can be escaped by preceding them with a backslash character.

Here are four psv cells (the second item spans two columns; the last contains an escaped separator):

|One 2+|Two and three |A \| separator character

csv is the quasi-standard row oriented Comma Separated Values (CSV) format commonly used to import and export spreadsheet and database data.

23.3. Table attributes

Tables can be customized by the following attributes:

format

psv (default), dsv or csv (See Table Data Formats).

separator

The cell separator. A Python regular expression (psv and dsv formats) or a single character (csv format).

frame

Defines the table border and can take the following values: topbot (top and bottom), all (all sides), none and sides (left and right sides). The default value is all.

grid

Defines which ruler lines are drawn between table rows and columns. The grid attribute value can be any of the following values: none, cols, rows and all. The default value is all.

align

Use the align attribute to horizontally align the table on the page (works with HTML outputs only, has no effect on DocBook outputs). The following values are valid: left, right, and center.

float

Use the float attribute to float the table left or right on the page (works with HTML outputs only, has no effect on DocBook outputs). Floating only makes sense in conjunction with a table width attribute value of less than 100% (otherwise the table will take up all the available space). float and align attributes are mutually exclusive. Use the unfloat::[] block macro to stop floating.

halign

Use the halign attribute to horizontally align all cells in a table. The following values are valid: left, right, and center (defaults to left). Overridden by Column specifiers and Cell specifiers.

valign

Use the valign attribute to vertically align all cells in a table. The following values are valid: top, bottom, and middle (defaults to top). Overridden by Column specifiers and Cell specifiers.

options

The options attribute can contain comma separated values, for example: header, footer. By default header and footer rows are omitted. See attribute options for a complete list of available table options.

cols

The cols attribute is a comma separated list of column specifiers. For example cols="2<p,2*,4p,>".

  • If cols is present it must specify all columns.

  • If the cols attribute is not specified the number of columns is calculated as the number of data items in the first line of the table.

  • The degenerate form for the cols attribute is an integer specifying the number of columns e.g. cols=4.

width

The width attribute is expressed as a percentage value ("1%"…"99%"). The width specifies the table width relative to the available width. HTML backends use this value to set the table width attribute. It’s a bit more complicated with DocBook, see the DocBook table widths sidebar.

filter

The filter attribute defines an external shell command that is invoked for each cell. The built-in asciidoc table style is implemented using a filter.

DocBook table widths

The AsciiDoc docbook backend generates CALS tables. CALS tables do not support a table width attribute — table width can only be controlled by specifying absolute column widths.

Specifying absolute column widths is not media independent because different presentation media have different physical dimensions. To get round this limitation both DocBook XSL Stylesheets and dblatex have implemented table width processing instructions for setting the table width as a percentage of the available width. AsciiDoc emits these processing instructions if the width attribute is set along with proportional column widths (the AsciiDoc docbook backend pageunits attribute defaults to *).

To generate DocBook tables with absolute column widths set the pageunits attribute to a CALS absolute unit such as pt and set the pagewidth attribute to match the width of the presentation media.

23.4. Column Specifiers

Column specifiers define how columns are rendered and appear in the table cols attribute. A column specifier consists of an optional column multiplier followed by optional alignment, width and style values and is formatted like:

[<multiplier>*][<align>][<width>][<style>]

  • All components are optional. The multiplier must be first and the style last. The order of <align> or <width> is not important.

  • Column <width> can be either an integer proportional value (1…) or a percentage (1%…100%). The default value is 1. To ensure portability across different backends, there is no provision for absolute column widths (not to be confused with output column width markup attributes which are available in both percentage and absolute units).

  • The <align> column alignment specifier is formatted like: 

    [<horizontal>][.<vertical>]

    Where <horizontal> and <vertical> are one of the following characters: <, ^ or > which represent left, center and right horizontal alignment or top, middle and bottom vertical alignment respectively.

  • A <multiplier> can be used to specify repeated columns e.g. cols="4*<" specifies four left-justified columns. The default multiplier value is 1.

  • The <style> name specifies a table style to used to markup column cells (you can use the full style names if you wish but the first letter is normally sufficient).

  • Column specific styles are not applied to header rows.

23.5. Cell Specifiers

Cell specifiers allow individual cells in psv formatted tables to be spanned, multiplied, aligned and styled. Cell specifiers prefix psv | delimiters and are formatted like:

[<span>*|+][<align>][<style>]

  • <span> specifies horizontal and vertical cell spans (+ operator) or the number of times the cell is replicated (* operator). <span> is formatted like: 

    [<colspan>][.<rowspan>]

    Where <colspan> and <rowspan> are integers specifying the number of columns and rows to span.

  • <align> specifies horizontal and vertical cell alignment an is the same as in column specifiers.

  • A <style> value is the first letter of table style name.

For example, the following psv formatted cell will span two columns and the text will be centered and emphasized:

`2+^e| Cell text`

23.6. Table styles

Table styles can be applied to the entire table (by setting the style attribute in the table’s attribute list) or on a per column basis (by specifying the style in the table’s cols attribute). Table data can be formatted using the following predefined styles:

default

The default style: AsciiDoc inline text formatting; blank lines are treated as paragraph breaks.

emphasis

Like default but all text is emphasised.

monospaced

Like default but all text is in a monospaced font.

strong

Like default but all text is bold.

header

Apply the same style as the table header. Normally used to create a vertical header in the first column.

asciidoc

With this style table cells can contain any of the AsciiDoc elements that are allowed inside document sections. This style runs asciidoc(1) as a filter to process cell contents. See also Docbook table limitations.

literal

No text formatting; monospaced font; all line breaks are retained (the same as the AsciiDoc LiteralBlock element).

verse

All line breaks are retained (just like the AsciiDoc verse paragraph style).

23.7. Markup attributes

AsciiDoc makes a number of attributes available to table markup templates and tags. Column specific attributes are available when substituting the colspec cell data tags.

pageunits

DocBook backend only. Specifies table column absolute width units. Defaults to *.

pagewidth

DocBook backend only. The nominal output page width in pageunit units. Used to calculate CALS tables absolute column and table widths. Defaults to 425.

tableabswidth

Integer value calculated from width and pagewidth attributes. In pageunit units.

tablepcwidth

Table width expressed as a percentage of the available width. Integer value (0..100).

colabswidth

Integer value calculated from cols column width, width and pagewidth attributes. In pageunit units.

colpcwidth

Column width expressed as a percentage of the table width. Integer value (0..100).

colcount

Total number of table columns.

rowcount

Total number of table rows.

halign

Horizontal cell content alignment: left, right or center.

valign

Vertical cell content alignment: top, bottom or middle.

colnumber, colstart

The number of the leftmost column occupied by the cell (1…).

colend

The number of the rightmost column occupied by the cell (1…).

colspan

Number of columns the cell should span.

rowspan

Number of rows the cell should span (1…).

morerows

Number of additional rows the cell should span (0…).

23.8. Nested tables

An alternative psv separator character ! can be used (instead of |) in nested tables. This allows a single level of table nesting. Columns containing nested tables must use the asciidoc style. An example can be found in ./examples/website/newtables.txt.

23.9. DocBook table limitations

Fully implementing tables is not trivial, some DocBook toolchains do better than others. AsciiDoc HTML table outputs are rendered correctly in all the popular browsers — if your DocBook generated tables don’t look right compare them with the output generated by the AsciiDoc xhtml11 backend or try a different DocBook toolchain. Here is a list of things to be aware of:

  • Although nested tables are not legal in DocBook 4 the FOP and dblatex toolchains will process them correctly. If you use a2x(1) you will need to include the --no-xmllint option to suppress DocBook validation errors.

    Note In theory you can nest DocBook 4 tables one level using the entrytbl element, but not all toolchains process entrytbl.

  • DocBook only allows a subset of block elements inside table cells so not all AsciiDoc elements produce valid DocBook inside table cells. If you get validation errors running a2x(1) try the --no-xmllint option, toolchains will often process nested block elements such as sidebar blocks and floating titles correctly even though, strictly speaking, they are not legal.

  • Text formatting in cells using the monospaced table style will raise validation errors because the DocBook literal element was not designed to support formatted text (using the literal element is a kludge on the part of AsciiDoc as there is no easy way to set the font style in DocBook.

  • Cell alignments are ignored for verse, literal or asciidoc table styles.

24. Manpage Documents

Sooner or later, if you program in a UNIX environment, you’re going to have to write a man page.

By observing a couple of additional conventions (detailed below) you can write AsciiDoc files that will generate HTML and PDF man pages plus the native manpage roff format. The easiest way to generate roff manpages from AsciiDoc source is to use the a2x(1) command. The following example generates a roff formatted manpage file called asciidoc.1 (a2x(1) uses asciidoc(1) to convert asciidoc.1.txt to DocBook which it then converts to roff using DocBook XSL Stylesheets):

a2x --doctype manpage --format manpage asciidoc.1.txt
Viewing and printing manpage files

Use the man(1) command to view the manpage file:

$ man -l asciidoc.1

To print a high quality man page to a postscript printer:

$ man -l -Tps asciidoc.1 | lpr

You could also create a PDF version of the man page by converting PostScript to PDF using ps2pdf(1):

$ man -l -Tps asciidoc.1 | ps2pdf - asciidoc.1.pdf

The ps2pdf(1) command is included in the Ghostscript distribution.

To find out more about man pages view the man(7) manpage (man 7 man and man man-pages commands).

24.1. Document Header

A manpage document Header is mandatory. The title line contains the man page name followed immediately by the manual section number in brackets, for example ASCIIDOC(1). The title name should not contain white space and the manual section number is a single digit optionally followed by a single character.

24.2. The NAME Section

The first manpage section is mandatory, must be titled NAME and must contain a single paragraph (usually a single line) consisting of a list of one or more comma separated command name(s) separated from the command purpose by a dash character. The dash must have at least one white space character on either side. For example:

printf, fprintf, sprintf - print formatted output

24.3. The SYNOPSIS Section

The second manpage section is mandatory and must be titled SYNOPSIS.

24.4. refmiscinfo attributes

In addition to the automatically created man page intrinsic attributes you can assign DocBook refmiscinfo element source, version and manual values using AsciiDoc {mansource}, {manversion} and {manmanual} attributes respectively. This example is from the AsciiDoc header of a man page source file:

:man source:   AsciiDoc
:man version:  {revnumber}
:man manual:   AsciiDoc Manual

25. Mathematical Formulas

The asciimath and latexmath passthrough macros along with asciimath and latexmath passthrough blocks provide a (backend dependent) mechanism for rendering mathematical formulas. You can use the following math markups:

Note The latexmath macro used to include LaTeX Math in DocBook outputs is not the same as the latexmath macro used to include LaTeX MathML in XHTML outputs. LaTeX Math applies to DocBook outputs that are processed by dblatex and is normally used to generate PDF files. LaTeXMathML is very much a subset of LaTeX Math and applies to XHTML documents.

25.1. LaTeX Math

LaTeX math can be included in documents that are processed by dblatex(1). Example inline formula:

latexmath:[$C = \alpha + \beta Y^{\gamma} + \epsilon$]

For more examples see the AsciiDoc website or the distributed doc/latexmath.txt file.

25.2. ASCIIMathML

ASCIIMathML formulas can be included in XHTML documents generated using the xhtml11 and html5 backends. To enable ASCIIMathML support you must define the asciimath attribute, for example using the -a asciimath command-line option. Example inline formula:

asciimath:[`x/x={(1,if x!=0),(text{undefined},if x=0):}`]

For more examples see the AsciiDoc website or the distributed doc/asciimathml.txt file.

25.3. LaTeXMathML

LaTeXMathML allows LaTeX Math style formulas to be included in XHTML documents generated using the AsciiDoc xhtml11 and html5 backends. AsciiDoc uses the original LaTeXMathML by Douglas Woodall. LaTeXMathML is derived from ASCIIMathML and is for users who are more familiar with or prefer using LaTeX math formulas (it recognizes a subset of LaTeX Math, the differences are documented on the LaTeXMathML web page). To enable LaTeXMathML support you must define the latexmath attribute, for example using the -a latexmath command-line option. Example inline formula:

latexmath:[$\sum_{n=1}^\infty \frac{1}{2^n}$]

For more examples see the AsciiDoc website or the distributed doc/latexmathml.txt file.

25.4. MathML

MathML is a low level XML markup for mathematics. AsciiDoc has no macros for MathML but users familiar with this markup could use passthrough macros and passthrough blocks to include MathML in output documents.

26. Configuration Files

AsciiDoc source file syntax and output file markup is largely controlled by a set of cascading, text based, configuration files. At runtime The AsciiDoc default configuration files are combined with optional user and document specific configuration files.

26.1. Configuration File Format

Configuration files contain named sections. Each section begins with a section name in square brackets []. The section body consists of the lines of text between adjacent section headings.

  • Section names consist of one or more alphanumeric, underscore or dash characters and cannot begin or end with a dash.

  • Lines starting with a # character are treated as comments and ignored.

  • If the section name is prefixed with a + character then the section contents is appended to the contents of an already existing same-named section.

  • Otherwise same-named sections and section entries override previously loaded sections and section entries (this is sometimes referred to as cascading). Consequently, downstream configuration files need only contain those sections and section entries that need to be overridden.

Tip When creating custom configuration files you only need to include the sections and entries that differ from the default configuration.
Tip The best way to learn about configuration files is to read the default configuration files in the AsciiDoc distribution in conjunction with asciidoc(1) output files. You can view configuration file load sequence by turning on the asciidoc(1) -v (--verbose) command-line option.

AsciiDoc reserves the following section names for specific purposes:

miscellaneous

Configuration options that don’t belong anywhere else.

attributes

Attribute name/value entries.

specialcharacters

Special characters reserved by the backend markup.

tags

Backend markup tags.

quotes

Definitions for quoted inline character formatting.

specialwords

Lists of words and phrases singled out for special markup.

replacements, replacements2, replacements3

Find and replace substitution definitions.

specialsections

Used to single out special section names for specific markup.

macros

Macro syntax definitions.

titles

Heading, section and block title definitions.

paradef-*

Paragraph element definitions.

blockdef-*

DelimitedBlock element definitions.

listdef-*

List element definitions.

listtags-*

List element tag definitions.

tabledef-*

Table element definitions.

tabletags-*

Table element tag definitions.

Each line of text in these sections is a section entry. Section entries share the following syntax:

name=value

The entry value is set to value.

name=

The entry value is set to a zero length string.

name!

The entry is undefined (deleted from the configuration). This syntax only applies to attributes and miscellaneous sections.

Section entry behavior

  • All equals characters inside the name must be escaped with a backslash character.

  • name and value are stripped of leading and trailing white space.

  • Attribute names, tag entry names and markup template section names consist of one or more alphanumeric, underscore or dash characters. Names should not begin or end with a dash.

  • A blank configuration file section (one without any entries) deletes any preceding section with the same name (applies to non-markup template sections).

26.2. Miscellaneous section

The optional [miscellaneous] section specifies the following name=value options:

newline

Output file line termination characters. Can include any valid Python string escape sequences. The default value is \r\n (carriage return, line feed). Should not be quoted or contain explicit spaces (use \x20 instead). For example: 

$ asciidoc -a 'newline=\n' -b docbook mydoc.txt
outfilesuffix

The default extension for the output file, for example outfilesuffix=.html. Defaults to backend name.

tabsize

The number of spaces to expand tab characters, for example tabsize=4. Defaults to 8. A tabsize of zero suppresses tab expansion (useful when piping included files through block filters). Included files can override this option using the tabsize attribute.

pagewidth, pageunits

These global table related options are documented in the Table Configuration File Definitions sub-section.

Note [miscellaneous] configuration file entries can be set using the asciidoc(1) -a (--attribute) command-line option.

26.3. Titles section

sectiontitle

Two line section title pattern. The entry value is a Python regular expression containing the named group title.

underlines

A comma separated list of document and section title underline character pairs starting with the section level 0 and ending with section level 4 underline. The default setting is: 

underlines="==","--","~~","^^","++"
sect0…sect4

One line section title patterns. The entry value is a Python regular expression containing the named group title.

blocktitle

BlockTitle element pattern. The entry value is a Python regular expression containing the named group title.

subs

A comma separated list of substitutions that are performed on the document header and section titles. Defaults to normal substitution.

26.4. Tags section

The [tags] section contains backend tag definitions (one per line). Tags are used to translate AsciiDoc elements to backend markup.

An AsciiDoc tag definition is formatted like <tagname>=<starttag>|<endtag>. For example:

emphasis=<em>|</em>

In this example asciidoc(1) replaces the | character with the emphasized text from the AsciiDoc input file and writes the result to the output file.

Use the {brvbar} attribute reference if you need to include a | pipe character inside tag text.

26.5. Attributes section

The optional [attributes] section contains predefined attributes.

If the attribute value requires leading or trailing spaces then the text text should be enclosed in quotation mark (") characters.

To delete a attribute insert a name! entry in a downstream configuration file or use the asciidoc(1) --attribute name! command-line option (an attribute name suffixed with a ! character deletes the attribute)

26.6. Special Characters section

The [specialcharacters] section specifies how to escape characters reserved by the backend markup. Each translation is specified on a single line formatted like:

<special_character>=<translated_characters>

Special characters are normally confined to those that resolve markup ambiguity (in the case of HTML and XML markups the ampersand, less than and greater than characters). The following example causes all occurrences of the < character to be replaced by &lt;.

<=&lt;

26.7. Quoted Text section

Quoting is used primarily for text formatting. The [quotes] section defines AsciiDoc quoting characters and their corresponding backend markup tags. Each section entry value is the name of a of a [tags] section entry. The entry name is the character (or characters) that quote the text. The following examples are taken from AsciiDoc configuration files:

[quotes]
_=emphasis
[tags]
emphasis=<em>|</em>

You can specify the left and right quote strings separately by separating them with a | character, for example:

``|''=quoted

Omitting the tag will disable quoting, for example, if you don’t want superscripts or subscripts put the following in a custom configuration file or edit the global asciidoc.conf configuration file:

[quotes]
^=
~=

Unconstrained quotes are differentiated from constrained quotes by prefixing the tag name with a hash character, for example:

__=#emphasis
Quoted text behavior

  • Quote characters must be non-alphanumeric.

  • To minimize quoting ambiguity try not to use the same quote characters in different quote types.

26.8. Special Words section

The [specialwords] section is used to single out words and phrases that you want to consistently format in some way throughout your document without having to repeatedly specify the markup. The name of each entry corresponds to a markup template section and the entry value consists of a list of words and phrases to be marked up. For example:

[specialwords]
strongwords=NOTE IMPORTANT
[strongwords]
<strong>{words}</strong>

The examples specifies that any occurrence of NOTE or IMPORTANT should appear in a bold font.

Words and word phrases are treated as Python regular expressions: for example, the word ^NOTE would only match NOTE if appeared at the start of a line.

AsciiDoc comes with three built-in Special Word types: emphasizedwords, monospacedwords and strongwords, each has a corresponding (backend specific) markup template section. Edit the configuration files to customize existing Special Words and to add new ones.

Special word behavior

  • Word list entries must be separated by space characters.

  • Word list entries with embedded spaces should be enclosed in quotation (") characters.

  • A [specialwords] section entry of the form name=word1 [word2…] adds words to existing name entries.

  • A [specialwords] section entry of the form name undefines (deletes) all existing name words.

  • Since word list entries are processed as Python regular expressions you need to be careful to escape regular expression special characters.

  • By default Special Words are substituted before Inline Macros, this may lead to undesirable consequences. For example the special word foobar would be expanded inside the macro call http://www.foobar.com[]. A possible solution is to emphasize whole words only by defining the word using regular expression characters, for example \bfoobar\b.

  • If the first matched character of a special word is a backslash then the remaining characters are output without markup i.e. the backslash can be used to escape special word markup. For example the special word \\?\b[Tt]en\b will mark up the words Ten and ten only if they are not preceded by a backslash.

26.9. Replacements section

[replacements], [replacements2] and [replacements3] configuration file entries specify find and replace text and are formatted like:

<find_pattern>=<replacement_text>

The find text can be a Python regular expression; the replace text can contain Python regular expression group references.

Use Replacement shortcuts for often used macro references, for example (the second replacement allows us to backslash escape the macro name):

NEW!=image:./images/smallnew.png[New!]
\\NEW!=NEW!

The only difference between the three replacement types is how they are applied. By default replacements and replacement2 are applied in normal substitution contexts whereas replacements3 needs to be configured explicitly and should only be used in backend configuration files.

Replacement behavior

  • The built-in replacements can be escaped with a backslash.

  • If the find or replace text has leading or trailing spaces then the text should be enclosed in quotation (") characters.

  • Since the find text is processed as a regular expression you need to be careful to escape regular expression special characters.

  • Replacements are performed in the same order they appear in the configuration file replacements section.

26.10. Markup Template Sections

Markup template sections supply backend markup for translating AsciiDoc elements. Since the text is normally backend dependent you’ll find these sections in the backend specific configuration files. Template sections differ from other sections in that they contain a single block of text instead of per line name=value entries. A markup template section body can contain:

  • Attribute references

  • System macro calls.

  • A document content placeholder

The document content placeholder is a single | character and is replaced by text from the source element. Use the {brvbar} attribute reference if you need a literal | character in the template.

26.11. Configuration file names, precedence and locations

Configuration files have a .conf file name extension; they are loaded from the following locations:

  1. The directory containing the asciidoc executable.

  2. If there is no asciidoc.conf file in the directory containing the asciidoc executable then load from the global configuration directory (normally /etc/asciidoc or /usr/local/etc/asciidoc) i.e. the global configuration files directory is skipped if AsciiDoc configuration files are installed in the same directory as the asciidoc executable. This allows both a system wide copy and multiple local copies of AsciiDoc to coexist on the same host PC.

  3. The user’s $HOME/.asciidoc directory (if it exists).

  4. The directory containing the AsciiDoc source file.

  5. Explicit configuration files specified using:

    • The conf-files attribute (one or more file names separated by a | character). These files are loaded in the order they are specified and prior to files specified using the --conf-file command-line option.

    • The asciidoc(1) --conf-file) command-line option. The --conf-file option can be specified multiple times, in which case configuration files will be processed in the same order they appear on the command-line.

  6. Backend plugin configuration files are loaded from subdirectories named like backends/<backend> in locations 1, 2 and 3.

  7. Filter configuration files are loaded from subdirectories named like filters/<filter> in locations 1, 2 and 3.

Configuration files from the above locations are loaded in the following order:

  • The [attributes] section only from:

    • asciidoc.conf in location 3

    • Files from location 5.

      This first pass makes locally set attributes available in the global asciidoc.conf file.

  • asciidoc.conf from locations 1, 2, 3.

  • attributes, titles and specialcharacters sections from the asciidoc.conf in location 4.

  • The document header is parsed at this point and we can assume the backend and doctype have now been defined.

  • Backend plugin <backend>.conf and <backend>-<doctype>.conf files from locations 6. If a backend plugin is not found then try locations 1, 2 and 3 for <backend>.conf and <backend>-<doctype>.conf backend configuration files.

  • Filter conf files from locations 7.

  • lang-<lang>.conf from locations 1, 2, 3.

  • asciidoc.conf from location 4.

  • <backend>.conf and <backend>-<doctype>.conf from location 4.

  • Filter conf files from location 4.

  • <docfile>.conf and <docfile>-<backend>.conf from location 4.

  • Configuration files from location 5.

Where:

  • <backend> and <doctype> are values specified by the asciidoc(1) -b (--backend) and -d (--doctype) command-line options.

  • <infile> is the path name of the AsciiDoc input file without the file name extension.

  • <lang> is a two letter country code set by the the AsciiDoc lang attribute.

Note

The backend and language global configuration files are loaded after the header has been parsed. This means that you can set most attributes in the document header. Here’s an example header:

Life's Mysteries
================
:author: Hu Nose
:doctype: book
:toc:
:icons:
:data-uri:
:lang: en
:encoding: iso-8859-1

Attributes set in the document header take precedence over configuration file attributes.
Tip Use the asciidoc(1) -v (--verbose) command-line option to see which configuration files are loaded and the order in which they are loaded.

27. Document Attributes

A document attribute is comprised of a name and a textual value and is used for textual substitution in AsciiDoc documents and configuration files. An attribute reference (an attribute name enclosed in braces) is replaced by the corresponding attribute value. Attribute names are case insensitive and can only contain alphanumeric, dash and underscore characters.

There are four sources of document attributes (from highest to lowest precedence):

  • Command-line attributes.

  • AttributeEntry, AttributeList, Macro and BlockId elements.

  • Configuration file [attributes] sections.

  • Intrinsic attributes.

Within each of these divisions the last processed entry takes precedence.

Note If an attribute is not defined then the line containing the attribute reference is dropped. This property is used extensively in AsciiDoc configuration files to facilitate conditional markup generation.

28. Attribute Entries

The AttributeEntry block element allows document attributes to be assigned within an AsciiDoc document. Attribute entries are added to the global document attributes dictionary. The attribute name/value syntax is a single line like:

:<name>: <value>

For example:

:Author Initials: JB

This will set an attribute reference {authorinitials} to the value JB in the current document.

To delete (undefine) an attribute use the following syntax:

:<name>!:
AttributeEntry behavior

  • The attribute entry line begins with colon — no white space allowed in left margin.

  • AsciiDoc converts the <name> to a legal attribute name (lower case, alphanumeric, dash and underscore characters only — all other characters deleted). This allows more human friendly text to be used.

  • Leading and trailing white space is stripped from the <value>.

  • Lines ending in a space followed by a plus character are continued to the next line, for example: 

    :description: AsciiDoc is a text document format for writing notes, +
              documentation, articles, books, slideshows, web pages +
              and man pages.

  • If the <value> is blank then the corresponding attribute value is set to an empty string.

  • Attribute references contained in the entry <value> will be expanded.

  • By default AttributeEntry values are substituted for specialcharacters and attributes (see above), if you want to change or disable AttributeEntry substitution use the inline macro syntax.

  • Attribute entries in the document Header are available for header markup template substitution.

  • Attribute elements override configuration file and intrinsic attributes but do not override command-line attributes.

Here are some more attribute entry examples:

AsciiDoc User Manual
====================
:author:    Stuart Rackham
:email:     srackham@gmail.com
:revdate:   April 23, 2004
:revnumber: 5.1.1

Which creates these attributes:

{author}, {firstname}, {lastname}, {authorinitials}, {email},
{revdate}, {revnumber}

The previous example is equivalent to this document header:

AsciiDoc User Manual
====================
Stuart Rackham <srackham@gmail.com>
5.1.1, April 23, 2004

28.1. Setting configuration entries

A variant of the Attribute Entry syntax allows configuration file section entries and markup template sections to be set from within an AsciiDoc document:

:<section_name>.[<entry_name>]: <entry_value>

Where <section_name> is the configuration section name, <entry_name> is the name of the entry and <entry_value> is the optional entry value. This example sets the default labeled list style to horizontal:

:listdef-labeled.style: horizontal

It is exactly equivalent to a configuration file containing:

[listdef-labeled]
style=horizontal

  • If the <entry_name> is omitted then the entire section is substituted with the <entry_value>. This feature should only be used to set markup template sections. The following example sets the xref2 inline macro markup template: 

    :xref2-inlinemacro.: <a href="#{1}">{2?{2}}</a>

  • No substitution is performed on configuration file attribute entries and they cannot be undefined.

  • This feature can only be used in attribute entries — configuration attributes cannot be set using the asciidoc(1) command --attribute option.

Attribute entries promote clarity and eliminate repetition

URLs and file names in AsciiDoc macros are often quite long — they break paragraph flow and readability suffers. The problem is compounded by redundancy if the same name is used repeatedly. Attribute entries can be used to make your documents easier to read and write, here are some examples:

:1:         http://freshmeat.net/projects/asciidoc/
:homepage:  http://methods.co.nz/asciidoc/[AsciiDoc home page]
:new:       image:./images/smallnew.png[]
:footnote1: footnote:[A meaningless latin term]
Using previously defined attributes: See the {1}[Freshmeat summary]
or the {homepage} for something new {new}. Lorem ispum {footnote1}.
Note

  • The attribute entry definition must precede it’s usage.

  • You are not limited to URLs or file names, entire macro calls or arbitrary lines of text can be abbreviated.

  • Shared attributes entries could be grouped into a separate file and included in multiple documents.

29. Attribute Lists

  • An attribute list is a comma separated list of attribute values.

  • The entire list is enclosed in square brackets.

  • Attribute lists are used to pass parameters to macros, blocks (using the AttributeList element) and inline quotes.

The list consists of zero or more positional attribute values followed by zero or more named attribute values. Here are three examples: a single unquoted positional attribute; three unquoted positional attribute values; one positional attribute followed by two named attributes; the unquoted attribute value in the final example contains comma (&#44;) and double-quote (&#34;) character entities:

[Hello]
[quote, Bertrand Russell, The World of Mathematics (1956)]
["22 times", backcolor="#0e0e0e", options="noborders,wide"]
[A footnote&#44; &#34;with an image&#34; image:smallnew.png[]]
Attribute list behavior

  • If one or more attribute values contains a comma the all string values must be quoted (enclosed in double quotation mark characters).

  • If the list contains any named or quoted attributes then all string attribute values must be quoted.

  • To include a double quotation mark (") character in a quoted attribute value the the quotation mark must be escaped with a backslash.

  • List attributes take precedence over existing attributes.

  • List attributes can only be referenced in configuration file markup templates and tags, they are not available elsewhere in the document.

  • Setting a named attribute to None undefines the attribute.

  • Positional attributes are referred to as {1},{2},{3},…

  • Attribute {0} refers to the entire list (excluding the enclosing square brackets).

  • Named attribute names cannot contain dash characters.

29.1. Options attribute

If the attribute list contains an attribute named options it is processed as a comma separated list of option names:

  • Each name generates an attribute named like <option>-option (where <option> is the option name) with an empty string value. For example [options="opt1,opt2,opt3"] is equivalent to setting the following three attributes [opt1-option="",opt2-option="",opt2-option=""].

  • If you define a an option attribute globally (for example with an attribute entry) then it will apply to all elements in the document.

  • AsciiDoc implements a number of predefined options which are listed in the Attribute Options appendix.

29.2. Macro Attribute lists

Macros calls are suffixed with an attribute list. The list may be empty but it cannot be omitted. List entries are used to pass attribute values to macro markup templates.

30. Attribute References

An attribute reference is an attribute name (possibly followed by an additional parameters) enclosed in curly braces. When an attribute reference is encountered it is evaluated and replaced by its corresponding text value. If the attribute is undefined the line containing the attribute is dropped.

There are three types of attribute reference: Simple, Conditional and System.

Attribute reference evaluation

  • You can suppress attribute reference expansion by placing a backslash character immediately in front of the opening brace character.

  • By default attribute references are not expanded in LiteralParagraphs, ListingBlocks or LiteralBlocks.

  • Attribute substitution proceeds line by line in reverse line order.

  • Attribute reference evaluation is performed in the following order: Simple then Conditional and finally System.

30.1. Simple Attributes References

Simple attribute references take the form {<name>}. If the attribute name is defined its text value is substituted otherwise the line containing the reference is dropped from the output.

30.2. Conditional Attribute References

Additional parameters are used in conjunction with attribute names to calculate a substitution value. Conditional attribute references take the following forms:

{<names>=<value>}

<value> is substituted if the attribute <names> is undefined otherwise its value is substituted. <value> can contain simple attribute references.

{<names>?<value>}

<value> is substituted if the attribute <names> is defined otherwise an empty string is substituted. <value> can contain simple attribute references.

{<names>!<value>}

<value> is substituted if the attribute <names> is undefined otherwise an empty string is substituted. <value> can contain simple attribute references.

{<names>#<value>}

<value> is substituted if the attribute <names> is defined otherwise the undefined attribute entry causes the containing line to be dropped. <value> can contain simple attribute references.

{<names>%<value>}

<value> is substituted if the attribute <names> is not defined otherwise the containing line is dropped. <value> can contain simple attribute references.

{<names>@<regexp>:<value1>[:<value2>]}

<value1> is substituted if the value of attribute <names> matches the regular expression <regexp> otherwise <value2> is substituted. If attribute <names> is not defined the containing line is dropped. If <value2> is omitted an empty string is assumed. The values and the regular expression can contain simple attribute references. To embed colons in the values or the regular expression escape them with backslashes.

{<names>$<regexp>:<value1>[:<value2>]}

Same behavior as the previous ternary attribute except for the following cases:

{<names>$<regexp>:<value>}

Substitutes <value> if <names> matches <regexp> otherwise the result is undefined and the containing line is dropped.

{<names>$<regexp>::<value>}

Substitutes <value> if <names> does not match <regexp> otherwise the result is undefined and the containing line is dropped.

The attribute <names> parameter normally consists of a single attribute name but it can be any one of the following:

  • A single attribute name which evaluates to the attributes value.

  • Multiple , separated attribute names which evaluates to an empty string if one or more of the attributes is defined, otherwise it’s value is undefined.

  • Multiple + separated attribute names which evaluates to an empty string if all of the attributes are defined, otherwise it’s value is undefined.

Conditional attributes with single attribute names are evaluated first so they can be used inside the multi-attribute conditional <value>.

30.2.1. Conditional attribute examples

Conditional attributes are mainly used in AsciiDoc configuration files — see the distribution .conf files for examples.

Attribute equality test

If {backend} is docbook45 or xhtml11 the example evaluates to “DocBook 4.5 or XHTML 1.1 backend” otherwise it evaluates to “some other backend”: 

{backend@docbook45|xhtml11:DocBook 4.5 or XHTML 1.1 backend:some other backend}
Attribute value map

This example maps the frame attribute values [topbot, all, none, sides] to [hsides, border, void, vsides]: 

{frame@topbot:hsides}{frame@all:border}{frame@none:void}{frame@sides:vsides}

30.3. System Attribute References

System attribute references generate the attribute text value by executing a predefined action that is parametrized by one or more arguments. The syntax is {<action>:<arguments>}.

{counter:<attrname>[:<seed>]}

Increments the document attribute (if the attribute is undefined it is set to 1). Returns the new attribute value.

  • Counters generate global (document wide) attributes.

  • The optional <seed> specifies the counter’s initial value; it can be a number or a single letter; defaults to 1.

  • <seed> can contain simple and conditional attribute references.

  • The counter system attribute will not be executed if the containing line is dropped by the prior evaluation of an undefined attribute.

{counter2:<attrname>[:<seed>]}

Same as counter except the it always returns a blank string.

{eval:<expression>}

Substitutes the result of the Python <expression>.

  • If <expression> evaluates to None or False the reference is deemed undefined and the line containing the reference is dropped from the output.

  • If the expression evaluates to True the attribute evaluates to an empty string.

  • <expression> can contain simple and conditional attribute references.

  • The eval system attribute can be nested inside other system attributes.

{eval3:<command>}

Passthrough version of {eval:<expression>} — the generated output is written directly to the output without any further substitutions.

{include:<filename>}

Substitutes contents of the file named <filename>.

  • The included file is read at the time of attribute substitution.

  • If the file does not exist a warning is emitted and the line containing the reference is dropped from the output file.

  • Tabs are expanded based on the current tabsize attribute value.

{set:<attrname>[!][:<value>]}

Sets or unsets document attribute. Normally only used in configuration file markup templates (use AttributeEntries in AsciiDoc documents).

  • If the attribute name is followed by an exclamation mark the attribute becomes undefined.

  • If <value> is omitted the attribute is set to a blank string.

  • <value> can contain simple and conditional attribute references.

  • Returns a blank string unless the attribute is undefined in which case the return value is undefined and the enclosing line will be dropped.

{set2:<attrname>[!][:<value>]}

Same as set except that the attribute scope is local to the template.

{sys:<command>}

Substitutes the stdout generated by the execution of the shell <command>.

{sys2:<command>}

Substitutes the stdout and stderr generated by the execution of the shell <command>.

{sys3:<command>}

Passthrough version of {sys:<command>} — the generated output is written directly to the output without any further substitutions.

{template:<template>}

Substitutes the contents of the configuration file section named <template>. Attribute references contained in the template are substituted.

System reference behavior

  • System attribute arguments can contain non-system attribute references.

  • Closing brace characters inside system attribute arguments must be escaped with a backslash.

31. Intrinsic Attributes

Intrinsic attributes are simple attributes that are created automatically from: AsciiDoc document header parameters; asciidoc(1) command-line arguments; attributes defined in the default configuration files; the execution context. Here’s the list of predefined intrinsic attributes:

{amp}                 ampersand (&) character entity
{asciidoc-args}       used to pass inherited arguments to asciidoc filters
{asciidoc-confdir}    the asciidoc(1) global configuration directory
{asciidoc-dir}        the asciidoc(1) application directory
{asciidoc-file}       the full path name of the asciidoc(1) script
{asciidoc-version}    the version of asciidoc(1)
{author}              author's full name
{authored}            empty string '' if {author} or {email} defined,
{authorinitials}      author initials (from document header)
{backend-<backend>}   empty string ''
{<backend>-<doctype>} empty string ''
{backend}             document backend specified by `-b` option
{backend-confdir}     the directory containing the <backend>.conf file
{backslash}           backslash character
{basebackend-<base>}  empty string ''
{basebackend}         html or docbook
{blockname}           current block name (note 8).
{brvbar}              broken vertical bar (|) character
{docdate}             document last modified date
{docdir}              document input directory name  (note 5)
{docfile}             document file name  (note 5)
{docname}             document file name without extension (note 6)
{doctime}             document last modified time
{doctitle}            document title (from document header)
{doctype-<doctype>}   empty string ''
{doctype}             document type specified by `-d` option
{email}               author's email address (from document header)
{empty}               empty string ''
{encoding}            specifies input and output encoding
{filetype-<fileext>}  empty string ''
{filetype}            output file name file extension
{firstname}           author first name (from document header)
{gt}                  greater than (>) character entity
{id}                  running block id generated by BlockId elements
{indir}               input file directory name (note 2,5)
{infile}              input file name (note 2,5)
{lastname}            author last name (from document header)
{ldquo}               Left double quote character (note 7)
{level}               title level 1..4 (in section titles)
{listindex}           the list index (1..) of the most recent list item
{localdate}           the current date
{localtime}           the current time
{lsquo}               Left single quote character (note 7)
{lt}                  less than (<) character entity
{manname}             manpage name (defined in NAME section)
{manpurpose}          manpage (defined in NAME section)
{mantitle}            document title minus the manpage volume number
{manvolnum}           manpage volume number (1..8) (from document header)
{middlename}          author middle name (from document header)
{nbsp}                non-breaking space character entity
{notitle}             do not display the document title
{outdir}              document output directory name (note 2)
{outfile}             output file name (note 2)
{python}              the full path name of the Python interpreter executable
{rdquo}               Right double quote character (note 7)
{reftext}             running block xreflabel generated by BlockId elements
{revdate}             document revision date (from document header)
{revnumber}           document revision number (from document header)
{rsquo}               Right single quote character (note 7)
{sectnum}             formatted section number (in section titles)
{sp}                  space character
{showcomments}        send comment lines to the output
{title}               section title (in titled elements)
{two-colons}          Two colon characters
{two-semicolons}      Two semicolon characters
{user-dir}            the ~/.asciidoc directory (if it exists)
{verbose}             defined as '' if --verbose command option specified
{wj}                  Word-joiner
{zwsp}                Zero-width space character entity
Note

  1. Intrinsic attributes are global so avoid defining custom attributes with the same names.

  2. {outfile}, {outdir}, {infile}, {indir} attributes are effectively read-only (you can set them but it won’t affect the input or output file paths).

  3. See also the Backend Attributes section for attributes that relate to AsciiDoc XHTML file generation.

  4. The entries that translate to blank strings are designed to be used for conditional text inclusion. You can also use the ifdef, ifndef and endif System macros for conditional inclusion.
    [Conditional inclusion using ifdef and ifndef macros differs from attribute conditional inclusion in that the former occurs when the file is read while the latter occurs when the contents are written.]

  5. {docfile} and {docdir} refer to root document specified on the asciidoc(1) command-line; {infile} and {indir} refer to the current input file which may be the root document or an included file. When the input is being read from the standard input (stdin) these attributes are undefined.

  6. If the input file is the standard input and the output file is not the standard output then {docname} is the output file name sans file extension.

  7. See non-English usage of quotation marks.

  8. The {blockname} attribute identifies the style of the current block. It applies to delimited blocks, lists and tables. Here is a list of {blockname} values (does not include filters or custom block and style names):

    delimited blocks

    comment, sidebar, open, pass, literal, verse, listing, quote, example, note, tip, important, caution, warning, abstract, partintro

    lists

    arabic, loweralpha, upperalpha, lowerroman, upperroman, labeled, labeled3, labeled4, qanda, horizontal, bibliography, glossary

    tables

    table

32. Block Element Definitions

The syntax and behavior of Paragraph, DelimitedBlock, List and Table block elements is determined by block definitions contained in AsciiDoc configuration file sections.

Each definition consists of a section title followed by one or more section entries. Each entry defines a block parameter controlling some aspect of the block’s behavior. Here’s an example:

[blockdef-listing]
delimiter=^-{4,}$
template=listingblock
presubs=specialcharacters,callouts

Configuration file block definition sections are processed incrementally after each configuration file is loaded. Block definition section entries are merged into the block definition, this allows block parameters to be overridden and extended by later loading configuration files.

AsciiDoc Paragraph, DelimitedBlock, List and Table block elements share a common subset of configuration file parameters:

delimiter

A Python regular expression that matches the first line of a block element — in the case of DelimitedBlocks and Tables it also matches the last line.

template

The name of the configuration file markup template section that will envelope the block contents. The pipe (|) character is substituted for the block contents. List elements use a set of (list specific) tag parameters instead of a single template. The template name can contain attribute references allowing dynamic template selection a the time of template substitution.

options

A comma delimited list of element specific option names. In addition to being used internally, options are available during markup tag and template substitution as attributes with an empty string value named like <option>-option (where <option> is the option name). See attribute options for a complete list of available options.

subs, presubs, postsubs

  • presubs and postsubs are lists of comma separated substitutions that are performed on the block contents. presubs is applied first, postsubs (if specified) second.

  • subs is an alias for presubs.

  • If a filter is allowed (Paragraphs, DelimitedBlocks and Tables) and has been specified then presubs and postsubs substitutions are performed before and after the filter is run respectively.

  • Allowed values: specialcharacters, quotes, specialwords, replacements, macros, attributes, callouts.

  • The following composite values are also allowed:

    none

    No substitutions.

    normal

    The following substitutions in the following order: specialcharacters, quotes, attributes, specialwords, replacements, macros, replacements2.

    verbatim

    The following substitutions in the following order: specialcharacters and callouts.

  • normal and verbatim substitutions can be redefined by with subsnormal and subsverbatim entries in a configuration file [miscellaneous] section.

  • The substitutions are processed in the order in which they are listed and can appear more than once.

filter

This optional entry specifies an executable shell command for processing block content (Paragraphs, DelimitedBlocks and Tables). The filter command can contain attribute references.

posattrs

Optional comma separated list of positional attribute names. This list maps positional attributes (in the block’s attribute list) to named block attributes. The following example, from the QuoteBlock definition, maps the first and section positional attributes: 

posattrs=attribution,citetitle
style

This optional parameter specifies the default style name.

<stylename>-style

Optional style definition (see Styles below).

The following block parameters behave like document attributes and can be set in block attribute lists and style definitions: template, options, subs, presubs, postsubs, filter.

32.1. Styles

A style is a set of block parameter bundled as a single named parameter. The following example defines a style named verbatim:

verbatim-style=template="literalblock",subs="verbatim"

If a block’s attribute list contains a style attribute then the corresponding style parameters are be merged into the default block definition parameters.

  • All style parameter names must be suffixed with -style and the style parameter value is in the form of a list of named attributes.

  • The template style parameter is mandatory, other parameters can be omitted in which case they inherit their values from the default block definition parameters.

  • Multi-item style parameters (subs,presubs,postsubs,posattrs) must be specified using Python tuple syntax (rather than a simple list of values as they in separate entries) e.g. postsubs=("callouts",) not postsubs="callouts".

32.2. Paragraphs

Paragraph translation is controlled by [paradef-*] configuration file section entries. Users can define new types of paragraphs and modify the behavior of existing types by editing AsciiDoc configuration files.

Here is the shipped Default paragraph definition:

[paradef-default]
delimiter=(?P<text>\S.*)
template=paragraph

The normal paragraph definition has a couple of special properties:

  1. It must exist and be defined in a configuration file section named [paradef-default].

  2. Irrespective of its position in the configuration files default paragraph document matches are attempted only after trying all other paragraph types.

Paragraph specific block parameter notes:

delimiter

This regular expression must contain the named group text which matches the text on the first line. Paragraphs are terminated by a blank line, the end of file, or the start of a DelimitedBlock.

options

The listelement option specifies that paragraphs of this type will automatically be considered part of immediately preceding list items. The skip option causes the paragraph to be treated as a comment (see CommentBlocks).

Paragraph processing proceeds as follows:

  1. The paragraph text is aligned to the left margin.

  2. Optional presubs inline substitutions are performed on the paragraph text.

  3. If a filter command is specified it is executed and the paragraph text piped to its standard input; the filter output replaces the paragraph text.

  4. Optional postsubs inline substitutions are performed on the paragraph text.

  5. The paragraph text is enveloped by the paragraph’s markup template and written to the output file.

32.3. Delimited Blocks

DelimitedBlock options values are:

sectionbody

The block contents are processed as a SectionBody.

skip

The block is treated as a comment (see CommentBlocks). Preceding attribute lists and block titles are not consumed.

presubs, postsubs and filter entries are ignored when sectionbody or skip options are set.

DelimitedBlock processing proceeds as follows:

  1. Optional presubs substitutions are performed on the block contents.

  2. If a filter is specified it is executed and the block’s contents piped to its standard input. The filter output replaces the block contents.

  3. Optional postsubs substitutions are performed on the block contents.

  4. The block contents is enveloped by the block’s markup template and written to the output file.

Tip Attribute expansion is performed on the block filter command before it is executed, this is useful for passing arguments to the filter.

32.4. Lists

List behavior and syntax is determined by [listdef-*] configuration file sections. The user can change existing list behavior and add new list types by editing configuration files.

List specific block definition notes:

type

This is either bulleted,numbered,labeled or callout.

delimiter

A Python regular expression that matches the first line of a list element entry. This expression can contain the named groups text (bulleted groups), index and text (numbered lists), label and text (labeled lists).

tags

The <name> of the [listtags-<name>] configuration file section containing list markup tag definitions. The tag entries (list, entry, label, term, text) map the AsciiDoc list structure to backend markup; see the listtags sections in the AsciiDoc distributed backend .conf configuration files for examples.

32.5. Tables

Table behavior and syntax is determined by [tabledef-*] and [tabletags-*] configuration file sections. The user can change existing table behavior and add new table types by editing configuration files. The following [tabledef-*] section entries generate table output markup elements:

colspec

The table colspec tag definition.

headrow, footrow, bodyrow

Table header, footer and body row tag definitions. headrow and footrow table definition entries default to bodyrow if they are undefined.

headdata, footdata, bodydata

Table header, footer and body data tag definitions. headdata and footdata table definition entries default to bodydata if they are undefined.

paragraph

If the paragraph tag is specified then blank lines in the cell data are treated as paragraph delimiters and marked up using this tag.

Table behavior is also influenced by the following [miscellaneous] configuration file entries:

pagewidth

This integer value is the printable width of the output media. See table attributes.

pageunits

The units of width in output markup width attribute values.

Table definition behavior

  • The output markup generation is specifically designed to work with the HTML and CALS (DocBook) table models, but should be adaptable to most XML table schema.

  • Table definitions can be “mixed in” from multiple cascading configuration files.

  • New table definitions inherit the default table and table tags definitions ([tabledef-default] and [tabletags-default]) so you only need to override those conf file entries that require modification.

33. Filters

AsciiDoc filters allow external commands to process AsciiDoc Paragraphs, DelimitedBlocks and Table content. Filters are primarily an extension mechanism for generating specialized outputs. Filters are implemented using external commands which are specified in configuration file definitions.

There’s nothing special about the filters, they’re just standard UNIX filters: they read text from the standard input, process it, and write to the standard output.

The asciidoc(1) command --filter option can be used to install and remove filters. The same option is used to unconditionally load a filter.

Attribute substitution is performed on the filter command prior to execution — attributes can be used to pass parameters from the AsciiDoc source document to the filter.

Warning Filters sometimes included executable code. Before installing a filter you should verify that it is from a trusted source.

33.1. Filter Search Paths

If the filter command does not specify a directory path then asciidoc(1) recursively searches for the executable filter command:

  • First it looks in the user’s $HOME/.asciidoc/filters directory.

  • Next the global filters directory (usually /etc/asciidoc/filters or /usr/local/etc/asciidoc) directory is searched.

  • Then it looks in the asciidoc(1) ./filters directory.

  • Finally it relies on the executing shell to search the environment search path ($PATH).

Standard practice is to install each filter in it’s own sub-directory with the same name as the filter’s style definition. For example the music filter’s style name is music so it’s configuration and filter files are stored in the filters/music directory.

33.2. Filter Configuration Files

Filters are normally accompanied by a configuration file containing a Paragraph or DelimitedBlock definition along with corresponding markup templates.

While it is possible to create new Paragraph or DelimitedBlock definitions the preferred way to implement a filter is to add a style to the existing Paragraph and ListingBlock definitions (all filters shipped with AsciiDoc use this technique). The filter is applied to the paragraph or delimited block by preceding it with an attribute list: the first positional attribute is the style name, remaining attributes are normally filter specific parameters.

asciidoc(1) auto-loads all .conf files found in the filter search paths unless the container directory also contains a file named __noautoload__ (see previous section). The __noautoload__ feature is used for filters that will be loaded manually using the --filter option.

33.3. Example Filter

AsciiDoc comes with a toy filter for highlighting source code keywords and comments. See also the ./filters/code/code-filter-readme.txt file.

Note The purpose of this toy filter is to demonstrate how to write a filter — it’s much to simplistic to be passed off as a code syntax highlighter. If you want a full featured multi-language highlighter use the source code highlighter filter.

33.4. Built-in filters

The AsciiDoc distribution includes source, music, latex and graphviz filters, details are on the AsciiDoc website.

Table 11. Built-in filters list
Filter name Description

music

A music filter is included in the distribution ./filters/ directory. It translates music in LilyPond or ABC notation to standard classical notation.

source

A source code highlight filter is included in the distribution ./filters/ directory.

latex

The AsciiDoc LaTeX filter translates LaTeX source to a PNG image that is automatically inserted into the AsciiDoc output documents.

graphviz

Gouichi Iisaka has written a Graphviz filter for AsciiDoc. Graphviz generates diagrams from a textual specification. Gouichi Iisaka’s Graphviz filter is included in the AsciiDoc distribution. Here are some AsciiDoc Graphviz examples.

33.5. Filter plugins

Filter plugins are a mechanism for distributing AsciiDoc filters. A filter plugin is a Zip file containing the files that constitute a filter. The asciidoc(1) --filter option is used to load and manage filer plugins.

  • Filter plugins take precedence over built-in filters with the same name.

  • By default filter plugins are installed in $HOME/.asciidoc/filters/<filter> where <filter> is the filter name.

34. Plugins

The AsciiDoc plugin architecture is an extension mechanism that allows additional backends, filters and themes to be added to AsciiDoc.

  • A plugin is a Zip file containing an AsciiDoc backend, filter or theme (configuration files, stylesheets, scripts, images).

  • The asciidoc(1) --backend, --filter and --theme command-line options are used to load and manage plugins. Each of these options responds to the plugin management install, list, remove and build commands.

  • The plugin management command names are reserved and cannot be used for filter, backend or theme names.

  • The plugin Zip file name always begins with the backend, filter or theme name.

Plugin commands and conventions are documented in the asciidoc(1) man page. You can find lists of plugins on the AsciiDoc website.

35. Help Commands

The asciidoc(1) command has a --help option which prints help topics to stdout. The default topic summarizes asciidoc(1) usage:

$ asciidoc --help

To print a help topic specify the topic name as a command argument. Help topic names can be shortened so long as they are not ambiguous. Examples:

$ asciidoc --help manpage
$ asciidoc -h m              # Short version of previous example.
$ asciidoc --help syntax
$ asciidoc -h s              # Short version of previous example.

35.1. Customizing Help

To change, delete or add your own help topics edit a help configuration file. The help file name help-<lang>.conf is based on the setting of the lang attribute, it defaults to help.conf (English). The help file location will depend on whether you want the topics to apply to all users or just the current user.

The help topic files have the same named section format as other configuration files. The help.conf files are stored in the same locations and loaded in the same order as other configuration files.

When the --help command-line option is specified AsciiDoc loads the appropriate help files and then prints the contents of the section whose name matches the help topic name. If a topic name is not specified default is used. You don’t need to specify the whole help topic name on the command-line, just enough letters to ensure it’s not ambiguous. If a matching help file section is not found a list of available topics is printed.

36. Tips and Tricks

36.1. Know Your Editor

Writing AsciiDoc documents will be a whole lot more pleasant if you know your favorite text editor. Learn how to indent and reformat text blocks, paragraphs, lists and sentences. Tips for vim users follow.

36.2. Vim Commands for Formatting AsciiDoc

36.2.1. Text Wrap Paragraphs

Use the vim :gq command to reformat paragraphs. Setting the textwidth sets the right text wrap margin; for example:

:set textwidth=70

To reformat a paragraph:

  1. Position the cursor at the start of the paragraph.

  2. Type gq}.

Execute :help gq command to read about the vim gq command.

Tip

  • Assign the gq} command to the Q key with the nnoremap Q gq} command or put it in your ~/.vimrc file to so it’s always available (see the Example ~/.vimrc file).

  • Put set commands in your ~/.vimrc file so you don’t have to enter them manually.

  • The Vim website (http://www.vim.org) has a wealth of resources, including scripts for automated spell checking and ASCII Art drawing.

36.2.2. Format Lists

The gq command can also be used to format bulleted, numbered and callout lists. First you need to set the comments, formatoptions and formatlistpat (see the Example ~/.vimrc file).

Now you can format simple lists that use dash, asterisk, period and plus bullets along with numbered ordered lists:

  1. Position the cursor at the start of the list.

  2. Type gq}.

36.2.3. Indent Paragraphs

Indent whole paragraphs by indenting the fist line with the desired indent and then executing the gq} command.

36.2.4. Example ~/.vimrc File

" Use bold bright fonts.
set background=dark

" Show tabs and trailing characters.
set listchars=tab:»·,trail:·
set list

" Don't highlight searched text.
highlight clear Search

" Don't move to matched text while search pattern is being entered.
set noincsearch

" Reformat paragraphs and list.
nnoremap R gq}

" Delete trailing white space and Dos-returns and to expand tabs to spaces.
nnoremap S :set et<CR>:retab!<CR>:%s/[\r \t]\+$//<CR>

autocmd BufRead,BufNewFile *.txt,README,TODO,CHANGELOG,NOTES
        \ setlocal autoindent expandtab tabstop=8 softtabstop=2 shiftwidth=2 filetype=asciidoc
        \ textwidth=70 wrap formatoptions=tcqn
        \ formatlistpat=^\\s*\\d\\+\\.\\s\\+\\\\|^\\s*<\\d\\+>\\s\\+\\\\|^\\s*[a-zA-Z.]\\.\\s\\+\\\\|^\\s*[ivxIVX]\\+\\.\\s\\+
        \ comments=s1:/*,ex:*/,://,b:#,:%,:XCOMM,fb:-,fb:*,fb:+,fb:.,fb:>

36.3. Troubleshooting

AsciiDoc diagnostic features are detailed in the Diagnostics appendix.

36.4. Gotchas

Incorrect character encoding

If you get an error message like 'UTF-8' codec can't decode ... then you source file contains invalid UTF-8 characters — set the AsciiDoc encoding attribute for the correct character set (typically ISO-8859-1 (Latin-1) for European languages).

Invalid output

AsciiDoc attempts to validate the input AsciiDoc source but makes no attempt to validate the output markup, it leaves that to external tools such as xmllint(1) (integrated into a2x(1)). Backend validation cannot be hardcoded into AsciiDoc because backends are dynamically configured. The following example generates valid HTML but invalid DocBook (the DocBook literal element cannot contain an emphasis element): 

+monospaced text with an _emphasized_ word+
Misinterpreted text formatting

You can suppress markup expansion by placing a backslash character immediately in front of the element. The following example suppresses inline monospaced formatting: 

\+1 for C++.
Overlapping text formatting

Overlapping text formatting will generate illegal overlapping markup tags which will result in downstream XML parsing errors. Here’s an example: 

Some *strong markup _that overlaps* emphasized markup_.
Ambiguous underlines

A DelimitedBlock can immediately follow a paragraph without an intervening blank line, but be careful, a single line paragraph underline may be misinterpreted as a section title underline resulting in a “closing block delimiter expected” error.

Ambiguous ordered list items

Lines beginning with numbers at the end of sentences will be interpreted as ordered list items. The following example (incorrectly) begins a new list with item number 1999: 

He was last sighted in
1999. Since then things have moved on.

The list item out of sequence warning makes it unlikely that this problem will go unnoticed.

Special characters in attribute values

Special character substitution precedes attribute substitution so if attribute values contain special characters you may, depending on the substitution context, need to escape the special characters yourself. For example: 

$ asciidoc -a 'orgname=Bill &amp; Ben Inc.' mydoc.txt
Attribute lists

If any named attribute entries are present then all string attribute values must be quoted. For example: 

["Desktop screenshot",width=32]

36.5. Combining separate documents

You have a number of stand-alone AsciiDoc documents that you want to process as a single document. Simply processing them with a series of include macros won’t work because the documents contain (level 0) document titles. The solution is to create a top level wrapper document and use the leveloffset attribute to push them all down one level. For example:

Combined Document Title
=======================

// Push titles down one level.
:leveloffset: 1

include::document1.txt[]

// Return to normal title levels.
:leveloffset: 0

A Top Level Section
-------------------
Lorum ipsum.

// Push titles down one level.
:leveloffset: 1

include::document2.txt[]

include::document3.txt[]

The document titles in the included documents will now be processed as level 1 section titles, level 1 sections as level 2 sections and so on.

  • Put a blank line between the include macro lines to ensure the title of the included document is not seen as part of the last paragraph of the previous document.

  • You won’t want non-title document header lines (for example, Author and Revision lines) in the included files — conditionally exclude them if they are necessary for stand-alone processing.

36.6. Processing document sections separately

You have divided your AsciiDoc document into separate files (one per top level section) which are combined and processed with the following top level document:

Combined Document Title
=======================
Joe Bloggs
v1.0, 12-Aug-03

include::section1.txt[]

include::section2.txt[]

include::section3.txt[]

You also want to process the section files as separate documents. This is easy because asciidoc(1) will quite happily process section1.txt, section2.txt and section3.txt separately — the resulting output documents contain the section but have no document title.

36.7. Processing document snippets

Use the -s (--no-header-footer) command-line option to suppress header and footer output, this is useful if the processed output is to be included in another file. For example:

$ asciidoc -sb docbook section1.txt

asciidoc(1) can be used as a filter, so you can pipe chunks of text through it. For example:

$ echo 'Hello *World!*' | asciidoc -s -
<div class="paragraph"><p>Hello <strong>World!</strong></p></div>

36.8. Badges in HTML page footers

See the [footer] section in the AsciiDoc distribution xhtml11.conf configuration file.

36.9. Pretty printing AsciiDoc output

If the indentation and layout of the asciidoc(1) output is not to your liking you can:

  1. Change the indentation and layout of configuration file markup template sections. The {empty} attribute is useful for outputting trailing blank lines in markup templates.

  2. Use Dave Raggett’s HTML Tidy program to tidy asciidoc(1) output. Example: 

    $ asciidoc -b docbook -o - mydoc.txt | tidy -indent -xml >mydoc.xml

  3. Use the xmllint(1) format option. Example: 

    $ xmllint --format mydoc.xml

36.10. Supporting minor DocBook DTD variations

The conditional inclusion of DocBook SGML markup at the end of the distribution docbook45.conf file illustrates how to support minor DTD variations. The included sections override corresponding entries from preceding sections.

36.11. Creating stand-alone HTML documents

If you’ve ever tried to send someone an HTML document that includes stylesheets and images you’ll know that it’s not as straight-forward as exchanging a single file. AsciiDoc has options to create stand-alone documents containing embedded images, stylesheets and scripts. The following AsciiDoc command creates a single file containing embedded images, CSS stylesheets, and JavaScript (for table of contents and footnotes):

$ asciidoc -a data-uri -a icons -a toc -a max-width=55em article.txt

You can view the HTML file here: http://www.methods.co.nz/asciidoc/article-standalone.html

36.12. Shipping stand-alone AsciiDoc source

Reproducing presentation documents from someone else’s source has one major problem: unless your configuration files are the same as the creator’s you won’t get the same output.

The solution is to create a single backend specific configuration file using the asciidoc(1) -c (--dump-conf) command-line option. You then ship this file along with the AsciiDoc source document plus the asciidoc.py script. The only end user requirement is that they have Python installed (and that they consider you a trusted source). This example creates a composite HTML configuration file for mydoc.txt:

$ asciidoc -cb xhtml11 mydoc.txt > mydoc-xhtml11.conf

Ship mydoc.txt, mydoc-html.conf, and asciidoc.py. With these three files (and a Python interpreter) the recipient can regenerate the HMTL output:

$ ./asciidoc.py -eb xhtml11 mydoc.txt

The -e (--no-conf) option excludes the use of implicit configuration files, ensuring that only entries from the mydoc-html.conf configuration are used.

36.13. Inserting blank space

Adjust your style sheets to add the correct separation between block elements. Inserting blank paragraphs containing a single non-breaking space character {nbsp} works but is an ad hoc solution compared to using style sheets.

36.14. Closing open sections

You can close off section tags up to level N by calling the eval::[Section.setlevel(N)] system macro. This is useful if you want to include a section composed of raw markup. The following example includes a DocBook glossary division at the top section level (level 0):

ifdef::basebackend-docbook[]

eval::[Section.setlevel(0)]

+++++++++++++++++++++++++++++++
<glossary>
  <title>Glossary</title>
  <glossdiv>
  ...
  </glossdiv>
</glossary>
+++++++++++++++++++++++++++++++
endif::basebackend-docbook[]

36.15. Validating output files

Use xmllint(1) to check the AsciiDoc generated markup is both well formed and valid. Here are some examples:

$ xmllint --nonet --noout --valid docbook-file.xml
$ xmllint --nonet --noout --valid xhtml11-file.html
$ xmllint --nonet --noout --valid --html html4-file.html

The --valid option checks the file is valid against the document type’s DTD, if the DTD is not installed in your system’s catalog then it will be fetched from its Internet location. If you omit the --valid option the document will only be checked that it is well formed.

The online W3C Markup Validation Service is the defacto standard when it comes to validating HTML (it validates all HTML standards including HTML5).

Glossary

Block element

An AsciiDoc block element is a document entity composed of one or more whole lines of text.

Inline element

AsciiDoc inline elements occur within block element textual content, they perform formatting and substitution tasks.

Formal element

An AsciiDoc block element that has a BlockTitle. Formal elements are normally listed in front or back matter, for example lists of tables, examples and figures.

Verbatim element

The word verbatim indicates that white space and line breaks in the source document are to be preserved in the output document.

Appendix A: Migration Notes

Version 7 to version 8

  • A new set of quotes has been introduced which may match inline text in existing documents — if they do you’ll need to escape the matched text with backslashes.

  • The index entry inline macro syntax has changed — if your documents include indexes you may need to edit them.

  • Replaced a2x(1) --no-icons and --no-copy options with their negated equivalents: --icons and --copy respectively. The default behavior has also changed — the use of icons and copying of icon and CSS files must be specified explicitly with the --icons and --copy options.

The rationale for the changes can be found in the AsciiDoc CHANGELOG.

Note If you want to disable unconstrained quotes, the new alternative constrained quotes syntax and the new index entry syntax then you can define the attribute asciidoc7compatible (for example by using the -a asciidoc7compatible command-line option).

Appendix B: Packager Notes

Read the README and INSTALL files (in the distribution root directory) for install prerequisites and procedures. The distribution Makefile.in (used by configure to generate the Makefile) is the canonical installation procedure.

Appendix C: AsciiDoc Safe Mode

AsciiDoc safe mode skips potentially dangerous scripted sections in AsciiDoc source files by inhibiting the execution of arbitrary code or the inclusion of arbitrary files.

The safe mode is disabled by default, it can be enabled with the asciidoc(1) --safe command-line option.

Safe mode constraints

  • eval, sys and sys2 executable attributes and block macros are not executed.

  • include::<filename>[] and include1::<filename>[] block macro files must reside inside the parent file’s directory.

  • {include:<filename>} executable attribute files must reside inside the source document directory.

  • Passthrough Blocks are dropped.

Warning

The safe mode is not designed to protect against unsafe AsciiDoc configuration files. Be especially careful when:

  1. Implementing filters.

  2. Implementing elements that don’t escape special characters.

  3. Accepting configuration files from untrusted sources.

Appendix D: Using AsciiDoc with non-English Languages

AsciiDoc can process UTF-8 character sets but there are some things you need to be aware of:

  • If you are generating output documents using a DocBook toolchain then you should set the AsciiDoc lang attribute to the appropriate language (it defaults to en (English)). This will ensure things like table of contents, figure and table captions and admonition captions are output in the specified language. For example: 

    $ a2x -a lang=es doc/article.txt

  • If you are outputting HTML directly from asciidoc(1) you’ll need to set the various *_caption attributes to match your target language (see the list of captions and titles in the [attributes] section of the distribution lang-*.conf files). The easiest way is to create a language .conf file (see the AsciiDoc’s lang-en.conf file).

    Note You still use the NOTE, CAUTION, TIP, WARNING, IMPORTANT captions in the AsciiDoc source, they get translated in the HTML output file.

  • asciidoc(1) automatically loads configuration files named like lang-<lang>.conf where <lang> is a two letter language code that matches the current AsciiDoc lang attribute. See also Configuration File Names and Locations.

Appendix E: Vim Syntax Highlighter

Syntax highlighting is incredibly useful, in addition to making reading AsciiDoc documents much easier syntax highlighting also helps you catch AsciiDoc syntax errors as you write your documents.

The AsciiDoc ./vim/ distribution directory contains Vim syntax highlighter and filetype detection scripts for AsciiDoc. Syntax highlighting makes it much easier to spot AsciiDoc syntax errors.

If Vim is installed on your system the AsciiDoc installer (install.sh) will automatically install the vim scripts in the Vim global configuration directory (/etc/vim).

You can also turn on syntax highlighting by adding the following line to the end of you AsciiDoc source files:

// vim: set syntax=asciidoc:
Tip Bold fonts are often easier to read, use the Vim :set background=dark command to set bold bright fonts.
Note There are a number of alternative syntax highlighters for various editors listed on the AsciiDoc website.

Limitations

The current implementation does a reasonable job but on occasions gets things wrong:

  • Nested quoted text formatting is highlighted according to the outer format.

  • If a closing Example Block delimiter is sometimes mistaken for a title underline. A workaround is to insert a blank line before the closing delimiter.

  • Lines within a paragraph starting with equals characters may be highlighted as single-line titles.

  • Lines within a paragraph beginning with a period may be highlighted as block titles.

Appendix F: Attribute Options

Here is the list of predefined attribute list options:

Option Backends AsciiDoc Elements Description

autowidth

xhtml11, html5, html4

table

The column widths are determined by the browser, not the AsciiDoc cols attribute. If there is no width attribute the table width is also left up to the browser.

unbreakable

xhtml11, html5

block elements

unbreakable attempts to keep the block element together on a single printed page c.f. the breakable and unbreakable docbook (XSL/FO) options below.

breakable, unbreakable

docbook (XSL/FO)

table, example, block image

The breakable options allows block elements to break across page boundaries; unbreakable attempts to keep the block element together on a single page. If neither option is specified the default XSL stylesheet behavior prevails.

compact

docbook, xhtml11, html5

bulleted list, numbered list

Minimizes vertical space in the list

footer

docbook, xhtml11, html5, html4

table

The last row of the table is rendered as a footer.

header

docbook, xhtml11, html5, html4

table

The first row of the table is rendered as a header.

pgwide

docbook (XSL/FO)

table, block image, horizontal labeled list

Specifies that the element should be rendered across the full text width of the page irrespective of the current indentation.

strong

xhtml11, html5, html4

labeled lists

Emboldens label text.

Appendix G: Diagnostics

The asciidoc(1) --verbose command-line option prints additional information to stderr: files processed, filters processed, warnings, system attribute evaluation.

A special attribute named trace enables the output of element-by-element diagnostic messages detailing output markup generation to stderr. The trace attribute can be set on the command-line or from within the document using Attribute Entries (the latter allows tracing to be confined to specific portions of the document).

  • Trace messages print the source file name and line number and the trace name followed by related markup.

  • trace names are normally the names of AsciiDoc elements (see the list below).

  • The trace message is only printed if the trace attribute value matches the start of a trace name. The trace attribute value can be any Python regular expression. If a trace value is not specified all trace messages will be printed (this can result in large amounts of output if applied to the whole document).

  • In the case of inline substitutions:

    • The text before and after the substitution is printed; the before text is preceded by a line containing <<< and the after text by a line containing >>>.

    • The subs trace value is an alias for all inline substitutions.

Trace names
<blockname> block close
<blockname> block open
<subs>
dropped line (a line containing an undefined attribute reference).
floating title
footer
header
list close
list entry close
list entry open
list item close
list item open
list label close
list label open
list open
macro block (a block macro)
name (man page NAME section)
paragraph
preamble close
preamble open
push blockname
pop blockname
section close
section open: level <level>
subs (all inline substitutions)
table

Where:

  • <level> is section level number 0…4.

  • <blockname> is a delimited block name: comment, sidebar, open, pass, listing, literal, quote, example.

  • <subs> is an inline substitution type: specialcharacters,quotes,specialwords, replacements, attributes,macros,callouts, replacements2, replacements3.

Command-line examples:

  1. Trace the entire document. 

    $ asciidoc -a trace mydoc.txt

  2. Trace messages whose names start with quotes or macros: 

    $ asciidoc -a 'trace=quotes|macros'  mydoc.txt

  3. Print the first line of each trace message: 

    $ asciidoc -a trace mydoc.txt 2>&1 | grep ^TRACE:

Attribute Entry examples:

  1. Begin printing all trace messages: 

    :trace:

  2. Print only matched trace messages: 

    :trace: quotes|macros

  3. Turn trace messages off: 

    :trace!:

Appendix H: Backend Attributes

This table contains a list of optional attributes that influence the generated outputs.

Name Backends Description

badges

xhtml11, html5

Link badges (XHTML 1.1 and CSS) in document footers. By default badges are omitted (badges is undefined).

Note The path names of images, icons and scripts are relative path names to the output document not the source document.

data-uri

xhtml11, html5

Embed images using the data: uri scheme.

css-signature

html5, xhtml11

Set a CSS signature for the document (sets the id attribute of the HTML body element). CSS signatures provide a mechanism that allows users to personalize the document appearance. The term CSS signature was coined by Eric Meyer.

disable-javascript

xhtml11, html5

If the disable-javascript attribute is defined the asciidoc.js JavaScript is not embedded or linked to the output document. By default AsciiDoc automatically embeds or links the asciidoc.js JavaScript to the output document. The script dynamically generates table of contents and footnotes.

docinfo, docinfo1, docinfo2

All backends

These three attributes control which document information files will be included in the the header of the output file:

docinfo

Include <filename>-docinfo.<ext>

docinfo1

Include docinfo.<ext>

docinfo2

Include docinfo.<ext> and <filename>-docinfo.<ext>

Where <filename> is the file name (sans extension) of the AsciiDoc input file and <ext> is .html for HTML outputs or .xml for DocBook outputs. If the input file is the standard input then the output file name is used. The following example will include the mydoc-docinfo.xml docinfo file in the DocBook mydoc.xml output file:

$ asciidoc -a docinfo -b docbook mydoc.txt

This next example will include docinfo.html and mydoc-docinfo.html docinfo files in the HTML output file:

$ asciidoc -a docinfo2 -b html4 mydoc.txt

encoding

html4, html5, xhtml11, docbook

Set the input and output document character set encoding. For example the --attribute encoding=ISO-8859-1 command-line option will set the character set encoding to ISO-8859-1.

  • The default encoding is UTF-8.

  • This attribute specifies the character set in the output document.

  • The encoding name must correspond to a Python codec name or alias.

  • The encoding attribute can be set using an AttributeEntry inside the document header. For example: 

    :encoding: ISO-8859-1

icons

xhtml11, html5

Link admonition paragraph and admonition block icon images and badge images. By default icons is undefined and text is used in place of icon images.

iconsdir

html4, html5, xhtml11, docbook

The name of the directory containing linked admonition icons, navigation icons and the callouts sub-directory (the callouts sub-directory contains callout number images). iconsdir defaults to ./images/icons.

imagesdir

html4, html5, xhtml11, docbook

If this attribute is defined it is prepended to the target image file name paths in inline and block image macros.

keywords, description, title

html4, html5, xhtml11

The keywords and description attributes set the correspondingly named HTML meta tag contents; the title attribute sets the HTML title tag contents. Their principle use is for SEO (Search Engine Optimisation). All three are optional, but if they are used they must appear in the document header (or on the command-line). If title is not specified the AsciiDoc document title is used.

linkcss

html5, xhtml11

Link CSS stylesheets and JavaScripts. By default linkcss is undefined in which case stylesheets and scripts are automatically embedded in the output document.

max-width

html5, xhtml11

Set the document maximum display width (sets the body element CSS max-width property).

numbered

html4, html5, xhtml11, docbook (XSL Stylesheets)

Adds section numbers to section titles. The docbook backend ignores numbered attribute entries after the document header.

plaintext

All backends

If this global attribute is defined all inline substitutions are suppressed and block indents are retained. This option is useful when dealing with large amounts of imported plain text.

quirks

xhtml11

Include the xhtml11-quirks.conf configuration file and xhtml11-quirks.css stylesheet to work around IE6 browser incompatibilities. This feature is deprecated and its use is discouraged — documents are still viewable in IE6 without it.

revremark

docbook

A short summary of changes in this document revision. Must be defined prior to the first document section. The document also needs to be dated to output this attribute.

scriptsdir

html5, xhtml11

The name of the directory containing linked JavaScripts. See HTML stylesheets and JavaScript locations.

sgml

docbook45

The --backend=docbook45 command-line option produces DocBook 4.5 XML. You can produce the older DocBook SGML format using the --attribute sgml command-line option.

stylesdir

html5, xhtml11

The name of the directory containing linked or embedded stylesheets. See HTML stylesheets and JavaScript locations.

stylesheet

html5, xhtml11

The file name of an optional additional CSS stylesheet.

theme

html5, xhtml11

Use alternative stylesheet (see Stylesheets).

toc

html5, xhtml11, docbook (XSL Stylesheets)

Adds a table of contents to the start of an article or book document. The toc attribute can be specified using the --attribute toc command-line option or a :toc: attribute entry in the document header. The toc attribute is defined by default when the docbook backend is used. To disable table of contents generation undefine the toc attribute by putting a :toc!: attribute entry in the document header or from the command-line with an --attribute toc! option.

xhtml11 and html5 backends

  • JavaScript needs to be enabled in your browser.

  • The following example generates a numbered table of contents using a JavaScript embedded in the mydoc.html output document: 

    $ asciidoc -a toc -a numbered mydoc.txt

toc2

html5, xhtml11

Adds a scrollable table of contents in the left hand margin of an article or book document. Use the max-width attribute to change the content width. In all other respects behaves the same as the toc attribute.

toc-placement

html5, xhtml11

When set to auto (the default value) asciidoc(1) will place the table of contents in the document header. When toc-placement is set to manual the TOC can be positioned anywhere in the document by placing the toc::[] block macro at the point you want the TOC to appear.

Note If you use toc-placement then you also have to define the toc attribute.

toc-title

html5, xhtml11

Sets the table of contents title (defaults to Table of Contents).

toclevels

html5, xhtml11

Sets the number of title levels (1..4) reported in the table of contents (see the toc attribute above). Defaults to 2 and must be used with the toc attribute. Example usage:

$ asciidoc -a toc -a toclevels=3 doc/asciidoc.txt

Appendix I: License

AsciiDoc is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 (GPLv2) as published by the Free Software Foundation.

AsciiDoc is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License version 2 for more details.

Copyright © 2002-2011 Stuart Rackham.

================================================ FILE: docs/asciidoc-userguide_files/Content.css ================================================ /* ShareMeNot is licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php Copyright (c) 2012 University of Washington Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* * Every property is !important to prevent any styles declared on the web page * from overriding ours. */ .sharemenotReplacementButton { border: none !important; cursor: pointer !important; height: auto !important; width: auto !important; } .sharemenotOriginalButton { border: none !important; height: 1.5em !important; } ================================================ FILE: docs/asciidoc-userguide_files/asciidoc.css ================================================ /* Shared CSS for AsciiDoc xhtml11 and html5 backends */ /* Default font. */ body { font-family: Georgia,serif; } /* Title font. */ h1, h2, h3, h4, h5, h6, div.title, caption.title, thead, p.table.header, #toctitle, #author, #revnumber, #revdate, #revremark, #footer { font-family: Arial,Helvetica,sans-serif; } body { margin: 1em 5% 1em 5%; } a { color: blue; text-decoration: underline; } a:visited { color: fuchsia; } em { font-style: italic; color: navy; } strong { font-weight: bold; color: #083194; } h1, h2, h3, h4, h5, h6 { color: #527bbd; margin-top: 1.2em; margin-bottom: 0.5em; line-height: 1.3; } h1, h2, h3 { border-bottom: 2px solid silver; } h2 { padding-top: 0.5em; } h3 { float: left; } h3 + * { clear: left; } h5 { font-size: 1.0em; } div.sectionbody { margin-left: 0; } hr { border: 1px solid silver; } p { margin-top: 0.5em; margin-bottom: 0.5em; } ul, ol, li > p { margin-top: 0; } ul > li { color: #aaa; } ul > li > * { color: black; } .monospaced, code, pre { font-family: "Courier New", Courier, monospace; font-size: inherit; color: navy; padding: 0; margin: 0; } #author { color: #527bbd; font-weight: bold; font-size: 1.1em; } #email { } #revnumber, #revdate, #revremark { } #footer { font-size: small; border-top: 2px solid silver; padding-top: 0.5em; margin-top: 4.0em; } #footer-text { float: left; padding-bottom: 0.5em; } #footer-badges { float: right; padding-bottom: 0.5em; } #preamble { margin-top: 1.5em; margin-bottom: 1.5em; } div.imageblock, div.exampleblock, div.verseblock, div.quoteblock, div.literalblock, div.listingblock, div.sidebarblock, div.admonitionblock { margin-top: 1.0em; margin-bottom: 1.5em; } div.admonitionblock { margin-top: 2.0em; margin-bottom: 2.0em; margin-right: 10%; color: #606060; } div.content { /* Block element content. */ padding: 0; } /* Block element titles. */ div.title, caption.title { color: #527bbd; font-weight: bold; text-align: left; margin-top: 1.0em; margin-bottom: 0.5em; } div.title + * { margin-top: 0; } td div.title:first-child { margin-top: 0.0em; } div.content div.title:first-child { margin-top: 0.0em; } div.content + div.title { margin-top: 0.0em; } div.sidebarblock > div.content { background: #ffffee; border: 1px solid #dddddd; border-left: 4px solid #f0f0f0; padding: 0.5em; } div.listingblock > div.content { border: 1px solid #dddddd; border-left: 5px solid #f0f0f0; background: #f8f8f8; padding: 0.5em; } div.quoteblock, div.verseblock { padding-left: 1.0em; margin-left: 1.0em; margin-right: 10%; border-left: 5px solid #f0f0f0; color: #888; } div.quoteblock > div.attribution { padding-top: 0.5em; text-align: right; } div.verseblock > pre.content { font-family: inherit; font-size: inherit; } div.verseblock > div.attribution { padding-top: 0.75em; text-align: left; } /* DEPRECATED: Pre version 8.2.7 verse style literal block. */ div.verseblock + div.attribution { text-align: left; } div.admonitionblock .icon { vertical-align: top; font-size: 1.1em; font-weight: bold; text-decoration: underline; color: #527bbd; padding-right: 0.5em; } div.admonitionblock td.content { padding-left: 0.5em; border-left: 3px solid #dddddd; } div.exampleblock > div.content { border-left: 3px solid #dddddd; padding-left: 0.5em; } div.imageblock div.content { padding-left: 0; } span.image img { border-style: none; } a.image:visited { color: white; } dl { margin-top: 0.8em; margin-bottom: 0.8em; } dt { margin-top: 0.5em; margin-bottom: 0; font-style: normal; color: navy; } dd > *:first-child { margin-top: 0.1em; } ul, ol { list-style-position: outside; } ol.arabic { list-style-type: decimal; } ol.loweralpha { list-style-type: lower-alpha; } ol.upperalpha { list-style-type: upper-alpha; } ol.lowerroman { list-style-type: lower-roman; } ol.upperroman { list-style-type: upper-roman; } div.compact ul, div.compact ol, div.compact p, div.compact p, div.compact div, div.compact div { margin-top: 0.1em; margin-bottom: 0.1em; } tfoot { font-weight: bold; } td > div.verse { white-space: pre; } div.hdlist { margin-top: 0.8em; margin-bottom: 0.8em; } div.hdlist tr { padding-bottom: 15px; } dt.hdlist1.strong, td.hdlist1.strong { font-weight: bold; } td.hdlist1 { vertical-align: top; font-style: normal; padding-right: 0.8em; color: navy; } td.hdlist2 { vertical-align: top; } div.hdlist.compact tr { margin: 0; padding-bottom: 0; } .comment { background: yellow; } .footnote, .footnoteref { font-size: 0.8em; } span.footnote, span.footnoteref { vertical-align: super; } #footnotes { margin: 20px 0 20px 0; padding: 7px 0 0 0; } #footnotes div.footnote { margin: 0 0 5px 0; } #footnotes hr { border: none; border-top: 1px solid silver; height: 1px; text-align: left; margin-left: 0; width: 20%; min-width: 100px; } div.colist td { padding-right: 0.5em; padding-bottom: 0.3em; vertical-align: top; } div.colist td img { margin-top: 0.3em; } @media print { #footer-badges { display: none; } } #toc { margin-bottom: 2.5em; } #toctitle { color: #527bbd; font-size: 1.1em; font-weight: bold; margin-top: 1.0em; margin-bottom: 0.1em; } div.toclevel0, div.toclevel1, div.toclevel2, div.toclevel3, div.toclevel4 { margin-top: 0; margin-bottom: 0; } div.toclevel2 { margin-left: 2em; font-size: 0.9em; } div.toclevel3 { margin-left: 4em; font-size: 0.9em; } div.toclevel4 { margin-left: 6em; font-size: 0.9em; } span.aqua { color: aqua; } span.black { color: black; } span.blue { color: blue; } span.fuchsia { color: fuchsia; } span.gray { color: gray; } span.green { color: green; } span.lime { color: lime; } span.maroon { color: maroon; } span.navy { color: navy; } span.olive { color: olive; } span.purple { color: purple; } span.red { color: red; } span.silver { color: silver; } span.teal { color: teal; } span.white { color: white; } span.yellow { color: yellow; } span.aqua-background { background: aqua; } span.black-background { background: black; } span.blue-background { background: blue; } span.fuchsia-background { background: fuchsia; } span.gray-background { background: gray; } span.green-background { background: green; } span.lime-background { background: lime; } span.maroon-background { background: maroon; } span.navy-background { background: navy; } span.olive-background { background: olive; } span.purple-background { background: purple; } span.red-background { background: red; } span.silver-background { background: silver; } span.teal-background { background: teal; } span.white-background { background: white; } span.yellow-background { background: yellow; } span.big { font-size: 2em; } span.small { font-size: 0.6em; } span.underline { text-decoration: underline; } span.overline { text-decoration: overline; } span.line-through { text-decoration: line-through; } div.unbreakable { page-break-inside: avoid; } /* * xhtml11 specific * * */ div.tableblock { margin-top: 1.0em; margin-bottom: 1.5em; } div.tableblock > table { border: 3px solid #527bbd; } thead, p.table.header { font-weight: bold; color: #527bbd; } p.table { margin-top: 0; } /* Because the table frame attribute is overriden by CSS in most browsers. */ div.tableblock > table[frame="void"] { border-style: none; } div.tableblock > table[frame="hsides"] { border-left-style: none; border-right-style: none; } div.tableblock > table[frame="vsides"] { border-top-style: none; border-bottom-style: none; } /* * html5 specific * * */ table.tableblock { margin-top: 1.0em; margin-bottom: 1.5em; } thead, p.tableblock.header { font-weight: bold; color: #527bbd; } p.tableblock { margin-top: 0; } table.tableblock { border-width: 3px; border-spacing: 0px; border-style: solid; border-color: #527bbd; border-collapse: collapse; } th.tableblock, td.tableblock { border-width: 1px; padding: 4px; border-style: solid; border-color: #527bbd; } table.tableblock.frame-topbot { border-left-style: hidden; border-right-style: hidden; } table.tableblock.frame-sides { border-top-style: hidden; border-bottom-style: hidden; } table.tableblock.frame-none { border-style: hidden; } th.tableblock.halign-left, td.tableblock.halign-left { text-align: left; } th.tableblock.halign-center, td.tableblock.halign-center { text-align: center; } th.tableblock.halign-right, td.tableblock.halign-right { text-align: right; } th.tableblock.valign-top, td.tableblock.valign-top { vertical-align: top; } th.tableblock.valign-middle, td.tableblock.valign-middle { vertical-align: middle; } th.tableblock.valign-bottom, td.tableblock.valign-bottom { vertical-align: bottom; } /* * manpage specific * * */ body.manpage h1 { padding-top: 0.5em; padding-bottom: 0.5em; border-top: 2px solid silver; border-bottom: 2px solid silver; } body.manpage h2 { border-style: none; } body.manpage div.sectionbody { margin-left: 3em; } @media print { body.manpage div#toc { display: none; } } ================================================ FILE: docs/asciidoc-userguide_files/asciidoc.js ================================================ var asciidoc = { // Namespace. ///////////////////////////////////////////////////////////////////// // Table Of Contents generator ///////////////////////////////////////////////////////////////////// /* Author: Mihai Bazon, September 2002 * http://students.infoiasi.ro/~mishoo * * Table Of Content generator * Version: 0.4 * * Feel free to use this script under the terms of the GNU General Public * License, as long as you do not remove or alter this notice. */ /* modified by Troy D. Hanson, September 2006. License: GPL */ /* modified by Stuart Rackham, 2006, 2009. License: GPL */ // toclevels = 1..4. toc: function (toclevels) { function getText(el) { var text = ""; for (var i = el.firstChild; i != null; i = i.nextSibling) { if (i.nodeType == 3 /* Node.TEXT_NODE */) // IE doesn't speak constants. text += i.data; else if (i.firstChild != null) text += getText(i); } return text; } function TocEntry(el, text, toclevel) { this.element = el; this.text = text; this.toclevel = toclevel; } function tocEntries(el, toclevels) { var result = new Array; var re = new RegExp('[hH]([1-'+(toclevels+1)+'])'); // Function that scans the DOM tree for header elements (the DOM2 // nodeIterator API would be a better technique but not supported by all // browsers). var iterate = function (el) { for (var i = el.firstChild; i != null; i = i.nextSibling) { if (i.nodeType == 1 /* Node.ELEMENT_NODE */) { var mo = re.exec(i.tagName); if (mo && (i.getAttribute("class") || i.getAttribute("className")) != "float") { result[result.length] = new TocEntry(i, getText(i), mo[1]-1); } iterate(i); } } } iterate(el); return result; } var toc = document.getElementById("toc"); if (!toc) { return; } // Delete existing TOC entries in case we're reloading the TOC. var tocEntriesToRemove = []; var i; for (i = 0; i < toc.childNodes.length; i++) { var entry = toc.childNodes[i]; if (entry.nodeName.toLowerCase() == 'div' && entry.getAttribute("class") && entry.getAttribute("class").match(/^toclevel/)) tocEntriesToRemove.push(entry); } for (i = 0; i < tocEntriesToRemove.length; i++) { toc.removeChild(tocEntriesToRemove[i]); } // Rebuild TOC entries. var entries = tocEntries(document.getElementById("content"), toclevels); for (var i = 0; i < entries.length; ++i) { var entry = entries[i]; if (entry.element.id == "") entry.element.id = "_toc_" + i; var a = document.createElement("a"); a.href = "#" + entry.element.id; a.appendChild(document.createTextNode(entry.text)); var div = document.createElement("div"); div.appendChild(a); div.className = "toclevel" + entry.toclevel; toc.appendChild(div); } if (entries.length == 0) toc.parentNode.removeChild(toc); }, ///////////////////////////////////////////////////////////////////// // Footnotes generator ///////////////////////////////////////////////////////////////////// /* Based on footnote generation code from: * http://www.brandspankingnew.net/archive/2005/07/format_footnote.html */ footnotes: function () { // Delete existing footnote entries in case we're reloading the footnodes. var i; var noteholder = document.getElementById("footnotes"); if (!noteholder) { return; } var entriesToRemove = []; for (i = 0; i < noteholder.childNodes.length; i++) { var entry = noteholder.childNodes[i]; if (entry.nodeName.toLowerCase() == 'div' && entry.getAttribute("class") == "footnote") entriesToRemove.push(entry); } for (i = 0; i < entriesToRemove.length; i++) { noteholder.removeChild(entriesToRemove[i]); } // Rebuild footnote entries. var cont = document.getElementById("content"); var spans = cont.getElementsByTagName("span"); var refs = {}; var n = 0; for (i=0; i" + n + "]"; spans[i].setAttribute("data-note", note); } noteholder.innerHTML += "
" + "" + n + ". " + note + "
"; var id =spans[i].getAttribute("id"); if (id != null) refs["#"+id] = n; } } if (n == 0) noteholder.parentNode.removeChild(noteholder); else { // Process footnoterefs. for (i=0; i" + n + "]"; } } } }, install: function(toclevels) { var timerId; function reinstall() { asciidoc.footnotes(); if (toclevels) { asciidoc.toc(toclevels); } } function reinstallAndRemoveTimer() { clearInterval(timerId); reinstall(); } timerId = setInterval(reinstall, 500); if (document.addEventListener) document.addEventListener("DOMContentLoaded", reinstallAndRemoveTimer, false); else window.onload = reinstallAndRemoveTimer; } } ================================================ FILE: docs/asciidoc-userguide_files/layout2.css ================================================ body { margin: 0; } #layout-menu-box { position: fixed; left: 0px; top: 0px; width: 160px; height: 100%; z-index: 1; background-color: #f4f4f4; } #layout-content-box { position: relative; margin-left: 160px; background-color: white; } h1 { margin-top: 0.5em; } #layout-banner { color: white; background-color: #73a0c5; font-family: Arial,Helvetica,sans-serif; text-align: left; padding: 0.8em 20px; } #layout-title { font-family: "Courier New", Courier, monospace; font-size: 3.5em; font-weight: bold; letter-spacing: 0.2em; margin: 0; } #layout-description { font-size: 1.2em; letter-spacing: 0.1em; } #layout-menu { height: 100%; border-right: 3px solid #eeeeee; padding-top: 0.8em; padding-left: 15px; padding-right: 0.8em; font-size: 1.0em; font-family: Arial,Helvetica,sans-serif; font-weight: bold; } #layout-menu a { line-height: 2em; margin-left: 0.5em; } #layout-menu a:link, #layout-menu a:visited, #layout-menu a:hover { color: #527bbd; text-decoration: none; } #layout-menu a:hover { color: navy; text-decoration: none; } #layout-menu #page-source { border-top: 2px solid silver; margin-top: 0.2em; } #layout-content { padding-top: 0.2em; padding-left: 1.0em; padding-right: 0.4em; } @media print { #layout-banner-box { display: none; } #layout-menu-box { display: none; } #layout-content-box { margin-top: 0; margin-left: 0; } } ================================================ FILE: docs/example_book.txt ================================================ Book Title Goes Here ==================== Author's Name v1.0, 2003-12 :doctype: book [dedication] Example Dedication ------------------ Optional dedication. This document is an AsciiDoc book skeleton containing briefly annotated example elements plus a couple of example index entries and footnotes. Books are normally used to generate DocBook markup and the titles of the preface, appendix, bibliography, glossary and index sections are significant ('specialsections'). [preface] Example Preface --------------- Optional preface. Preface Sub-section ~~~~~~~~~~~~~~~~~~~ Preface sub-section body. The First Chapter ----------------- Chapters can contain sub-sections nested up to three deep. footnote:[An example footnote.] indexterm:[Example index entry] Chapters can have their own bibliography, glossary and index. And now for something completely different: ((monkeys)), lions and tigers (Bengal and Siberian) using the alternative syntax index entries. (((Big cats,Lions))) (((Big cats,Tigers,Bengal Tiger))) (((Big cats,Tigers,Siberian Tiger))) Note that multi-entry terms generate separate index entries. Here are a couple of image examples: an image:images/smallnew.png[] example inline image followed by an example block image: .Tiger block image image::images/tiger.png[Tiger image] Followed by an example table: .An example table [width="60%",options="header"] |============================================== | Option | Description | -a 'USER GROUP' | Add 'USER' to 'GROUP'. | -R 'GROUP' | Disables access to 'GROUP'. |============================================== .An example example =============================================== Lorum ipum... =============================================== [[X1]] Sub-section with Anchor ~~~~~~~~~~~~~~~~~~~~~~~ Sub-section at level 2. Chapter Sub-section ^^^^^^^^^^^^^^^^^^^ Sub-section at level 3. Chapter Sub-section +++++++++++++++++++ Sub-section at level 4. This is the maximum sub-section depth supported by the distributed AsciiDoc configuration. footnote:[A second example footnote.] The Second Chapter ------------------ An example link to anchor at start of the <>. indexterm:[Second example index entry] An example link to a bibliography entry <>. The Third Chapter ----------------- Book chapters are at level 1 and can contain sub-sections. :numbered!: [appendix] Example Appendix ---------------- One or more optional appendixes go here at section level 1. Appendix Sub-section ~~~~~~~~~~~~~~~~~~~ Sub-section body. [bibliography] Example Bibliography -------------------- The bibliography list is a style of AsciiDoc bulleted list. [bibliography] .Books - [[[taoup]]] Eric Steven Raymond. 'The Art of Unix Programming'. Addison-Wesley. ISBN 0-13-142901-9. - [[[walsh-muellner]]] Norman Walsh & Leonard Muellner. 'DocBook - The Definitive Guide'. O'Reilly & Associates. 1999. ISBN 1-56592-580-7. [bibliography] .Articles - [[[abc2003]]] Gall Anonim. 'An article', Whatever. 2003. [glossary] Example Glossary ---------------- Glossaries are optional. Glossaries entries are an example of a style of AsciiDoc labeled lists. [glossary] A glossary term:: The corresponding (indented) definition. A second glossary term:: The corresponding (indented) definition. [colophon] Example Colophon ---------------- Text at the end of a book describing facts about its production. [index] Example Index ------------- //////////////////////////////////////////////////////////////// The index is normally left completely empty, it's contents being generated automatically by the DocBook toolchain. //////////////////////////////////////////////////////////////// ================================================ FILE: epilogue.asciidoc ================================================ [appendix] [role="afterword"] == Obey the Testing Goat! Let's get back to the Testing Goat. "Groan", I hear you say—"Harry, the Testing Goat stopped being funny about 17 chapters ago". Bear with me; I'm going to use it to make a serious point. === Testing Is Hard ((("Testing Goat", "philosophy of"))) I think the reason the phrase "Obey the Testing Goat" first grabbed me when I saw it was that it spoke to the fact that testing is hard--not hard to do in and of itself, but hard to _stick to_, and hard to keep doing. It always feels easier to cut corners and skip a few tests. And it's doubly hard psychologically because the payoff is so disconnected from the point at which you put in the effort. A test you spend time writing now doesn't reward you immediately; it only helps much later--perhaps months later when it saves you from introducing a bug while refactoring, or catches a regression when you upgrade a dependency. Or, perhaps it pays you back in a way that's hard to measure, by encouraging you to write better-designed code, but you convince yourself you could have written it just as elegantly without tests. I myself started slipping when I was writing the https://github.com/hjwp/Book-TDD-Web-Dev-Python/tree/master/tests[test framework for this book]. Being quite a complex beast, it has tests of its own, but I cut several corners. So, coverage isn't perfect, and I now regret it because it's turned out quite unwieldy and ugly (go on; I've open sourced it now, so you can all point and laugh). [role="pagebreak-before less_space"] ==== Keep Your CI Builds Green ((("continuous integration (CI)", "tips"))) Another area that takes real hard work is continuous integration. You saw in <> that strange and unpredictable bugs sometimes occur in CI. When you're looking at these and thinking "it works fine on my machine", there's a strong temptation to just ignore them...but, if you're not careful, you start to tolerate a failing test suite in CI, and pretty soon your CI build is actually useless, and it feels like too much work to get it going again. Don't fall into that trap. Persist, and you'll find the reason that your test is failing, and you'll find a way to lock it down and make it deterministic, and green, again. ==== Take Pride in Your Tests, as You Do in Your Code One of the things that helps is((("tests", "taking pride in as in code"))) to stop thinking of your tests as being an incidental add-on to the "real" code, and to start thinking of them as being a part of the finished product that you're building--a part that should be just as finely polished and just as aesthetically pleasing, and a part you can be justly proud of delivering... So, do it because the Testing Goat says so. Do it because you know the payoff will be worth it, even if it's not immediate. Do it out of a sense of duty, or professionalism, or perfectionism, or sheer bloody-mindedness. Do it because it's a good thing to practice. And, eventually, do it because it makes software development more fun. //something about pairing? ==== Remember to Tip the Bar Staff This book wouldn't have been possible without the backing of my publisher, the wonderful O'Reilly Media. If you're reading the free edition online, I hope you'll consider buying a real copy...if you don't need one for yourself, then maybe as a gift for a friend? // TODO: add amazon link back in above === Don't Be a Stranger! I hope you enjoyed the book. Do get in touch and tell me what you thought! Harry * https://fosstodon.org/@hjwp * obeythetestinggoat@gmail.com ================================================ FILE: index.txt ================================================ Index A acceptance test (see functional tests/testing (FT)) acceptance tests, 397 aesthetics (see layout and style) agile movement in software development, 79 Ajax, 249, 269 ALLOWED_HOSTS, 151 Anderson, Ross, 51 Ansible, 166, 423–426 architectural solutions to test problems, 402 assertion messages, 264 AssertionError, 14, 44, 55 assertRegex, 83 assertTemplateUsed, 88 assertTrue function, 44 asynchronous JavaScript, 272–275 authentication backend, 285–293 customising, 245–247, 277 in Django, 282 login view, 281–284 minimum custom user model, 295–299 mocking (see mocks/mocking) Mozilla Persona, 242 pre-authentication, 303–306 testing logout, 300 testing view, 278 tests as documentation, 297 automation, in deployment, 132, 157–166 (see also deployment) automation, in provisioning, 166 B Bash, 141 Behavior-Driven Development (BDD) tools, 428 Bernhardt, Gary, 399, 404 best practices in testing, 397 Big Design Up Front, 79 black box test (see functional tests/testing (FT)) Boolean comparisons, 290 Bootstrap, 116–125 jumbotron, 123 large inputs, 124 rows and columns in, 120 table styling, 124 boundaries, 403 browsers, 428 browsers, headless, 372 C caching, 429 capture group, 101 CI server (see continuous integration (CI)) class-based generic views, 413–421 class-based views, 413 clean architecture, 404 code smell, 193, 302 collectstatic, 126–128 comments, 13, 84 commits, 16, 22, 27, 108 configuration management tools, 167 (see also Fabric) context managers, 177 continuous integration (CI), 365–385, cdvii adding required plugins, 368 best practices, 385 configuring Jenkins, 367 debugging with screenshots, 374–378 first build, 371 installing Jenkins, 365 JavaScript tests, 381–384 project setup, 369 Selenium race conditions, 378–381 for staging server test automation, 384 virtual display setup, 372–374 contracts, implicit, 355 cookies, 282, 304 Cross-Site Request Forgery (CSRF) error, 51 CSS (Cascading Style Sheets) framework, 114, 116 (see also Bootstrap) where Bootstrap won’t work, 124 cutting corners, cdvii D data migrations, 432–435 database deployment issues, 132 database location, 141 De-spiking, 251, 285–293 debugging, 19, 50, 249 Ajax, 249 Django debug screen, 146 improving error messages, 55 in continuous integration, 374–378 in JavaScript, 262 staging for, 306–310 switching DEBUG to false, 151 screenshots, for debugging, 374–378 dependencies, and deployment, 132 deployment, 411 adjusting database location, 141 automating, 153–155, 157–166 dependencies and, 132 deploying to live, 163 further reading on, 166 key points, 155 to live, 225 migrate, 147 Nginx, 144–147 overview, 153 production-ready, 148–152 vs. provisioning, 140 sample script, 158–161 saving progress, 156 staging, 225, 431 virtualenvs, 142–144 deployment testing, 131–156 danger areas, 132 domain name for, 135 manual provisioning for hosting, 136–140 overview, 133 design (see layout and style) Django, 4 admin site, 428 apps in, 20 authentication in, 245–247, 282 class-based views, 413–421 (see also class-based views) collectstatic, 126–128 custom user model, 295–299 debugging screen, 146, 151 field types, 61 foreign key relationship, 97 forms in (see forms) FormView, 414functional tests (FT) in (see functional tests/ testing (FT)) and Gunicorn, 148 LiveServerTestCase, 75 management commands, 311–314, 320 migrations, 60–62, 69–71, 225 model adjustment in, 95 model-layer validation, 175–187 Model-View-Controller (MVC), 22 notifications, 427 Object-Relational Mapper (ORM), 58–62 POST requests (see POST requests) as PythonAnywhere app, 410 running, 6 static files in, 121 static live server case, 122 template inheritance, 118–119 templates, 67–68, 88 test class in, 91 test client, 86, 91 test fixtures, 304 unit testing in, 21 URLs in, 22–27, 86, 92, 94, 100, 104, 106 validation quirk, 178 view functions in, 22, 87, 92, 103–106, 326 and virtualenvs, 142–144 Django-BrowserID, 243 documentation, tests as, 297 domain configuration, 139 domain names, 135 Don’t Test Constants rule, 38 double-loop TDD, 45, 323 DRY (don’t repeat yourself), 57, 396 duplicates, eliminating, 56, 211–221 E encryption, 430 end-to-end test (see functional tests/testing (FT)) error messages, 429 error pages, 428 exception handling, 293 expected failure, 14, 17 explicit waits, 254 exploratory coding, 195, 242 (see also spiking) F Fabric, 166, 314, 426 configuration, 163 installing, 157 sample deployment script, 158–161 Fake XMLHttpRequest, 269 Fixtures Div, 231–233 foreign key relationship, 97 forms advanced, 211–223 autogeneration, 195 customising form field input, 194 experimenting with, 194 find and replace in, 201 ModelForm, 195 save methods, 208 simple, 193–210 thin views, 210 tips for, 210 using in views, 198–207 validation testing and customising, 196 Fuctional Core, Imperative Shell architecture, 404 functional tests/testing (FT), 5, 397 automation of (see continuous integration (CI)) blank items, 169–175 cleanup, 75–78, 94, 387 de-duplication, 320 defining, 11 for de-spiking, 251 and developer stupidity, 213 for duplicate items, 211–221 for evaluating third-party systems, 253 isolation in, 75–78, 109 in JavaScript, 234–236 for layout and style, 113–116, 149, 173 multiple users, 387, 393–396 pros and cons, 363 in provisioning, 139 running unit tests only, 78 safeguards with, 317 splitting, 171 for staging sites, 132, 133 unittest model, 11–17 vs. unit tests, 20, 303 in views, 223 Index | 441G generator expression, 37 GET requests, 198, 205 get_user, 292, 293 Git repository setup, 7–10 reset --hard, 116 tags, 166, 226 global variables, 230 greedy regular expressions, 104 Gunicorn, 148–155, 165, 307, 425 H headless browsers, 372 helper functions/methods, 57, 172, 175, 206, 228, 350, 390–393 hexagonal architecture, 404 hosting options, 136 hosting, manual provisioning, 136–140 I Idempotency, 167 implicit waits, 16 in-memory model objects, 352 integrated tests, 351–363, 403 vs. integration test, 342 vs. isolated tests, 362, 397 pros and cons, 363 vs. unit tests, 59 integration tests, 342, 397 integrity errors, 217 isolated tests, 337, 403 (see also test isolation) vs. integrated tests, 362, 397 problems with, 400 pros and cons, 363 J JavaScript, 227 de-spiking in, 251 debug console, 262 functional test (FT) building in, 234–236 jQuery and Fixtures Div, 231–233 linters, 230 MVC frameworks, 429 onload boilerplate and namespacing, 236 QUnit, 229 442 | Index running tests in continuous integration, 381–384 spiking with, 242–256 (see also spiking) in TDD Cycle, 236 test runner setup, 228 testing notes, 237 Jenkins Security, 365–384 (see also continuous integration (CI)) adding required plugins, 368 configuring, 367 installing, 365 jQuery, 231–233, 236, 237 JSON fixtures, 304, 320 jumbotron, 123 L large inputs, 124 layout and style, 113–128 Bootstrap for (see Bootstrap) functional tests (FT) for, 173 large inputs, 124 overview, 128 rows and columns, 120 static files, 121, 126–128 table styling, 124 using a CSS framework for, 116 (see also Bootstrap) using our own CSS in, 124 what to functionally test for, 113 list comprehension, 37 LiveServerTestCase, 75 log messages, 320 logging, 307, 320 logging configuration, 318–320 M manage, 6 Meta, 196 meta-comments, 84 migrate, 147 migrations, 60–62, 69–71, 225, 226 (see also data migrations) database, 431–435 deleting, 97 testing, 431–435 minimum viable application, 11–14, 79 mocking library, 265MockMyID, 252 mocks/mocking in Boolean comparisons, 290 callbacks, 272–275 checking call arguments, 268 Django ORM, 292, 302 implicit contracts, 355 in JavaScript, 241, 258–275 initialize function test, 259–264 Internet requests, 285–295 for isolation, 338–341 mock library, 302 Mock side_effects, 339 namespacing, 258 in Outside-In TDD, 331 in Python, 278–284 risks, 354 sinon.js, 265 testing Django login, 284 model adjustments, 95–99 model-layer validation, 175–187 changes to test, 216 enforcing, 186 errors in View, 178–182 integrity errors, 217 POST requests, 183–187 preventing duplicates, 212 refactoring, 175, 184–186 unit testing, 177–178 at views level, 218 Model-View-Controller (MVC), 22, 429 ModelForm, 195 Mozilla Persona, 242 MVC frameworks, 22, 429 N namespacing, 236, 258 Nginx, 138, 144–147, 149, 165, 424 nonroot user creation, 137 notifications, 427 O ORM (Object-Relational Mapper), 58–62 ORM code, 347–350, 363 Outside-In TDD, 323–335 advantages, 323 controller layer, 326 defined, 335 vs. Inside-Out, 323 model layer, 330–333 pitfalls, 335 presentation layer, 325 template hierarchy, 327–329 views layer, 326–330, 333 P PaaS (Platform-as-a-Service), 136 Page pattern, 390–393, 396 parameters, capture group, 101 patch decorator, 279, 290, 302 patching, 287 payment systems, testing for, 253 performance testing, 429 Persona, 242, 252, 308–310, 429 PhantomJS, 381–384, 428 Platform-as-a-Service (PaaS), 136 POST requests, 203 processing, 52, 183–187 redirect after, 65 saving to database, 62–65 sending, 49–52, 90 Postgres, 427 private key authentication, 137 programming by wishful thinking, 328, 335 (see also Outside-In TDD) property Decorator, 334 provisioning, 136–140 with Ansible, 423–426 automation in, 166 functional tests (FT) in, 139 overview, 153 vs. deployment, 140 pure unit tests (see isolated tests) py.test, 430 Python adding to Jenkins, 369 PythonAnywhere, 136, 409 Q QuerySet, 59, 214–216 QUnit, 229, 237, 264, 269 R race conditions, 374, 389 Red, Green, Refactor, 56, 87, 170 Index | 443redirects, 65, 188 redundant code, 359 refactoring at application level, 183–186 Red, Green, Refactor, 56, 87, 170 removing hard-coded URLs, 187 and test isolation, 341, 362 tips, 190 unit tests, 175 with templates, 38–42 Refactoring Cat, 42, 109 relative import, 161, 173 render to string, 54 REST (Representational Site Transfer), 80 S screenshots, 411 scripts, automated, 132 secret key, 160 Security Engineering (Anderson), 51 security tests, 429 sed (stream editor), 165 Selenium, 4 and JavaScript, 237 best practices, 385 in continuous integration, 378–381 in continuous integration, 372 race conditions, 389 race conditions in, 378–381 upgrading, 84 for user interaction testing, 35–38 wait patterns, 16, 254, 387, 389 waits in, 379–381, 385 server configuration, 155 server options, 137 servers, 136–140 (see also staging server) session key, 304 sessions, 282 Shining Panda, 369 sinon.js, 265, 269, 272 skips, 170 spiking, 242–256, 275 browser-ID protocol, 244 de-spiking, 251 frontend and JavaScript code, 243 logging, 250 server-side authentication, 245–247 with JavaScript, 242 444 | Index SQLite, 427 staging server creating sessions, 311 debugging in, 306–310 pre-creating a session, 303–306 test automation with CI, 384 test database on, 311–317 staging sites, 132, 133, 135 static directories, 126–128 static files, 114, 121, 132, 149 static folder, site-wide, 256 static live server case, 122 string representation, 215 string substitutions, 101 style (see layout and style) superlists, 6, 8, 20, 108 superusers, 70 system boundaries, 403 system tests, 397 T table styling, 124 template inheritance, 118–119 template inheritance hierarchy, 327 template tag, 51 templates for refactoring, 38–42 Python variables in, 53–56 rendering items in, 67–68 separate, 88 test fixtures, 304, 320 test isolation, 109, 337–363 cleanup after, 359–361 collaborators, 343–345 complexity in, 362 forms layer, 347–350 full isolation, 342 interactions between layers, 355 isolated vs. integrated tests, 362 mocks/mocking for, 338–341 models layer, 351–353 ORM code, 347–350, 363 refactoring in, 341, 362 views layer, 337, 338–346, 353 test methods, 15 test organisation, 190 test skips, 170 test types, 363, 397test-driven development (TDD) advanced considerations in, 397–405 double-loop, 45, 323 further reading on, 405 Inside-Out, 323 iterating towards new design, 84 Java testing in, 236 new design implementation with, 81–84 Outside-In, 323–335 (see also Outside-In TDD) process flowchart, 81 process recap, 45–47 trivialities of, 33–35 TestCase, in Django, 21 testing best practices, 397 Testing Goat, 3, 108, 109, cdvii tests, as documentation, 297 thin views, 210 time.sleep, 50 tracebacks, 24, 54 triangulation, 56 U Ubuntu, 137 unit tests architectural solutions for, 402 context manager, 177 desired features of, 401 in Django, 21 for simple home page, 19–31 vs. functional tests, 303 vs. functional tests (FT), 20 vs. integrated tests, 59 pros and cons of, 398–401 refactoring, 175 unit-test/code cycle, 29–31 unittest, 134 unittest model, 11–17 Unix sockets, 150 Upstart, 151 URLs capturing parameters in, 101 distinct, 100 in Django, 22–27, 86, 92, 94, 100, 104, 106 pointing forms to, 94 urls.py, 25–27 user authentication, 241 user creation, 291 user input, saving, 49–72 user interaction testing, 35–38 user stories, 17, 170 V Vagrant, 426 validation, 169 (see also functional tests/testing (FT)) blank items, 169–175 model-layer, 175–187 (see also model-layer validation) VCS (version control system), 7–10 view functions, in Django, 22, 87, 92, 103–106 views layer, 337, 338–346, 353 model validation errors in, 178–182 views, what to test in, 223 virtual displays, 372 Virtualbox, 426 virtualenvs, 132, 142–144 W waits, 16, 254, 379–381, 385, 387, 389 warnings, 15 watch function, 265 websockets, 429 widgets, 194, 196 X Xvfb, 369, 373, 410 Y YAGNI, 80 Index | 445 ================================================ FILE: ix.html ================================================
================================================ FILE: load_toc.js ================================================ var httpRequest = new XMLHttpRequest(); httpRequest.onreadystatechange = function() { if (httpRequest.readyState === XMLHttpRequest.DONE) { if (httpRequest.status === 200) { document.getElementById('header').innerHTML += httpRequest.responseText; var subheaders = document.getElementsByClassName('sectlevel2'); var section; for (var i=0; i 5 pages 2: A simple example (?) ----------------------- eg Roman numerals, as per "Dive Into Python.". Slightly boring topic, everyone uses it, but it will do for now. * basic test - convert I * I-III * V + X * VI, VII, VIII * IV and IX * ... * Start simple, include exceptions. Aim to show: * unittest, how to run tests, possibly setUp, tearDown * red / green / refactor: the test/code cycle * making the minimal change * show how design grows organically, but stays neat * the psychological effect * as per Beck. "Am I saying you should always code like this? No. I'm saying you should always *be able to*. In DIP, this is pp 183-205, so 22 pages. I'd aim for half that. --> 10 pages (maybe split this into multiple chapters?) ===> total for part 1: 15 pages =================================================== PART 2 - Using TDD to build a basic web application =================================================== 3: Our first functional test with Selenium ------------------------------------------ * Briefly discuss difference between functional testing (AKA acceptance testing, integration testing, whatever) and unit testing * Write first test - Introduce Selenium, `setUp`, `tearDown` * Demonstrate we can get it to open a web browser, and navigate to a web page eg - google.com currently maybe 228 lines, 1200 words. --> 5 pages 4: Getting Django set-up and running ------------------------------------ * Change our test to look for the test server * Switch to Django LiveServerTestCase. Explain * Get the first test running and failing for a sensible reason * Create django project `django-admin.py startproject` * It worked! --> 3 pages 5: A static front page ---------------------- * Look for "Welcome to the Forums", or similar * `urls.py`, `direct_to_template` ? --> 3 pages 6: Super-users and the Django admin site ---------------------------------------- * Extend FT to try and log in * Explain the admin site * Database setup, `settings.py`, `syncdb`, `admin.py` * `runserver` to show login code * Explain difference between test database and real database * Fixtures --> 7 pages 7: First unit tests and Database model -------------------------------------- * Distinction between unit tests and functional tests * Extend FT to try and create a new topic * new app * `models.py` * test/code cycle --> 7 pages 8: Testing a view ----------------- * urls.py again * Test view as a function * assert on string contents --> 4 pages 9: Django's template system ---------------------------- * Introduce template syntax * Keep testing as a function * The, introduce the Django Test Client --> 6 pages 10: Reflections: what to test, what not to test ----------------------------------------------- * "Don't test constants" * Test logic * Tests for simple stuff should be simple, so not much effort --> 3 pages 11: Simple Forms ---------------- * Manually coded HTML * Refactor test classes --> 5 pages 12: User Authentication ----------------------- * Sign up, login/logout * Email? --> 5 pages 13: More advanced forms ----------------------- * Use Django Forms classes --> 6 pages 14: On Refactoring ------------------ * Martin Fowler * Tests critical * Methodical process - explain step by step --> 4 pages 15: Pagination -------------- * Extend various old unit tests and FTs --> 3 pages ===> total for part 2: 60 pages ====================================================== PART 3: More advanced testing for a more advanced site ====================================================== 15: Notifications ------------------------------ * Django Notifications, for post edits --> 5 pages 16: Adding style with MarkDown ------------------------------ * Using an external library --> 5 pages 17: Switching to OAuth: Mocking ------------------------------- * "Don't store passwords" * Discuss challenges of external dependencies --> 7 pages 18: Getting Dynamic: Testing Javascript part 1 ---------------------------------------------- * Simple input validation * Choose JS unit testing framework (probably Qunit, or YUI) --> 6 pages 19: Testing Javascript part 2 - Ajax ------------------------------------ * Dynamic previews of post input --> 5 pages 20: Getting pretty: Bootstrap ----------------------------- * Bring in nicer UI elements --> 4 pages 21: Getting pretty: Gravatar ---------------------------- * pictures for users --> 4 pages 22: The bottomless web page --------------------------- * More javascript bells and whistles --> 3 pages ===> total for part 3: 39 pages ============================== PART 4: Getting seriously sexy ============================== 24: Switching to a proper Database: PostgreSQL ---------------------------------------------- * show how Django makes this easy --> 10 pages 21: Websockets and Async on the server-side ------------------------------------------- * we want dynamic notifications of when new posts appear on a thread we're looking at * Need to spin up, Tornado/Twisted/Gevent as well as Django LiveServerTestCase * FT opens multiple browser tabs in parallel * Big change! --> 20 pages 22: Continuous Integration -------------------------- * Need to build 3 server types * Jenkins (or maybe buildbot) * Need to adapt Fts, maybe rely less on LiveServerTestCase --> 15 pages 23: Caching for screamingly fast performance -------------------------------------------- * unit testing `memcached` * Functionally testing performance * Apache `ab` testing --> 15 pages ===> total for part 4: 60 pages ================================================ FILE: misc/chapters_v2.rst ================================================ =================================================== PART 1 - An introduction to Test-Driven Development =================================================== 1: What is TDD? --------------- * Some evangelism * XP and Kent Beck, Agile methods * TDD versus testing afterwards, * Outline methodology current tutorial : 800 words, no illustrations. this shouldn't be more than 3-5 pages really --> 5 pages 2: A simple example (?) ----------------------- eg Roman numerals, as per "Dive Into Python.". Slightly boring topic, everyone uses it, but it will do for now. * basic test - convert I * I-III * V + X * VI, VII, VIII * IV and IX * ... * Start simple, include exceptions. Aim to show: * unittest, how to run tests, possibly setUp, tearDown * red / green / refactor: the test/code cycle * making the minimal change * show how design grows organically, but stays neat * the psychological effect * as per Beck. "Am I saying you should always code like this? No. I'm saying you should always *be able to*. In DIP, this is pp 183-205, so 22 pages. I'd aim for half that. --> 10 pages (maybe split this into multiple chapters?) ===> total for part 1: 15 pages =================================================== PART 2 - Using TDD to build a basic web application =================================================== 3: Our first functional test with Selenium ------------------------------------------ * Briefly discuss difference between functional testing (AKA acceptance testing, integration testing, whatever) and unit testing * Write first test - Introduce Selenium, `setUp`, `tearDown` * Demonstrate we can get it to open a web browser, and navigate to a web page eg - google.com currently maybe 228 lines, 1200 words. --> 5 pages 4: Getting Django set-up and running ------------------------------------ * Change our test to look for the test server * Switch to Django LiveServerTestCase. Explain * Get the first test running and failing for a sensible reason * Create django project `django-admin.py startproject` * It worked! --> 3 pages 5: A static front page ---------------------- * Look for "Welcome to the Forums", or similar * `urls.py`, `direct_to_template` ? --> 3 pages 6: Super-users and the Django admin site ---------------------------------------- * Extend FT to try and log in * Explain the admin site * Database setup, `settings.py`, `syncdb`, `admin.py` * `runserver` to show login code * Explain difference between test database and real database * Fixtures --> 7 pages 7: First unit tests and Database model -------------------------------------- * Distinction between unit tests and functional tests * Extend FT to try and create a new topic * new app * `models.py` * test/code cycle --> 7 pages 8: Testing a view ----------------- * urls.py again * Test view as a function * assert on string contents --> 4 pages 9: Django's template system ---------------------------- * Introduce template syntax * Keep testing as a function * The, introduce the Django Test Client --> 6 pages 10: Reflections: what to test, what not to test ----------------------------------------------- * "Don't test constants" * Test logic * Tests for simple stuff should be simple, so not much effort --> 3 pages 11: Simple Forms ---------------- * Manually coded HTML * Refactor test classes --> 5 pages 12: User Authentication ----------------------- * Sign up, login/logout * Email? --> 5 pages 13: More advanced forms ----------------------- * Use Django Forms classes --> 6 pages 14: On Refactoring ------------------ * Martin Fowler * Tests critical * Methodical process - explain step by step --> 4 pages 15: Pagination -------------- * Extend various old unit tests and FTs --> 3 pages ===> total for part 2: 60 pages ====================================================== PART 3: More advanced testing for a more advanced site ====================================================== 15: Notifications ------------------------------ * Django Notifications, for post edits --> 5 pages 16: Adding style with MarkDown ------------------------------ * Using an external library --> 5 pages 17: Switching to OAuth: Mocking ------------------------------- * "Don't store passwords" * Discuss challenges of external dependencies --> 7 pages 18: Getting Dynamic: Testing Javascript part 1 ---------------------------------------------- * Simple input validation * Choose JS unit testing framework (probably Qunit, or YUI) --> 6 pages 19: Testing Javascript part 2 - Ajax ------------------------------------ * Dynamic previews of post input --> 5 pages 20: Getting pretty: Bootstrap ----------------------------- * Bring in nicer UI elements --> 4 pages 21: Getting pretty: Gravatar ---------------------------- * pictures for users --> 4 pages 22: The bottomless web page --------------------------- * More javascript bells and whistles --> 3 pages ===> total for part 3: 39 pages ============================== PART 4: Getting seriously sexy ============================== 23: Switching Databases 1: PostgreSQL ---------------------------------------------- * show how Django makes this easy --> 10 pages 25: Websockets and Async on the server-side ------------------------------------------- * we want dynamic notifications of when new posts appear on a thread we're looking at * Need to spin up, Tornado/Twisted/Gevent as well as Django LiveServerTestCase * FT opens multiple browser tabs in parallel * Big change! --> 20 pages 24: Switching Databases 2: NoSQL and MongoDB ---------------------------------------------- * obligatory discussion of NoSQL and MongoDB * descrine installation, particularities of testing --> 10 pages 26: Continuous Integration -------------------------- * Need to build 3 server types * Jenkins (or maybe buildbot) * Need to adapt Fts, maybe rely less on LiveServerTestCase --> 15 pages 27: Caching for screamingly fast performance -------------------------------------------- * unit testing `memcached` * Functionally testing performance * Apache `ab` testing --> 15 pages ===> total for part 4: 60 pages ================================================ FILE: misc/chimera_comments_scraper.py ================================================ from __future__ import print_function from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions from selenium.webdriver.support.ui import WebDriverWait from selenium.common.exceptions import TimeoutException import re URLS = [ 'http://chimera.labs.oreilly.com/books/1234000000754/pr01.html', 'http://chimera.labs.oreilly.com/books/1234000000754/pr02.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch01.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch02.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch03.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch04.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch05.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch06.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch07.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch08.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch09.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch10.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch11.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch12.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch13.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch14.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch15.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch16.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch17.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch18.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch19.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch20.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch21.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ch22.html', 'http://chimera.labs.oreilly.com/books/1234000000754/pr03.html', 'http://chimera.labs.oreilly.com/books/1234000000754/apa.html', 'http://chimera.labs.oreilly.com/books/1234000000754/apb.html', 'http://chimera.labs.oreilly.com/books/1234000000754/apc.html', 'http://chimera.labs.oreilly.com/books/1234000000754/apd.html', 'http://chimera.labs.oreilly.com/books/1234000000754/ape.html', ] METADATA_PARSER = re.compile(r'Comment by (.+) (\d+) (.+ ago)') browser = webdriver.Firefox() wait = WebDriverWait(browser, 3) try: for url in URLS: page = url.partition('1234000000754/')[2] browser.get(url) browser.find_element_by_css_selector('#comments-link a').click() try: wait.until(expected_conditions.presence_of_element_located( (By.CLASS_NAME, 'comment') )) except TimeoutException: print("No comments on page %s" % (url,)) elements = browser.find_elements_by_css_selector('.comment') for element in elements: metadata = element.find_element_by_css_selector('.comment-body-top').text.strip() # print(repr(metadata)) parsed_metadata = METADATA_PARSER.search(metadata).groups() # print(parsed_metadata) by = parsed_metadata[0] date = parsed_metadata[1] + parsed_metadata[2] # if 'months' not in date and 'year' not in date: if 'year' not in date: comment = element.find_element_by_css_selector('.comment-body-bottom').text print('%s\t%s\t%s\t%s' % (page, by, date, comment)) finally: browser.quit() ================================================ FILE: misc/get_stats.py ================================================ #!/usr/bin/env python3 from collections import namedtuple import csv from datetime import datetime import os import re import subprocess Commit = namedtuple('Commit', ['hash', 'subject', 'date']) WordCount = namedtuple('WordCount', ['filename', 'lines', 'words']) FileWordCount = namedtuple('FileWordCount', ['date', 'subject', 'hash', 'lines', 'words']) BOOK_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) def get_log(): commits = [] log = subprocess.check_output(['git', 'log', '--format=%h|%s|%ai']).decode('utf8') for line in log.split('\n'): if line: hash, subject, datestring = line.split('|') date = datetime.strptime(datestring[:16], '%Y-%m-%d %H:%M') commits.append(Commit(hash=hash, subject=subject, date=date)) return commits def checkout_commit(hash): subprocess.check_call(['git', 'checkout', hash]) def get_wordcounts(): docs = [f for f in os.listdir(BOOK_ROOT) if f.endswith('.asciidoc')] wordcounts = [] for filename in docs: with open(os.path.join(BOOK_ROOT, filename)) as f: contents = f.read() lines = len(contents.split('\n')) words = len(contents.split()) filename = re.sub(r'_(\d)\.asciidoc', r'_0\1.asciidoc', filename) filename = re.sub(r'chapter(\d\d)\.asciidoc', r'chapter_\1.asciidoc', filename) wordcounts.append(WordCount(filename, lines=lines, words=words)) return wordcounts def main(): commits = get_log() all_wordcounts = {} filenames = set() try: for commit in commits: checkout_commit(commit.hash) all_wordcounts[commit] = get_wordcounts() filenames.update(set(wc.filename for wc in all_wordcounts[commit])) with open(os.path.join(BOOK_ROOT, 'wordcounts.tsv'), 'w') as csvfile: fieldnames = ['date.{}'.format(thing) for thing in ['year', 'month', 'day', 'hour']] fieldnames += ['subject', 'hash', 'date'] fieldnames.extend(sorted(filename + " (words)" for filename in filenames)) fieldnames.extend(sorted(filename + " (lines)" for filename in filenames)) writer = csv.DictWriter(csvfile, fieldnames, dialect="excel-tab") writer.writeheader() for commit, wordcounts in all_wordcounts.items(): row = {} row['hash'] = commit.hash row['subject'] = commit.subject row['date'] = '' row['date.year'] = commit.date.year row['date.month'] = commit.date.month row['date.day'] = commit.date.day row['date.hour'] = commit.date.hour for wordcount in wordcounts: row[wordcount.filename + " (words)"] = wordcount.words row[wordcount.filename + " (lines)"] = wordcount.lines writer.writerow(row) finally: checkout_commit('master') if __name__ == '__main__': main() ================================================ FILE: misc/get_stats.sh ================================================ dropbox stop python3 get_stats.py dropbox start ================================================ FILE: misc/isolation-talks/djangoisland.md ================================================ Outside-In TDD, Test Isolation, and Mocking =========================================== * Harry Percival, @hjwp, www.obeythetestinggoat.com * Outside-in TDD? * DHH "TDD is Dead": agree? disagree? * No idea? * Who doesn't know what a mock is? # PS if you're coming to my tutorial tomorrow: **INSTALL STUFF** * Python 3.3+ * Django 1.7 (from 1.7.x stable branch on gh) * Selenium * Firefox -- instructions in preface of my book, available online, www.obeythetestinggoat.com Conclusion ========== *Listen to your tests* - ppl complain about "too many mocks": are there architectural solutions that would solve your problem? - Ports & Adapters Hexagonal / Clean architecture - Functional Core Imperative Shell - are you testing at the right level? *Further reading* - see chapter 22 in my book (available free online) for a reading list. www.obeythetestinggoat.com # What do we want from our tests anyway? * Correctness * Clean, maintainable code * Productive workflow # On the Pros and Cons of Different Types of Test Functional tests:: * Provide the best guarantee that your application really works correctly, from the point of view of the user. * But: it's a slower feedback cycle, * And they don't necessarily help you write clean code. Integrated tests (reliant on, eg, the ORM or the Django Test Client):: * Are quick to write, * Easy to understand, * Will warn you of any integration issues, * But may not always drive good design (that's up to you!). * And are usually slower than isolated tests Isolated ("mocky") tests:: * These involve the most hard work. * They can be harder to read and understand, * But: these are the best ones for guiding you towards better design. * And they run the fastest. # PS If you're coming to my tutorial tomorrow: **INSTALL STUFF** * Python 3.3+ * Django 1.7 (from 1.7.x stable branch on gh) * Selenium * Firefox -- instructions in preface of my book, available online, www.obeythetestinggoat.com ================================================ FILE: misc/isolation-talks/djangoisland.py ================================================ # models.py from django import models class List(models.Model): pass class Item(models.Model): text = models.TextField(default='') list = models.ForeignKey(List, default=None) ================================================ FILE: misc/isolation-talks/extra_styling_for_djangoisland.css ================================================ body, a, em { color: white; } body div.listingblock { margin-bottom: 100ex; } p code, table code { color: white; } div.admonitionblock { display: none; } ================================================ FILE: misc/isolation-talks/outline.txt ================================================ * 787a587 First real draft of My Lists FT. --ch18l001-- * 9a0b007 do-nothing my lists link in navbar. --ch18l002-1-- * 82ad580 add actual URL for my_lists to template --ch18l002-2-- * 785c527 test for my lists url and template. --ch18l003-- * b29a119 URL for my lists. --ch18l004-- * 7347a49 minimal view for my_lists, just renders template. --ch18l005-- * 9e2a450 minimal my_lists.html template. --ch18l006-- * 00ca8c4 Add block for list_form. --ch18l007-1-- * 735a457 Add block for extra_content inside bs row + col. --ch18l007-2-- * b769dcb flesh out my_lists.html. --ch18l010-- * 3f20d8b test passes owner to my_lists template. --ch18l011-- * c9b24e9 view passes owner to my_lists template. --ch18l012-- * 51a85da add user to other view test. --ch18l013-- * e0dc807 test that new list view saves owner. --ch18l014-- * 3d98981 (tag: revisit_this_point_with_isolated_tests) Attempt saving list owner in view. --ch18l01 * 58c5f5f test lists can have owners. --ch18l018-- * 5d18abd extra test that list owner is optional. --ch18l020-- * a6f70ab optional owner field on list model. --ch18l021-- * 8349ba0 Migration for list owner. --ch18l022-- * f0dba4f only save list owner if user is logged in. --ch18l023-- * c0890b8 test .name attribute of list model. --ch18l024-- * b08c8c4 lists have a .name attribute. --ch18l025-- * bdbf725 fixup key ordering in list owner migration. * 04ef40a add another dunderfuture import in 18 * 456b99e (origin/chapter_18, chapter_18) fix encodingey thing in migraiton * 81a4ca9 first attempt at a mocky test for list owner saveage. --ch19l003-- * 871347c deliberately break by assigning owner after save. --ch19l004-- * fc78466 use side_effect to check ordering. --ch19l005-- * ec2ae28 revert views.py back to correctish owner saving. --ch19l006-- * 96249c7 revert test back to non-mocky one with a skip. rename test class. --ch19l008-- * 4240536 placeholder new_list2 view. --ch19l009-- * 4e98e70 new isolated test for new view. --ch19l010-- * 4e8ba73 import NewListForm in views.py. --ch19l011-- * a670e27 placeholder NewListForm. --ch19l012-- * 6be5ea9 start on using form in view. --ch19l012-2-- * e9088b4 new test that form is saved. --ch19l013-- * 3e5c30a save form with owner. --ch19l014-- * 4953532 test we redirect if valid. --ch19l015-- * 7752d0a redirect to saved form object. --ch19l016-- * abcd3cc test that we render home template if form invalid. --ch19l017-- * 4a87c03 deliberately slightly broken view. --ch19l018-- * 4b5ccd7 test form not saved in invalid case. --ch19l019-- * 954f352 fix logic error, view now done. --ch19l020-- * 09020a8 First isolated test for form save. --ch19l021-- * 680b7ce second isolated test for forms authenticated user case. --ch19l022-- * 3ad4afd add import into forms. --ch19l023-- * d51d93b placeholder for List.create_new. --ch19l024-- * 32d344e First cut of form save. --ch19l025-- * 9b904f3 test create_new at models layer. --ch19l026-- * 730497f first cut of create_new staticmethod. --ch19l027-- * 043c609 test create_new with owner. --ch19l028-- * cc52837 test lists can have optional owner. --ch19l029-- * e0ea36a owner field on List. --ch19l030-1-- * f4499ad migration for list owner --ch19l030-2-- * faeadf0 create_new now saves owner --ch19l030-3-- * 7638d7b fix old view so that it conditionally saves user. --ch19l031-- * f7c0596 switch out new list function in urls. --ch19l032-- * b18d2cd switch out integrated test to use new view. --ch19l033-- * 466957d placeholder test that form save returns the new list. --ch19l038-1-- * 925a74c placeholder test for create_new returning new list --ch19l038-2-- * 3af89e0 return list from form save. --ch19l039-1-- * 09d53cc flesh out test that create_new should return list. --ch19l039-2-- * f90648c return newly created list in List.create_new --ch19l039-3-- * d78dd00 test list.name is first item text. --ch19l040-- * 55a283e list.name is first item text. --ch19l041-- * 86ab021 remove unused code from forms * 10e0449 remove old new_lists view from test_views. --ch19l045-- * 7a175c6 rename to new_list from new_list2 in urls.py. --ch19l046-- * 656c3e6 delete old new_list from views.py. --ch19l047-- * d952cee (origin/chapter_19) Strip out irrelephant integrated view tests. --ch19l048-- ================================================ FILE: misc/isolation-talks/webcast-commits.hist ================================================ * 787a587 First real draft of My Lists FT. --ch18l001-- * 9a0b007 do-nothing my lists link in navbar. --ch18l002-1-- * 82ad580 add actual URL for my_lists to template --ch18l002-2-- * 785c527 test for my lists url and template. --ch18l003-- * b29a119 URL for my lists. --ch18l004-- * 7347a49 minimal view for my_lists, just renders template. --ch18l005-- * 9e2a450 minimal my_lists.html template. --ch18l006-- * 00ca8c4 Add block for list_form. --ch18l007-1-- * 735a457 Add block for extra_content inside bs row + col. --ch18l007-2-- * b769dcb flesh out my_lists.html. --ch18l010-- * 3f20d8b test passes owner to my_lists template. --ch18l011-- * c9b24e9 view passes owner to my_lists template. --ch18l012-- * 51a85da add user to other view test. --ch18l013-- * e0dc807 test that new list view saves owner. --ch18l014-- * 3d98981 (tag: revisit_this_point_with_isolated_tests) Attempt saving list owner in view. --ch18l015-- * 58c5f5f test lists can have owners. --ch18l018-- * 5d18abd extra test that list owner is optional. --ch18l020-- * a6f70ab optional owner field on list model. --ch18l021-- * 8349ba0 Migration for list owner. --ch18l022-- * f0dba4f only save list owner if user is logged in. --ch18l023-- * c0890b8 test .name attribute of list model. --ch18l024-- * b08c8c4 lists have a .name attribute. --ch18l025-- * bdbf725 fixup key ordering in list owner migration. * 04ef40a add another dunderfuture import in 18 * 456b99e (HEAD, origin/chapter_18, chapter_18) fix encodingey thing in migraiton * 3d98981 (tag: revisit_this_point_with_isolated_tests) Attempt saving list owner in view. --ch18l015-- * 81a4ca9 first attempt at a mocky test for list owner saveage. --ch19l003-- * 871347c deliberately break by assigning owner after save. --ch19l004-- * fc78466 use side_effect to check ordering. --ch19l005-- * ec2ae28 revert views.py back to correctish owner saving. --ch19l006-- * 96249c7 revert test back to non-mocky one with a skip. rename test class. --ch19l008-- * 4240536 placeholder new_list2 view. --ch19l009-- * 4e98e70 new isolated test for new view. --ch19l010-- * 4e8ba73 import NewListForm in views.py. --ch19l011-- * a670e27 placeholder NewListForm. --ch19l012-- * 6be5ea9 start on using form in view. --ch19l012-2-- * e9088b4 new test that form is saved. --ch19l013-- * 3e5c30a save form with owner. --ch19l014-- * 4953532 test we redirect if valid. --ch19l015-- * 7752d0a redirect to saved form object. --ch19l016-- * abcd3cc test that we render home template if form invalid. --ch19l017-- * 4a87c03 deliberately slightly broken view. --ch19l018-- * 4b5ccd7 test form not saved in invalid case. --ch19l019-- * 954f352 fix logic error, view now done. --ch19l020-- * 09020a8 First isolated test for form save. --ch19l021-- * 680b7ce second isolated test for forms authenticated user case. --ch19l022-- * 3ad4afd add import into forms. --ch19l023-- * d51d93b placeholder for List.create_new. --ch19l024-- * 32d344e First cut of form save. --ch19l025-- * 9b904f3 test create_new at models layer. --ch19l026-- * 730497f first cut of create_new staticmethod. --ch19l027-- * 043c609 test create_new with owner. --ch19l028-- * cc52837 test lists can have optional owner. --ch19l029-- * e0ea36a owner field on List. --ch19l030-1-- * f4499ad migration for list owner --ch19l030-2-- * faeadf0 create_new now saves owner --ch19l030-3-- * 7638d7b fix old view so that it conditionally saves user. --ch19l031-- * f7c0596 switch out new list function in urls. --ch19l032-- * b18d2cd switch out integrated test to use new view. --ch19l033-- * 466957d placeholder test that form save returns the new list. --ch19l038-1-- * 925a74c placeholder test for create_new returning new list --ch19l038-2-- * 3af89e0 return list from form save. --ch19l039-1-- * 09d53cc flesh out test that create_new should return list. --ch19l039-2-- * f90648c return newly created list in List.create_new --ch19l039-3-- * d78dd00 test list.name is first item text. --ch19l040-- * 55a283e list.name is first item text. --ch19l041-- * 86ab021 remove unused code from forms * 10e0449 remove old new_lists view from test_views. --ch19l045-- * 7a175c6 rename to new_list from new_list2 in urls.py. --ch19l046-- * 656c3e6 delete old new_list from views.py. --ch19l047-- * d952cee (HEAD, origin/chapter_19, chapter_19) Strip out irrelephant integrated view tests. --ch19l048-- ================================================ FILE: misc/plot.py ================================================ from datetime import datetime import numpy from matplotlib import pyplot import csv def get_data_from_csv(): with open('wordcounts.tsv') as f: reader = csv.DictReader(f, dialect="excel-tab") data = [] for ix, row in enumerate(reader): fixed_row = {} if ix > 4: break for field in reader.fieldnames: if 'words' in field: val = row[field] if val: fixed_row[field] = val else: fixed_row[field] = 0 date = datetime(int(row['date.year']), int(row['date.month']), int(row['date.day']), int(row['date.hour']),) fixed_row['date'] = date data.append(fixed_row) return data # print(len(data)) data = {} data['date'] = [1, 4, 3, 2] data['words1'] = [0, 3, 3, 5] data['words2'] = [4, 6, 0, 2] array = [data['date'], data['words1'], data['words2']] numpy.sort(array, 0) data = get_data_from_csv() data.sort(key=lambda d: d['date']) x = [d['date'] for d in data] y = [ [d[key] for d in data] for key in data[0].keys() if 'words' in key ] pyplot.stackplot(x, y) # pyplot.stackplot(data['date'], [values for (field, values) in data.items() if 'words' in field]) # for (field, values) in data.items(): # if 'words' in field: # pyplot.plot(data['date'], values) pyplot.show() ================================================ FILE: misc/reddit_post.md ================================================ Hi Python-Redditors!, I'm somehow writing a book on TDD and Python for O'Reilly, despite feeling massively underqualified. Please help me to instill the book with more wisdom than I can muster on my own! Here's a link where you can take a look at what I have so far (8 chapters): http://www.obeythetestinggoat.com **My background:** I've only been a professional programmer for about 4 years, but I was lucky enough to fall in with a bunch of XP fanatics at Resolver Systems, now [PythonAnywhere](http://www.pythonanywhere.com). Over the past few years I've learned an awful lot, and I'm trying to share what I've learned with other people who are starting out. Someone described learning TDD as being a step on the journey from amateur coder to professional, maybe that's a good way of putting it. For the past couple of years I've been running beginners' TDD / Django workshops at EuroPython and Pycons, and they've been well-received. They're probably what must have fooled O'Reilly into thinking I could write a book! **The book** The concept is to take the user through building a web app from scratch, but using full TDD all along the way. That involves: * Functional tests using Selenium * "unit" tests using the Django test runner * All Django goodness including views, models, templates, forms, admin etc * Unit testing javascript * Tips on deployment, and how to test against a staging site ...and lots more. You'll find a [proposed chapter outline](http://chimera.labs.oreilly.com/books/1234000000754/ch09.html) at the end of the book. I've made a good start, 8 chapters, taking us all the way to deploying a minimum viable app, but I really need some more feedback. * Am I covering the right stuff? Would you add / remove any topics? * Am I teaching what you consider to be best practice? Some people, for example, think that touching the database in a thing you call a unit test is an absolute no-no (cf [past discussion on reddit](http://www.reddit.com/r/django/comments/1c67rl/is_tddjangotutorial_truly_a_good_resource_i_want/)). Do you think that's important? Should I acknowledge the controversy, and maybe refer people to an appendix which discusses how to avoid hitting the db in unit tests? Or should I throw out the Django test runner? * I'm also telling people to use a very belt & braces technique, where we write functional tests *and* unit tests for pretty much every single line of code. Is that overkill? Or do you think that it's a good thing to learn, even if you later relax the rules when you get a bit more experience (I make this argument in chapter 4) * My latest chapter is [all about deployment](http://chimera.labs.oreilly.com/books/1234000000754/ch08.html) -- lots of people do different things in this area, we had a good discussion of it on the Python-UK mailing list. What do you think of what I have so far? **Why help?** Perhaps you're thinking "Why should I help this guy to write a better book and make more money, despite the blatant massive lacunae in his knowledge?". Well, one thing that I hope will sway your cold, hard heart is that I am pledging to make the book available for free under a CC license, alongside the official paid-for printed and ebook versions. I insisted on that as part of my contract, and O'Reilly were totally happy with it (props to them for being forward-thinking). I really hope that this book can distill something of the sum total of all the knowledge in the Python testing community, over and above my own, and be a useful resource for that whole community, and not just those that can afford to pay for a book... If that's not enough, how about I promise I'll buy you a beer one day? Thanks in advance, Harry ================================================ FILE: misc/redditnotesresponse.txt ================================================ Thanks again for your detailed and very helpful comments! First off, on your general suggestion that Git and Deployment are 'orthogonal' to TDD -- I tend to agree, especially on the former. I guess it's about the picture I have in my head of my target audience -- people who are relatively new to programming, who've maybe hacked together a few programs, or maybe just finished a degree. Basically, it's me 4 years ago. If I look back on what I've learned over the past few years, things I think are absolutely indispensable skills, TDD is one, but how Git fits into my development process is also a huge part of it. Experienced developers will probably already know git, so they can just skip over my explanations (which are pretty much confined to chapter 1), but my hope is that beginners will get a lot out of it. It's all part of the flow of a more mature development process -- test, edit, commit, as someone put it in another thread. Same thing with deployment -- although I think I've made a case for how testing *does* interact with deployment and staging sites, much more so than it does with VCS. But the point is that you're not a web developer until you've deployed your app, so, again, my hope is that beginners will be getting something out of that chapter. Re showing non-best-practice first, my aim is to start with the simplest possible thing as a way of introducing the core concept. Then I can bring in the best practice once I feel the reader is ready for them. I mean, I can't throw in git, virtualenvs, south, postgres, gunicorn, CDNs, selenium, unittest, django, memcached and all the rest into the very first chapter! Perhaps I do sometimes take it a little too far though... I'll have another read-through. Thanks for the suggestion re the FT comments spec too. I guess some of that is a matter of style, or what you're used to... The comments are consciously quite vague, whilst the selenium assertions are specific, precisely because the former represent human-language requirements and the latter start to be implementation-dependent. You could imagine changing the implementation without changing the implementation. I'm thinking of including a chapter or appendix on BDD though, so I may be able to introduce a different style there. I really don't know enough about it yet, so if you 're interested in a collaboration, do get in touch. Re chapter 7, I know I'm in a danger area there. I certainly don't think you should test cosmetics, and as you diagnosed, the main point of the test is as a smoke test that the CSS has loaded correctly. So, would some kind of non-functional integration test be better? Maybe... The FT I guess adds the check that not only is the CSS available, but also that it was loaded correctly by the page... I guess I should just make it clearer about what the objective of that test is, and be more explicit about saying "don't test cosmetics". With that said, there's definitely a place for testing layout using functional tests... JavaScript-driven resize behaviour an example that springs to mind from PythonAnywhere Thanks for the validation re my belt + braces, FT + unit approach. Finally, onto the random notes! I've never used RequestFactory. What does it give you that instantiating a regular HttpRequest() doesn't? And, wow, thanks for telling me about addCleanup() -- I can't believe I'd never come across that (maybe it's because we used to write within the constraints of Python 2.6 unittest). Love it, will definitely be using it. Re starting + stopping firefox for each test, I just defaulted to that because it's what we do at PA. I think our intention there was about isolating FTs completely, making sure that cookies + sessions are clean each time, etc. That may not always be necessary, but at least once I do it very deliberately... I'll make a mention of it as a possible optimisation. Finally, on the overall dev process, I'm just alt-tabbing between a console and editor, and that's pretty much what I'm telling people to do. Automated file watchers are cool -- I think my ex-colleague JB wrote one called rerun -- but I think I'll leave them for people to discover on their own. CI I hope to cover in a later chapter, cf the outline... Once again, thanks again for your detailed comments. It's nice to have some validation, to learn a couple of things, and also to have a couple of decisions challenged. I'll go away and think about it all. And do get in touch if you want to work together on a BDD chapter -- if nothing else, I'd love to know what you use, and what your experience is with your tools, what you've ruled out, what you've preferred and why, possible pitfalls... ================================================ FILE: misc/tdd-flowchart.dot ================================================ digraph g { write_test [label="Write a test" shape=box] if_passes [label="Run the test.\nDoes it pass?" shape=diamond] code [label="Write minimal code" shape=box] refactor [label="Refactor (optional)" shape=box] subgraph along { rank = same; write_test -> if_passes ; if_passes -> refactor [label="Yes" color = green] ; refactor -> write_test; } subgraph down { rankdir = UD; if_passes -> code:w [label="No" color = red] ; } code:e -> if_passes } ================================================ FILE: outline_and_future_chapters.asciidoc ================================================ Outline to date & future chapters plan -------------------------------------- Thanks for reading this far! I'd really like your input on this too: What do you think of the book so far, and what do you think about the topics I'm proposing to cover in the list below? Email me at obeythetestinggoat@gmail.com! NB - when I say "book" below, they're all going to be parts of this book. I guess I should say "part" instead, but for some reason I've decided book sounds cooler. Like Lord of the Rings. * Preface (Chapter 0) - intro + pre-requisites BOOK 1: Building a minimum viable app with TDD ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Done: * Chapter 1 - Getting Django set up using a Functional Test * Chapter 2 - Extending our FT using the unittest module * Chapter 3 - Testing a simple home page with unit tests * Chapter 4 - What are we doing with all these tests? * Chapter 5 - Saving form submissions to the database * Chapter 6 - Getting to the minimum viable site * Chapter 7 - Prettification * Chapter 8 - Deploy! BOOK 2: Growing the site ~~~~~~~~~~~~~~~~~~~~~~~~ Done: * Chapter 9 - Input validation and test organisation * Chapter 10 - A simple form * Chapter 11 - More advanced Forms * Chapter 12 - Database migrations * Chapter 13 - Dipping our toes, very tentatively, into JavaScript * Chapter 14 - User authentication, integrating 3rd party plugins, and Mocking with JavaScript * Chapter 15 - Mocking 3rd party web services with Python mock * Chapter 16 - Server-side test database management * Chapter 17 - Continuous Integration (CI) with Jenkins Planned: Chapter 18: sharing lists ^^^^^^^^^^^^^^^^^^^^^^^^^ * email notifications * django notifications (?) * "claim" an existing list (?) * URLs would need to be less guessable More/Other possible contents ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ *Django stuff:* * switch database to eg postgres * uploading files? *Testing topics to work in* * more how to read a traceback (ref step 4 in chapter 3) * simplify the model test down to minimal/best practice. * talk about the "purist" approach to unit testing vs the django test client. * AKA "Functional core imperative shell" * selenium page pattern * coverage * alternative test runners -- py.test, nose (lots of ppl mentioned latter) * addCleanup * PhantomJS for faster Fts? * fixtures (factory boy?) * JS: mocking external web service to simulate errors * Splinter * difference between unittest.TestCase, django.test.TestCase, LiveServerTestCase * general troubleshooting tips (appendix, collecting all notes etc?) * LiveServerTestCase does not flush staging server DB. fix in CI chapter? * How to stop and start (expand on bit in ch4. refer to stop+start book) * some kind of indicator of where in the tdd cycle we are, in the margin? *Deployment stuff* * FT for 404 and 500 pages? * email integration BOOK 3: Trendy stuff ~~~~~~~~~~~~~~~~~~~~ Chapter 19 & 20: Javascript MVC frameworks ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * MVC tool (backbone / angular) * single page website (?) or bottomless web page? * switching to a full REST API * HTML5, eg LocalStorage * Encryption - client-side decrypt lists, for privacy? Chapter 21: Async ^^^^^^^^^^^^^^^^^ * websockets * tornado/gevent (or sthing based on Python 3 async??) * how to get django to talk to tornado: redis? (just for fun?) * for collaborative lists?? Chapter 22: Caching ^^^^^^^^^^^^^^^^^^^ * unit testing `memcached` * Functionally testing performance * Apache `ab` testing 5/6 chapters? Appendices ~~~~~~~~~~ Possible appendix topics ^^^^^^^^^^^^^^^^^^^^^^^^ * BDD (+2 from reddit) * Mobile (use selenium, link to using bootstrap?) * Payments... Some kind of shopping cart? * unit testing fabric scripts * testing tools pros & cons, eg django test client vs mocks, liverservertestcase vs roll-your-own * NoSQL / Redis / MongoDB? A PythonAnywhere ^^^^^^^^^^^^^^^^^ * Running Firefox Selenium sessions with pyVirtualDisplay * Setting up Django as a PythonAnywhere web app * Cleaning up /tmp * Screenshots B: Django class-based views ^^^^^^^^^^^^^^^^^^^^^^^^^^^ * refactoring, proving usefulness of view tests. C: Automated provisioning and configuration management with Ansible ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * light appendix. ================================================ FILE: part1.asciidoc ================================================ [[part1]] [part] [role="pagenumrestart"] == The Basics of TDD and Django [partintro] -- In this first part, I'm going to introduce the basics of 'test-driven development' (TDD). We'll build a real web application from scratch, writing tests first at every stage. We'll cover functional testing with Selenium, as well as unit testing, and see the difference between the two. I'll introduce the TDD workflow, red/green/refactor. I'll also be using a version control system (Git). We'll discuss how and when to do commits and integrate them with the TDD and web development workflow. We'll be using Django, the Python world's most popular web framework (probably). I've tried to introduce the Django concepts slowly and one at a time, and provide lots of links to further reading. If you're a total beginner to Django, I thoroughly recommend taking the time to read them. If you find yourself feeling a bit lost, take a couple of hours to go through the https://docs.djangoproject.com/en/5.2/intro[official Django tutorial] and then come back to the book. In <>, you'll also get to meet the Testing Goat... [WARNING] ==== Be careful with copy and paste. If you're working from a digital version of the book, it's natural to want to copy and paste code listings from the book as you're working through it. It's much better if you don't: typing things in by hand gets them into your muscle memory, and just feels much more real. You also inevitably make the occasional typo, and debugging is an important thing to learn. Quite apart from that, you'll find that the quirks of the PDF format mean that weird stuff often happens when you try to copy/paste from it... ==== -- ================================================ FILE: part2.asciidoc ================================================ [[part2]] [part] == Going to Production [partintro] [quote, 'https://oreil.ly/Q7UDe[DevOps Borat]'] ______________________________________________________________ Is all fun and game until you are need of put it in production. ______________________________________________________________ It's time to deploy the first version of our site and make it public. They say that if you wait until you feel _ready_ to ship, then you've waited too long. Is our site usable? Is it better than nothing? Can we make lists on it? Yes, yes, yes. No, you can't log in yet. No, you can't mark tasks as completed. But do we really need any of that stuff? Not really--and you can never be sure what your users are _actually_ going to do with your site once they get their hands on it. We think our users want to use the site for to-do lists, but maybe they actually want to use it to make "top 10 best fly-fishing spots" lists, for which you don't _need_ any kind of "mark completed" function. We won't know until we put it out there. Over the next couple of chapters we're going to go through and actually deploy our site to a real, live web server. You might be tempted to skip this bit--there's lots of daunting stuff in it, and maybe you think this isn't what you signed up for. But I _strongly_ urge you to give it a go. This is one of the sections of the book I'm most pleased with, and it's one that people often write to me about saying they were really glad they stuck through it. If you've never done a server deployment before, it will demystify a whole world for you, and there's nothing like the feeling of seeing your site live on the actual internet. Give it a buzzword name like "DevOps" if that's what it takes to convince you it's worth it. [role="notoc"] === The Danger Areas of Deployment Deploying a site to a live web server can be a tricky topic. Oft heard is the forlorn cry, "but it works on my machine!" ((("deployment", "danger areas of"))) Some of the danger areas of deployment include: Networking:: Once we're off our own machine, networking issues come in: making sure that DNS is routing our domain to the correct IP address for our server, making sure our server is configured to listen to traffic coming in from the world, making sure it's using the right ports, and making sure any firewalls in the way are configured to let traffic through. Dependencies:: We need to make sure that the packages our software relies on (Python, Django, and so on) are installed on the server and have the correct versions. The database:: There can be permissions and path issues, and we need to be careful about preserving data between deploys. Static files (CSS, JavaScript, images, etc.):: Web servers usually need special configuration for serving these. ((("static files", "challenges of"))) Security and configuration:: Once we're on the public internet, we need to worry more about security. Various settings that are really useful for local development (like the Django debug page) become dangerous in production (because they expose our source code in tracebacks). Reproducibility and divergence between local dev and prod:: All of the above add up to differences between your local development environment and the way code runs in production. We want to be able to reproduce the way things work on our machine, as closely as possible, in production (and vice versa) to give us as much confidence as possible that "it works on my machine" means "it's going to work in production". One way to approach the problem is to get a server and start manually configuring and installing everything, hacking about until it works, and maybe think about automating things later.footnote:[ This was, more or less, the approach I took in earlier editions of the book. With a fair bit of testing thrown in, of course.] But if there's one thing we've learned in the world of Agile/Lean software development, it's that taking smaller steps usually pays off. How can we take smaller, safer steps towards a production deployment? Can we _simulate_ the process of moving to a server so that we can iron out all the bugs before we actually take the plunge? Can we then make small changes one at a time, solving problems one by one, rather than having to bite off everything in one mouthful? Can we use our existing test suite to make sure things work on the server, as well as locally? Absolutely we can. And if you've looked at the table of contents, I'm sure you're already guessing that Docker is going to be part of the answer. [role="notoc"] === An Overview of Our Deployment Procedure Over the next three chapters, I'm going to go through a deployment procedure. It isn't meant to be the _perfect_ deployment procedure, so please don't take it as being best practice or a recommendation--it's meant to be an illustration, to show the kinds of issues involved in putting code into production, and where testing fits in. <>:: * Adapt our functional tests (FTs) so they can run against a container. * Build a minimal Dockerfile with everything we need to run our site. * Learn how to build and run a container on our machine. * Get a first cut of our code up and running inside Docker, with passing tests. <>:: * Gradually, incrementally change the container configuration to make it production-ready. * Regularly rerun the FTs to check we didn't break anything. * Address issues to do with the database, static files, secrets, and so on. <>:: * Set up a "staging" server,footnote:[ Some people prefer the term pre-prod or test environment. It's all the same idea.] using the same infrastructure that we plan to use for production. * Set up a real domain name and point it at this server. * Install Ansible and flush out any networking issues. [role="pagebreak-before less_space"] <>:: * Gradually build up an Ansible playbook to deploy our containers on a real server. * Again, use our FTs to check for any problems. * Learn how to SSH (Secure Shell) into the server to debug things, locate logs, and find other useful information. * Confidently deploy to production once we have a working deployment script for staging. [role="notoc"] === TDD and Docker Versus the Danger Areas of Deployment Hopefully you can start to see how the combination of TDD, Docker, staging, and automation are going to help minimise the risk of the various "danger areas": Containers as mini servers:: Containers will act as mini servers letting us flush out issues with dependencies, static files, and so on. A key advantage is that they'll give us a way of getting faster feedback cycles; because we can spin them up locally almost instaneously, we can very quicly see the effect of any changes. Packaging Python and system dependencies:: Our containers will package up both our Python and system dependencies, including a production-ready web server and static files system, as well as many production settings and configuration differences. This minimises the difference between what we can test locally, and what we will have on our servers. As we'll see, it will give us a reliable way to reproduce bugs we see in production, on our local machine. Fully automated FTs:: Our FTs mean that we'll have a fully automated way of checking that everything works. Running FTs on staging server:: Later, when we deploy our containers to a staging server, we can run the FTs against that too. It'll be slightly slower and might involve some fiddly compromises, but it'll give us one more layer of reassurance. Automating build and deployment:: Finally, by fully automating container creation and deployment, and by testing the end results of both these things, we maximise reproducibility, thus minimising the risk of deployment to production. Oh, but there's lots of fun stuff coming up! Just you wait! ================================================ FILE: part3.asciidoc ================================================ [[part3]] [part] == Forms and Validation [partintro] -- Now that we've got things into production, we'll spend a bit of time on validation, a core topic in web development. There's quite a lot of Django-specific content in this part, so if you weren't familiar with Django before starting on the book, you may find that taking a little time to run through the https://docs.djangoproject.com/en/5.2/intro/tutorial01/#creating-models[official Django tutorial] will complement the next few chapters nicely. With that said, there are lots of good lessons about test-driven development (TDD) in general in here too! So, alternatively, if you're not that interested in Django itself, don't worry too much about the details; instead, look out for the more general principles of testing. Here's a little preview of what we'll cover: * Splitting tests out across multiple files * Using a decorator for Selenium waits/polling * Database-layer validation and constraints * HTML5 form validation in the frontend * The Django forms framework * The trade-offs of frameworks in general, and when to stop using them * How far to go when testing for possible coding errors * An overview of all the typical tests for Django views -- ================================================ FILE: part4.asciidoc ================================================ [[part4]] [part] == More Advanced Topics in Testing [partintro] -- "Oh my gosh, what? Another section? Harry, I'm exhausted. It's already been four hundred pages; I don't think I can handle a whole nother section of the book. Particularly not if it's called 'Advanced’...maybe I can get away with just skipping it?" Oh no, you can't! This may be called the "advanced" section, but it's full of really important topics for test-driven development (TDD) and web development. No way can you skip it. If anything, it's 'even more important' than the first two sections. First off, we'll get into that sine qua non of web development: JavaScript. Seeing how TDD works in another language can give you a whole new perspective. We'll be talking about a key technique, "spiking", which is where you relax the strict rules of TDD and allow yourself a bit of exploratory hacking. TIP: A common objection to TDD is "how can I write tests if I don't even know what I'm doing?" Spiking is the bit where you get to play around and figure things out, so you can come back and do it test-first later. We'll be talking about how to integrate third-party systems, and how to test them. We'll cover mocking, which is hard to avoid in the world of Python testing.footnote:[Although not impossible! Check out the book https://www.cosmicpython.com[_Cosmic Python_], which has tips on testing without mocks. I happen to know that at least one of the two authors is incredibly wise.] We'll talk about test fixtures and server-side debugging, and how to set up a continuous integration (CI) environment. None of these things are take-it-or-leave-it, optional, luxury extras for your project--they're all vital! Inevitably, the learning curve does get a little steeper in this section. You may find yourself having to read things a couple of times before they sink in, or you may find that things don't work on the first go, and that you need to do a bit of debugging on your own. But I encourage you to persist with it! The harder it is, the more rewarding it is, right? And, remember, I'm always happy to help if you're stuck; just drop me an email at obeythetestinggoat@gmail.com. Come on; I promise the best is yet to come! -- ================================================ FILE: praise.forbook.asciidoc ================================================ ["dedication", role="praise"] == Praise for 'Test-Driven Development with Python' [quote, Michael Foord, Python Core Developer and Maintainer of unittest] ____ “In this book, Harry takes us on an adventure of discovery with Python and testing. It’s an excellent book, fun to read and full of vital information. It has my highest recommendations for anyone interested in testing with Python, learning Django or wanting to use Selenium. Testing is essential for developer sanity and it's a notoriously difficult field, full of trade-offs. Harry does a fantastic job of holding our attention whilst exploring real world testing practices.” ____ [quote, Kenneth Reitz, Fellow at Python Software Foundation] ____ “This book is far more than an introduction to Test Driven Development—it’s a complete best-practices crash course, from start to finish, into modern web application development with Python. Every web developer needs this book.” ____ [quote, Daniel and Audrey Roy Greenfeld, authors of "Two Scoops of Django" (Two Scoops Press)] ____ “Harry’s book is what we wish existed when we were learning Django. At a pace that’s achievable and yet delightfully challenging, it provides excellent instruction for Django and various test practices. The material on Selenium alone makes the book worth purchasing, but there's so much more!” ____ ================================================ FILE: praise.html ================================================

Praise for Test-Driven Development with Python

In this book, Harry takes us on an adventure of discovery with Python and testing. It’s an excellent book, fun to read, and full of vital information. It has my highest recommendations for anyone interested in testing with Python, learning Django, or wanting to use Selenium. Testing is essential for developer sanity and it's a notoriously difficult field, full of trade-offs. Harry does a fantastic job of holding our attention whilst exploring real-world testing practices.

Michael Foord, Python Core Developer and Maintainer of unittest

This book is far more than an introduction to test-driven development—it’s a complete best-practices crash course, from start to finish, into modern web application development with Python. Every web developer needs this book.

Kenneth Reitz,
Fellow at Python Software Foundation

Harry’s book is what we wish existed when we were learning Django. At a pace that’s achievable and yet delightfully challenging, it provides excellent instruction for Django and various test practices. The material on Selenium alone makes the book worth purchasing, but there's so much more!

Daniel and Audrey Roy Greenfeld, authors of Two Scoops of Django (Two Scoops Press)

================================================ FILE: pre-requisite-installations.asciidoc ================================================ [[pre-requisites]] [preface] == Prerequisites and Assumptions ((("prerequisite knowledge", id="prereq00"))) ((("Test-Driven Development (TDD)", "prerequisite knowledge assumed", id="TDDprereq00"))) Here's an outline of what I'm assuming about you and what you already know, as well as what software you'll need ready and installed on your computer. === Python 3 and Programming ((("Python 3", "introductory books on"))) I've tried to write this book with beginners in mind, but if you're new to programming, I'm assuming that you've already learned the basics of Python. So if you haven't already, do run through a Python beginner's tutorial or get an introductory book like https://www.manning.com/books/the-quick-python-book-third-edition[_The Quick Python Book_] or https://oreil.ly/think-python-3e[_Think Python_], or (just for fun) https://inventwithpython.com/invent4thed[_Invent Your Own Computer Games with Python_]—all of which are excellent introductions. If you're an experienced programmer but new to Python, you should get along just fine. Python is joyously simple to understand. You should be able to follow this book on Mac, Windows, or Linux. Detailed installation instructions for each OS follow. TIP: This book was tested against Python 3.14. If you're on an earlier version, you will find minor differences in the way things look in my command output listings (tracebacks won't have the `^^^^^^` carets marking error locations, for example), so you're best off upgrading, ideally, if you can. In any case, I expect you to have access to Python, to know how to launch it from a command line, and to know how to edit a Python file and run it. Again, have a look at the three books I recommended previously if you're in any doubt. === How HTML Works ((("HTML", "tutorials")))I'm also assuming you have a basic grasp of how the web works—what HTML is, what a POST request is, and so on. If you're not sure about those, you'll need to find a basic HTML tutorial; there are a few at https://developer.mozilla.org/Learn_web_development. If you can figure out how to create an HTML page on your PC and look at it in your browser, and understand what a form is and how it might work, then you're probably OK. === Django ((("Django framework", "tutorials")))The book uses the Django framework, which is (probably) the most well-established web framework in the Python world. I've written this book assuming that the reader has no prior knowledge of Django, but if you're new to Python _and_ new to web development _and_ new to testing, you may occasionally find that there's just one too many topics and sets of concepts to try and take on board. If that's the case, I recommend taking a break from the book, and taking a look at a Django tutorial. https://tutorial.djangogirls.org[DjangoGirls] is the best, most beginner-friendly tutorial I know of. Django's https://docs.djangoproject.com/en/5.2/intro/tutorial01[official tutorial] is also excellent for more experienced programmers. === JavaScript There's a little bit of JavaScript in the second half of the book. If you don't know JavaScript, don't worry about it until then. And if you find yourself a little confused, I'll recommend a couple of guides at that point. Read on for installation instructions. === Required Software Installations ((("software requirements", id="soft00"))) Aside from Python, you'll need: The Firefox web browser:: Selenium can actually drive any of the major browsers, but I chose Firefox because it's the least in hock to corporate interests. ((("Firefox", "benefits of")))((("web browsers", "Firefox"))) The Git version control system:: This is available for any platform, at http://git-scm.com. On Windows, it comes with the _bash_ command line, which is needed for the book. See <>. ((("Git", "downloading"))) A virtualenv with Python 3.14, Django 5.2, and Selenium 4 in it:: Python's virtualenv and pip tools now come bundled with Python (they didn't always used to, so this is a big hooray). ((("virtualenv (virtual environment)"))) Detailed instructions for preparing your virtualenv follow. .MacOS Notes ******************************************************************************* ((("MacOS"))) ((("Python 3", "installation and setup", "MacOS installation"))) MacOS installations for Python and Git are relatively straightforward: * Python 3.14 should install without a fuss from its http://www.python.org[downloadable installer]. It will automatically install `pip`, too. * Git's installer should also "just work". * You might also want to check out http://brew.sh[Homebrew]. It's a fairly reliable way of installing common Unix tools on a Mac.footnote:[I wouldn't recommend installing Firefox via Homebrew though: `brew` puts the Firefox binary in a strange location, and it confuses Selenium. You can work around it, but it's simpler to just install Firefox in the normal way.] Although the normal Python installer is now fine, you may find Homebrew useful in future. It does require you to download all 1.1 GB of Xcode, but that also gives you a C compiler, which is a useful side effect. * If you want to run multiple different versions of Python on your Mac, tools like https://docs.astral.sh/uv/guides/install-python[uv] or https://github.com/pyenv/pyenv[pyenv] can help. The downside is that each is one more fiddly tool to have to learn. But the key is to make sure, when creating your virtualenv, that you use the right version of Python. From then on, you shouldn't need to worry about it, at least not when following this book. Similarly to Windows, the test for all this is that you should be able to open a terminal and just run `git`, `python3`, or `pip` from anywhere. If you run into any trouble, the search terms "system path" and "command not found" should provide good troubleshooting resources. ******************************************************************************* [role="pagebreak-before less_space"] .Linux Notes ******************************************************************************* ((("Linux"))) ((("Python 3", "installation and setup", "Linux installation"))) If you're on Linux, I'm assuming you're already a glutton for punishment, so you don't need detailed installation instructions. But in brief, if Python 3.14 isn't available directly from your package manager, you can try the following: * On Ubuntu, you can install the https://oreil.ly/fHrpG[deadsnakes PPA]. Make sure you `apt install python3.14-venv` as well as just `python3.14` to un-break the default Debian version of Python. * Alternatively, https://docs.astral.sh/uv/guides/install-python[uv] and https://github.com/pyenv/pyenv[pyenv] both let you manage multiple Python versions on the same machine, but it is yet another thing to have to learn and remember. * Alternatively, compiling Python from source is actually surprisingly easy! However you install it, make sure you can run Python 3.14 from a terminal. ******************************************************************************* [[windows-notes]] .Windows Notes ******************************************************************************* ((("Windows", "tips"))) ((("Python 3", "installation and setup", "Windows installation"))) Windows users can sometimes feel a little neglected in the open source world. As macOS and Linux are so prevalent, it's easy to forget there's a world outside the Unix paradigm. Backslashes as directory separators? Drive letters? What? Still, it is absolutely possible to follow along with this book on Windows. Here are a few tips: * When you install Git for Windows, it will include "Git Bash". Use this as your main command prompt throughout the book, and you'll get all the useful GNU command-line tools like `ls`, `touch`, and `grep`, plus forward-slash directory [.keep-together]#separators#. * During the Git installation, you'll get the option to choose the default editor used by Git. Unless you're already a Vim user (or are desperate to learn), I'd suggest using a more familiar editor—even just Notepad! See <>. * Also in the Git installer, choose "Use Windows' default console"; otherwise, Python won't work properly in the Git Bash window. * When you install Python, tick the option that says "Add python.exe to PATH" as in <>, so that you can easily run Python from the command line. The test for all this is that you should be able to go to a Git Bash command prompt and just run `python` or `pip` from any folder. [role="width-95"] [[git-windows-default-editor]] .Choose a nice default editor for Git image::images/tdd3_0001.png["Screenshot of Git installer"] [role="width-95"] [[add-python-to-path]] .Add Python to the system path from the installer image::images/tdd3_0002.png["Screenshot of python installer"] // TODO: update screenshot above for 3.14 ******************************************************************************* [[firefox_gecko]] ==== Installing Firefox ((("Firefox", "installing"))) Firefox is available as a download for Windows and macOS from pass:[firefox.com]. On Linux, you probably already have it installed, but otherwise your package manager will have it. ((("geckodriver"))) Make sure you have the latest version, so that the "geckodriver" browser automation module is available. === Setting Up Your Virtualenv ((("Python 3", "installation and setup", "virtualenv set up and activation", id="P3installvirt00"))) ((("virtualenv (virtual environment)", "installation and setup", id="VEinstall00"))) ((("", startref="soft00"))) A Python virtualenv (short for virtual environment) is how you set up your environment for different Python projects. It enables you to use different packages (e.g., different versions of Django, and even different versions of Python) in each project. And because you're not installing things system-wide, it means you don't need root [keep-together]#permissions#. Let's create a virtualenv. I'm assuming you're working in a folder called _goat-book_, but you can name your work folder whatever you like. Stick to the name ".venv" for the virtualenv, though: [subs=quotes] .on Windows: ---- $ *cd goat-book* $ *py -3.14 -m venv .venv* ---- On Windows, the `py` executable is a shortcut for different Python versions. On Mac or Linux, we use `python3.14`: [subs=quotes] .on Mac/Linux: ---- $ *cd goat-book* $ *python3.14 -m venv .venv* ---- ==== Activating and Deactivating the Virtualenv Whenever you're working through the book, you'll want to make sure your virtualenv has been "activated". You can always tell when your virtualenv is active because, in your prompt, you'll see `(.venv)` in parentheses. But you can also check by running `which python` to check whether Python is currently the system-installed one or the virtualenv one. The command to activate the virtualenv is `source .venv/Scripts/activate` on Windows and `source .venv/bin/activate` on Mac/Linux. The command to deactivate is just `deactivate`. [role="pagebreak-before"] Try it out like this, on Windows: [subs=quotes] ---- $ *source .venv/Scripts/activate* (.venv)$ (.venv)$ *which python* /C/Users/harry/goat-book/.venv/Scripts/python (.venv)$ *deactivate* $ $ *which python* /c/Users/harry/AppData/Local/Programs/Python/Python312-32/python ---- Or like this, on Mac/Linux: [subs=quotes] ---- $ *source .venv/bin/activate* (.venv)$ (.venv)$ *which python* /home/harry/goat-book/.venv/bin/python (.venv)$ *deactivate* $ $ *which python* /usr/bin/python ---- TIP: Always make sure your virtualenv is active when working on the book. Look out for the `(.venv)` in your prompt, or run `which python` to check. .Virtualenvs and IDEs ******************************************************************************* If you're using an IDE like PyCharm or Visual Studio Code, you should be able to configure them to use the virtualenv as the default Python interpreter for the project. You should then be able to launch a terminal inside the IDE with the virtualenv already activated. ******************************************************************************* ==== Installing Django and Selenium ((("Django framework", "installation"))) ((("Selenium", "installation"))) We'll install Django 5.2 and the latest Selenium.footnote:[ You might be wondering why I'm not mentioning a specific version of Selenium. It's because Selenium is constantly being updated to keep up with changes in web browsers, and as we can't really pin our browser to a specific version, we're best off using the latest Selenium. It was version 4.24 at the time of writing. ] Remember to make sure your virtualenv is active first! [subs="specialcharacters,quotes"] ---- (.venv) $ *pip install "django<6" "selenium"* Collecting django<6 Downloading Django-5.2.3-py3-none-any.whl (8.0 MB) ---------------------------------------- 8.1/8.1 MB 7.6 MB/s eta 0:00:00 Collecting selenium Downloading selenium-4.24.0-py3-none-any.whl (6.5 MB) ---------------------------------------- 6.5/6.5 MB 6.3 MB/s eta 0:00:00 Collecting asgiref>=3.8.1 (from django<6) Downloading asgiref-3.8.1-py3-none-any.whl.metadata (9.3 kB) Collecting sqlparse>=0.3.1 (from django<6)Collecting sqlparse>=0.3.1 (from django<6) [...] Installing collected packages: sortedcontainers, websocket-client, urllib3, typing_extensions, sqlparse, sniffio, pysocks, idna, h11, certifi, attrs, asgiref, wsproto, outcome, django, trio, trio-websocket, selenium Successfully installed asgiref-3.8.1 attrs-25.3.0 certifi-2025.4.26 django-5.2.3 [...] selenium-4.32.0 [...] ---- Check that it works: [subs="specialcharacters,quotes"] ---- (.venv) $ *python -c "from selenium import webdriver; webdriver.Firefox()"* ---- This should pop open a Firefox web browser, which you'll then need to close. TIP: If you see an error, you'll need to debug it before you go further. On Linux/Ubuntu, I ran into https://github.com/mozilla/geckodriver/issues/2010[a bug], which needs to be fixed by setting an environment variable called `TMPDIR`. ==== Some Error Messages You're Likely to See When You Inevitably Fail to Activate Your Virtualenv ((("troubleshooting", "virtualenv activation"))) If you're new to virtualenvs--or even if you're not, to be honest--at some point you're 'guaranteed' to forget to activate it, and then you'll be staring at an error message. Happens to me all the time. Here are some of the things to look out for: ---- ModuleNotFoundError: No module named 'selenium' ---- Or: ---- ModuleNotFoundError: No module named 'django' [...] ImportError: Couldn't import Django. Are you sure it's installed and available on your PYTHONPATH environment variable? Did you forget to activate a virtual environment? ---- As always, look out for that `(.venv)` in your command prompt, and a quick `source .venv/Scripts/activate` or `source .venv/bin/activate` is probably what you need to get it working again. Here's another, for good measure: ---- bash: .venv/Scripts/activate: No such file or directory ---- This means you're not currently in the right directory for working on the project. Try a `cd goat-book`, or similar. Alternatively, if you're sure you're in the right place, you may have run into a bug from an older version of Python, where it wouldn't install an activate script that was compatible with Git Bash. Reinstall Python 3, and make sure you have version 3.6.3 or later, and then delete and re-create your virtualenv. If you see something like this, it's probably the same issue and you need to upgrade Python: ---- bash: @echo: command not found bash: .venv/Scripts/activate.bat: line 4: syntax error near unexpected token `( bash: .venv/Scripts/activate.bat: line 4: `if not defined PROMPT (' ---- Final one! Consider this: ---- 'source' is not recognized as an internal or external command, operable program or batch file. ---- If you see this, it's because you've launched the default Windows command prompt, +cmd+, instead of Git Bash. Close it and open the latter. .On Anaconda ******************************************************************************* Anaconda is another tool for managing different Python environments. It's particularly popular on Windows and for scientific computing, where it can be hard to get some of the compiled libraries to install. In the world of web programming, it's much less necessary, so _I recommend you do not use Anaconda for this book_. ******************************************************************************* Happy coding! ((("", startref="prereq00"))) ((("", startref="TDDprereq00"))) ((("", startref="P3installvirt00"))) ((("", startref="VEinstall00"))) NOTE: Did these instructions not work for you? Or have you got better ones? Get in touch: obeythetestinggoat@gmail.com! ================================================ FILE: preface.asciidoc ================================================ [[preface]] [preface] == Preface This book has been my attempt to share with the world the journey I took from "hacking" to "software engineering". It's mainly about testing, but there's a lot more to it, as you'll soon see. I want to thank you for reading it. If you bought a copy, then I'm very grateful. If you're reading the free online version, then I'm _still_ grateful that you've decided it's worth spending your time on. Who knows; perhaps once you get to the end, you'll decide it's good enough to buy a physical copy for yourself or a friend. ((("contact information"))) ((("questions and comments"))) ((("comments and questions"))) ((("feedback"))) If you have any comments, questions, or suggestions, I'd love to hear from you. You can reach me directly via obeythetestinggoat@gmail.com, or on Mastodon https://fosstodon.org/@hjwp[@hjwp]. You can also check out http://www.obeythetestinggoat.com[the website and my blog]. I hope you'll enjoy reading this book as much as I enjoyed writing it. ////////////////////////////////////////// === Third Edition Early Release History tbc .Third Edition Early Release Information ******************************************************************************* If you can see this, you are reading an early release of the third edition, either via www.obeythetestinggoat.com, or via the O'Reilly Learning site. Congratulations! At the time of writing, all of the code listings in the main book (the chapters up to 25, but not the appendices) have been updated to Python 3.14 and Django 5. We're still in tech review, and many chapters still need a little work, but the core of the book is there. Thanks for reading, and please do send any and all feedback! At this early release stage, feedback is more important than ever. You can reach me via obeythetestinggoat@gmail.com ******************************************************************************* ////////////////////////////////////////// === Why I Wrote a Book About Test-Driven Development _“Who are you, why have you written this book, and why should I read it?”_ I hear you ask. //IDEA: tighten up this section ((("Test-Driven Development (TDD)", "need for", id="TDDneed00"))) I was lucky enough early on in my career to fall in with a bunch of test-driven development (TDD) fanatics, and it made such a big impact on my programming that I was burning to share it with everyone. You might say I had the enthusiasm of a recent convert, and the learning experience was still a recent memory for me, so that's what led to the first edition, back in 2014. When I first learned Python (from Mark Pilgrim's excellent https://diveintopython3.net[_Dive Into Python_]), I came across the concept of TDD, and thought, "Yes. I can definitely see the sense in that". Perhaps you had a similar reaction when you first heard about TDD? It seemed like a really sensible approach, a really good habit to get into--like regularly flossing your teeth. Then came my first big project, and you can guess what happened--there was a client, there were deadlines, there was lots to do, and any good intentions about TDD went straight out of the window. And, actually, it was fine. I was fine. At first. At first I thought I didn't really need TDD because the website was small, and I could easily test whether things worked by just manually checking it out. Click this link _here_, choose that drop-down item _there_, and _this_ should happen. Easy. This whole "writing tests" thing sounded like it would have taken _ages_. And besides, I fancied myself, from the full height of my three weeks of adult coding experience, as being a pretty good programmer. I could handle it. Easy. Then came the fearful goddess Complexity. She soon showed me the limits of my experience. The project grew. Parts of the system started to depend on other parts. I did my best to follow good principles like DRY (don't repeat yourself), but that just led to some pretty dangerous territory. Soon, I was playing with multiple inheritance. Class hierarchies eight levels deep. `eval` statements. I became scared of making changes to my code. I was no longer sure what depended on what, and what might happen if I changed this code _over here_...oh gosh, I think that bit over there inherits from it...no, it doesn't; it's overridden. Oh, but it depends on that class variable. Right, well, as long as I override the override it should be fine. I'll just check--but checking was getting much harder. There were lots of sections for the site now, and clicking through them all manually was starting to get impractical. Better to leave well enough alone. Forget refactoring. Just make do. Soon I had a hideous, ugly mess of code. New development became painful. Not too long after this, I was lucky enough to get a job with a company called Resolver Systems (now https://www.pythonanywhere.com[PythonAnywhere]), where https://martinfowler.com/bliki/ExtremeProgramming.html[extreme programming (XP)] was the norm. The people there introduced me to rigorous TDD. Although my previous experience had certainly opened my mind to the possible benefits of automated testing, I still dragged my feet at every stage. ``I mean, testing in general might be a good idea, but 'really'? All these tests? Some of them seem like a total waste of time...what? Functional tests _as well as_ unit tests? Come on, that's overdoing it! And this TDD test/minimal-code-change/test cycle? This is just silly! We don't need all these baby steps! Come on—we can see what the right answer is; why don't we just skip to the end?'' Believe me, I second-guessed every rule, I suggested every shortcut, I demanded justifications for every seemingly pointless aspect of TDD—and I still came out seeing the wisdom of it all. I've lost count of the number of times I've thought, ``Thanks, tests'',footnote:[ https://oreil.ly/LGP3g[Thests].] as a functional test uncovers a regression we would never have predicted, or a unit test saves me from making a really silly logic error. Psychologically, it's made development a much less stressful process. It produces code that's a pleasure to work with.((("", startref="TDDneed00"))) So, let me tell you _all_ about it! === Aims of This Book My main aim is to impart a methodology--a way of doing web development, which I think makes for better web apps and happier developers. There's not much point in a book that just covers material you could find by googling, so this book isn't a guide to Python syntax, nor a tutorial on web development per se. Instead, I hope to teach you how to use TDD to get more reliably to our shared, holy goal: _clean code that works._ With that said: I will constantly refer to a real practical example, by building a web app from scratch using tools like Django, Selenium, jQuery, and mocks. I'm not assuming any prior knowledge of any of these, so you should come out the other end of this book with a decent introduction to those tools, as well as the discipline of TDD. In extreme programming we always pair-program, so I've imagined writing this book as if I was pairing with my previous self, having to explain how the tools work and answer questions about why we code in this particular way. So, if I ever take a bit of a patronising tone, it's because I'm not all that smart, and I have to be very patient with myself. And if I ever sound defensive, it's because I'm the kind of annoying person that systematically disagrees with whatever anyone else says, so sometimes it takes a lot of justifying to convince myself of anything. [role="pagebreak-before less_space"] === Outline I've split this book into four parts. <> (Chapters <> to <>): The Basics of TDD and Django:: We dive straight into building a simple web app using TDD. We start by writing a functional test (with Selenium), and then we go through the basics of Django--models, views, templates--with rigorous unit testing at every stage. I also introduce the Testing Goat. <> (Chapters <> to <>): Going to Production:: These chapters are all about deploying your web app to an actual server. We discuss how our tests, and the TDD practice of working incrementally, can take a lot of the pain and risk out of what is normally quite a fraught process. <> (Chapters <> to <>): Forms and Validation:: Here, we get into some of the details of the Django Forms framework, implementing validation, and data integrity using database constraints. We discuss using tests to explore unfamiliar APIs, and the limits of frameworks. <> (Chapters <> to <>): Advanced Topics in Testing:: Covers some of the more advanced topics in TDD, including spiking (where we relax the rules of TDD temporarily), mocking, working outside-in, and continuous integration (CI). Now, onto a little housekeeping... === Conventions Used in This Book ((("typographical conventions")))The following typographical conventions are used in this book: _Italic_:: Indicates new terms, URLs, email addresses, filenames, and file extensions `Constant width`:: Used for program listings and within paragraphs to refer to program elements such as variable or function names, databases, data types, environment variables, statements, and keywords +*Constant width bold*+:: Shows commands or other text that should be typed literally by the user [role="pagebreak-before"] Occasionally I will use the symbol: [subs="specialcharacters,quotes"] ---- [...] ---- to signify that some of the content has been skipped, to shorten long bits of output, or to skip down to a relevant section. You will also encounter the following callouts: TIP: This element signifies a tip or suggestion. NOTE: This element signifies a general note or aside. WARNING: This element indicates a warning or caution. === Submitting Errata ((("errata")))Spotted a mistake or a typo? The sources for this book are available on GitHub, and I'm always very happy to receive issues and pull requests: https://github.com/hjwp/Book-TDD-Web-Dev-Python[]. === Using Code Examples ((("code examples, obtaining and using")))Code examples are available at https://github.com/hjwp/book-example/[]; you'll find branches for each chapter there (e.g., https://github.com/hjwp/book-example/tree/chapter_03_unit_test_first_view[]). You can find a full list and some suggestions on ways of working with this repository in <>. This book is here to help you get your job done. In general, if example code is offered with this book, you may use it in your programs and documentation. You do not need to contact us for permission unless you’re reproducing a significant portion of the code. For example, writing a program that uses several chunks of code from this book does not require permission. Selling or distributing examples from O’Reilly books does require permission. Answering a question by citing this book and quoting example code does not require permission. Incorporating a significant amount of example code from this book into your product’s documentation does require permission. We appreciate, but do not require, attribution. An attribution usually includes the title, author, publisher, and ISBN. For example: “_Test-Driven Development with Python_, 3rd edition, by Harry J.W. Percival (O’Reilly). Copyright 2025 Harry Percival, 978-1-098-14871-3”. If you feel your use of code examples falls outside fair use or the permission given above, feel free to contact us at pass:[]. === O'Reilly Online Learning [role = "ormenabled"] [NOTE] ==== For more than 40 years, pass:[O’Reilly Media] has provided technology and business training, knowledge, and insight to help companies succeed. ==== Our unique network of experts and innovators share their knowledge and expertise through books, articles, and our online learning platform. O’Reilly’s online learning platform gives you on-demand access to live training courses, in-depth learning paths, interactive coding environments, and a vast collection of text and video from O'Reilly and 200+ other publishers. For more information, visit pass:[https://oreilly.com]. === How to Contact Us Please address comments and questions concerning this book to the publisher: ++++ ++++ We have a web page for this book, where we list errata, examples, and any additional information. You can access this page at link:$$https://oreil.ly/TDD-with-python-3e$$[]. ++++ ++++ For news and information about our books and courses, visit link:$$https://oreilly.com$$[]. Find us on LinkedIn: link:$$https://linkedin.com/company/oreilly-media$$[]. Watch us on YouTube: link:$$https://youtube.com/oreillymedia$$[]. [role="pagebreak-before less_space"] [[video_plug]] === Companion Video ((("companion video")))((("video-based instruction")))((("Test-Driven Development (TDD)", "video-based instruction"))) I've recorded a https://learning.oreilly.com/videos/test-driven-development/9781491919163[10-part video series to accompany this book].footnote:[The video has not been updated for the third edition, but the content is all mostly the same.] It covers the content of <>. If you find that you learn well from video-based material, then I encourage you to check it out. Over and above what's in the book, it should give you a feel for what the "flow" of TDD is like, flicking between tests and code and explaining the thought process as we go. Plus, I'm wearing a delightful yellow t-shirt. [[video-screengrab]] image::images/tdd3_00in01.png[screengrab from video] [role="pagebreak-before less_space"] === License for the Free Edition If you're reading the free edition of this book hosted at http://www.obeythetestinggoat.com, then the license is https://creativecommons.org/licenses/by-nc-nd/4.0/legalcode[Creative Commons Attribution-NonCommercial-NoDerivatives].footnote:[The no-derivs clause is there because O'Reilly wants to maintain some control over derivative works, but it often does grant permissions for things, so don't hesitate to get in touch if you want to build something based on this book.] I want to thank O'Reilly for its fantastic attitude towards licensing; most publishers aren't so forward-thinking. I see this as a "try before you buy" scheme really. If you're reading this book it's for professional reasons, so I hope that if you like it, you'll buy a copy--if not for yourself, then for a friend! O'Reilly has been great, it deserves your support. You'll find http://www.obeythetestinggoat.com[links to buy back on the home page]. ================================================ FILE: pygments-default.css ================================================ pre.pygments .hll { background-color: #ffffcc } pre.pygments { background: #f8f8f8; } pre.pygments .tok-c { color: #3D7B7B; font-style: italic } /* Comment */ pre.pygments .tok-err { border: 1px solid #FF0000 } /* Error */ pre.pygments .tok-k { color: #008000; font-weight: bold } /* Keyword */ pre.pygments .tok-o { color: #666666 } /* Operator */ pre.pygments .tok-ch { color: #3D7B7B; font-style: italic } /* Comment.Hashbang */ pre.pygments .tok-cm { color: #3D7B7B; font-style: italic } /* Comment.Multiline */ pre.pygments .tok-cp { color: #9C6500 } /* Comment.Preproc */ pre.pygments .tok-cpf { color: #3D7B7B; font-style: italic } /* Comment.PreprocFile */ pre.pygments .tok-c1 { color: #3D7B7B; font-style: italic } /* Comment.Single */ pre.pygments .tok-cs { color: #3D7B7B; font-style: italic } /* Comment.Special */ pre.pygments .tok-gd { color: #A00000 } /* Generic.Deleted */ pre.pygments .tok-ge { font-style: italic } /* Generic.Emph */ pre.pygments .tok-ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */ pre.pygments .tok-gr { color: #E40000 } /* Generic.Error */ pre.pygments .tok-gh { color: #000080; font-weight: bold } /* Generic.Heading */ pre.pygments .tok-gi { color: #008400 } /* Generic.Inserted */ pre.pygments .tok-go { color: #717171 } /* Generic.Output */ pre.pygments .tok-gp { color: #000080; font-weight: bold } /* Generic.Prompt */ pre.pygments .tok-gs { font-weight: bold } /* Generic.Strong */ pre.pygments .tok-gu { color: #800080; font-weight: bold } /* Generic.Subheading */ pre.pygments .tok-gt { color: #0044DD } /* Generic.Traceback */ pre.pygments .tok-kc { color: #008000; font-weight: bold } /* Keyword.Constant */ pre.pygments .tok-kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ pre.pygments .tok-kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ pre.pygments .tok-kp { color: #008000 } /* Keyword.Pseudo */ pre.pygments .tok-kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ pre.pygments .tok-kt { color: #B00040 } /* Keyword.Type */ pre.pygments .tok-m { color: #666666 } /* Literal.Number */ pre.pygments .tok-s { color: #BA2121 } /* Literal.String */ pre.pygments .tok-na { color: #687822 } /* Name.Attribute */ pre.pygments .tok-nb { color: #008000 } /* Name.Builtin */ pre.pygments .tok-nc { color: #0000FF; font-weight: bold } /* Name.Class */ pre.pygments .tok-no { color: #880000 } /* Name.Constant */ pre.pygments .tok-nd { color: #AA22FF } /* Name.Decorator */ pre.pygments .tok-ni { color: #717171; font-weight: bold } /* Name.Entity */ pre.pygments .tok-ne { color: #CB3F38; font-weight: bold } /* Name.Exception */ pre.pygments .tok-nf { color: #0000FF } /* Name.Function */ pre.pygments .tok-nl { color: #767600 } /* Name.Label */ pre.pygments .tok-nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ pre.pygments .tok-nt { color: #008000; font-weight: bold } /* Name.Tag */ pre.pygments .tok-nv { color: #19177C } /* Name.Variable */ pre.pygments .tok-ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ pre.pygments .tok-w { color: #bbbbbb } /* Text.Whitespace */ pre.pygments .tok-mb { color: #666666 } /* Literal.Number.Bin */ pre.pygments .tok-mf { color: #666666 } /* Literal.Number.Float */ pre.pygments .tok-mh { color: #666666 } /* Literal.Number.Hex */ pre.pygments .tok-mi { color: #666666 } /* Literal.Number.Integer */ pre.pygments .tok-mo { color: #666666 } /* Literal.Number.Oct */ pre.pygments .tok-sa { color: #BA2121 } /* Literal.String.Affix */ pre.pygments .tok-sb { color: #BA2121 } /* Literal.String.Backtick */ pre.pygments .tok-sc { color: #BA2121 } /* Literal.String.Char */ pre.pygments .tok-dl { color: #BA2121 } /* Literal.String.Delimiter */ pre.pygments .tok-sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ pre.pygments .tok-s2 { color: #BA2121 } /* Literal.String.Double */ pre.pygments .tok-se { color: #AA5D1F; font-weight: bold } /* Literal.String.Escape */ pre.pygments .tok-sh { color: #BA2121 } /* Literal.String.Heredoc */ pre.pygments .tok-si { color: #A45A77; font-weight: bold } /* Literal.String.Interpol */ pre.pygments .tok-sx { color: #008000 } /* Literal.String.Other */ pre.pygments .tok-sr { color: #A45A77 } /* Literal.String.Regex */ pre.pygments .tok-s1 { color: #BA2121 } /* Literal.String.Single */ pre.pygments .tok-ss { color: #19177C } /* Literal.String.Symbol */ pre.pygments .tok-bp { color: #008000 } /* Name.Builtin.Pseudo */ pre.pygments .tok-fm { color: #0000FF } /* Name.Function.Magic */ pre.pygments .tok-vc { color: #19177C } /* Name.Variable.Class */ pre.pygments .tok-vg { color: #19177C } /* Name.Variable.Global */ pre.pygments .tok-vi { color: #19177C } /* Name.Variable.Instance */ pre.pygments .tok-vm { color: #19177C } /* Name.Variable.Magic */ pre.pygments .tok-il { color: #666666 } /* Literal.Number.Integer.Long */ ================================================ FILE: pyproject.toml ================================================ [build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" requires-python = ">=3.14" [project] name = "goat-book" version = "0" # most requts are deliberately unpinned so we stay up to date with deps, # and CI will warn when things change. dependencies = [ "requests", "lxml", "lxml-stubs", "cssselect", "django<6", "django-types", "pygments", "docopt", "requests", "selenium<5", "pytest", # "pytest-xdist", "ruff", "black", # needed as a marker to tell django to use black "whitenoise", # from chap 10 on ] [tool.setuptools] packages = [] [tool.ruff.lint] select = [ # pycodestyle "E", # Pyflakes "F", # pyupgrade "UP", # flake8-bugbear "B", # flake8-simplify "SIM", # isort "I", # pylint "PL", ] ignore = [ "E741", # single-letter variable "E731", # allow lambdas "PLR2004", # magic values ] [tool.pyright] # these help pyright in neovide to find its way around venvPath = "." venv = ".venv" # most of the source for the book itself is untyped typeCheckingMode = "standard" [tool.pytest.ini_options] # -r N disables the post-test summary addopts = ["--tb=short", "-r N", "--color=yes"] ================================================ FILE: rename-chapter.sh ================================================ #!/bin/bash set -eux set -o pipefail OLD_CHAPTER=$1 NEW_NAME=$2 git mv "$OLD_CHAPTER.asciidoc" "$NEW_NAME.asciidoc" mv "$OLD_CHAPTER.html" "$NEW_NAME.html" || touch "$NEW_NAME.html" if [ -e "tests/test_$OLD_CHAPTER.py" ]; then git mv "tests/test_$OLD_CHAPTER.py" "tests/test_$NEW_NAME.py" fi git mv "source/$OLD_CHAPTER" "source/$NEW_NAME" cd "source/$NEW_NAME/superlists" git switch "$OLD_CHAPTER" git switch -c "$NEW_NAME" git push -u local git push -u origin cd ../../.. git grep -l "$OLD_CHAPTER" | xargs sed -i '' "s/$OLD_CHAPTER/$NEW_NAME/g" # make "test_$NEW_NAME" || echo -e "\a" echo git commit -am \'rename "$OLD_CHAPTER" to "$NEW_NAME".\' ================================================ FILE: research/js-testing.rst ================================================ Options: Qunit -- weird syntax, but seems popular YUI -- familiar jasmine - seems sane js-test-driver: by google, but seems to be server-only Sinon - for mocking http://www.letscodejavascript.com/ ================================================ FILE: run_test_tests.sh ================================================ #!/bin/bash PYTHONHASHSEED=0 pytest \ --failed-first \ --tb=short \ -k 'not test_listings_and_commands_and_output' \ "$@" \ tests/ # py.test --tb=short `ls tests/test_* | grep -v test_chapter | grep -v test_server` ================================================ FILE: server-quickstart.md ================================================ # Ultra-brief instructions for how to get a Linux server These instructions are meant as companion to the [server prep chapter of my book](https://www.obeythetestinggoat.com/book/chapter_11_server_prep.html). They're almost telegraphic in style, but I hope they're better than nothing! ## Use Digital Ocean I didn't want to make a specific recommendation in the book itself, but I'll make one here. I like [Digital Ocean](https://m.do.co/c/876844cd6b2e). Good value for money, fast servers, and you can get a couple of months' worth of free credit by following [my referral link](https://m.do.co/c/876844cd6b2e). ## Generate an SSH key SSH aka "secure shell" is a protocol for running a terminal session on a remote server, across the network. It involves authentication, as you might expect, and different types of it. You can use usernames and passwords, but public/private key authentication is more convenient, and (as always, arguably) more secure. If you've never created one before, the command is ```bash ssh-keygen ``` **NOTE** _If you're on Windows, you need to be using Git-Bash for `ssh-keygen` and `ssh` to work. There's more info in the [installation instructions chapter](https://www.obeythetestinggoat.com/book/pre-requisite-installations.html)_ Just accept all the defaults if you really want to just get started in a hurry, and no passphrase. Later on, you'll want to re-create a key with a passphrase for extra security, but that means you have to figure out how to save that passphrase in such a way that Ansible won't ask for it later, and I don't have time to write instructions for that now! Make a note of your "public key" ```bash cat ~/.ssh/id_rsa.pub ``` More info on public key authentication [here](https://www.linode.com/docs/guides/use-public-key-authentication-with-ssh/) and [here](https://docs.digitalocean.com/products/droplets/how-to/add-ssh-keys/) ## Start a Server aka VM aka Droplet A "droplet" is Digital Ocean's name for a server. Pick the default Ubuntu, the cheapest type, and whichever region is closest to you. You won't need access to the ancillary services that are available (Block storage, a VPC network, IPv6, User-Data, Monitoring, Back-Ups, etc, don't worry about those) * Choose **New SSH Key** and upload your public key from above Make a note of your server's IP address once it's started ## Log in for the first time ```bash ssh root@your-server-ip-address-here ``` It should just magically find your SSH key and log you in without any need for a password. Hooray! ## Create a non-root user It's good security practice to avoid using the root user directly. Let's create a non-root user with super-user ("sudo") privileges, a bit like you have on your own machine. ```bash useradd -m -s /bin/bash elspeth # add user named elspeth # -m creates a home folder, -s sets elspeth to use bash by default usermod -a -G sudo elspeth # add elspeth to the sudoers group # allow elspeth to sudo without retyping password echo 'elspeth ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/elspeth # set password for elspeth (you'll need to type one in) passwd elspeth su - elspeth # switch-user to being elspeth! ``` ## Add your public key to the non-root user as well. * Copy your public key to your clipboard, and then ```bash # as user elspeth mkdir -p ~/.ssh echo 'PASTE YOUR PUBLIC KEY HERE' >> ~/.ssh/authorized_keys ``` Now log out from the server, and verify you can SSH in as elspeth from your laptop ```bash ssh elspeth@your-server-ip-address-here ``` Also check you can use "sudo" as elspeth ```bash sudo echo hi ``` ## Registering a domain name There's one more thing you need to do in the book, which is to map a domain name to your server's IP address. If you don't already own a domain name you can use (you don't have to use the *www.* subdomain, you could use *superlists.yourdomain.com*), then you'll need to get one from a "domain registrar". There are loads out there, I quite like Gandi or the slightly-more-friendly 123-reg. There aren't any registrars offering free domain names any more, but the cheapest registrar I've found is https://www.ionos.co.uk/, where last I checked you could get a domain for one pound, like $1.50, for a year. But I haven't used them myself personally. # Pull requests and suggestions accepted! I literally threw these instructions together in 10 minutes flat, so I'm sure they could do with improvements. Please send in suggestions, typos, fixes, any common "gotchas" you ran into that you think I should mention. ================================================ FILE: source/blackify-chap.sh ================================================ #!/bin/bash set -e PREV=$1 CHAP=$2 # assumes a git remote local pointing at a local bare repo... REPO=local cd $CHAP/superlists git fetch $REPO STARTCOMMIT="$(git rev-parse $PREV)" ENDCOMMIT="$(git rev-parse $CHAP)" git switch $CHAP git reset --hard $REPO/$PREV ruff format . git commit -am"initial black commit" --allow-empty git rev-list $STARTCOMMIT^..$ENDCOMMIT| tail -r | xargs -n1 sh -c 'git co $0 -- . && ruff format . && git add . && git stwdiff && git commit -am "$(git show -s --format=%B $0)"' git diff -w $REPO/$CHAP cd ../.. ================================================ FILE: source/feed-thru-cherry-picks.sh ================================================ #!/bin/bash # replay all the commits for a given chapter ($CHAP) # onto the latest version of the previous chapter ($PREV) set -e PREV=$1 CHAP=$2 # assumes a git remote called "local" (eg pointing at a local bare repo...) # rather than using origin/github, # this allows us to feed through changes without pushing to github REPO=local cd "$CHAP/superlists" # determine the commit we want to start from, # which is the last commit of the $PREV branch as it was *before* our new version. # We assume that the repo version in $CHAP/superlists has *not* got this latest version yet. # (so that's why we don't do the git fetch until after this step) START_COMMIT="$(git rev-list -n 1 "$REPO/$PREV")" CHAP_COMMIT_LIST="$START_COMMIT..$REPO/$CHAP" # check START_COMMIT exists in $CHAP branch's history # https://stackoverflow.com/a/4129070/366221 if [ "$(git merge-base "$START_COMMIT" "$CHAP")" != "$START_COMMIT" ]; then echo "Error: $START_COMMIT is not in the history of $CHAP" exit 1 fi # now we pull down the latest version of $PREV git fetch "$REPO" # reset our chapter to the new version of the end of $PREV git switch "$CHAP" git reset --hard "$REPO/$PREV" # now cherry pick all the old commits from $CHAP onto this new base. # (we can't just use rebase because it does the wrong thing with trying to find common history) git cherry-pick -Xrename-threshold=20% "$CHAP_COMMIT_LIST" # display a little diff to sanity-check what we've done. git diff -w "$REPO/$CHAP" cd ../.. ================================================ FILE: source/fix-commit-numbers.py ================================================ #!/usr/bin/env python # use with # FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --msg-filter $PWD/../../fix-commit-numbers.py 3fc31f1b.. import re import sys while incoming := sys.stdin.readline(): # if m := re.match(r"(.+) --ch(\d+)l(\d+)(-?)(\d?)--", incoming.rstrip()): if m := re.match(r"(.+) (\(|--)ch(\d+)l(\d+)(-?)(\d?)(\)|--)", incoming.rstrip()): prefix, sep1, chap_num, listing_num, extra_dash, suffix, sep2 = m.groups() chap_num = int(chap_num) listing_num = int(listing_num) suffix = int(suffix) if suffix else None if chap_num == 14: pass elif suffix and listing_num == 30: listing_num = listing_num - 13 + suffix if listing_num == 30: assert suffix is None listing_num = 21 elif listing_num == 31: assert suffix is None listing_num = 22 elif listing_num == 32 and suffix == "1": listing_num = 23 elif listing_num == 32 and suffix == "2": listing_num = 24 elif listing_num == 33 and suffix is None: listing_num = 25 elif listing_num == 33 and suffix: listing_num = 26 elif listing_num == 34: listing_num = 27 elif listing_num == 35: listing_num = 28 elif listing_num == 36: assert suffix listing_num = 28 + suffix print(f"{prefix} {sep1}ch{chap_num}l{listing_num:03d}{sep2}") else: print(incoming, end="") ================================================ FILE: source/push-back.sh ================================================ #!/bin/bash set -e CHAP=$1 cd "$CHAP/superlists" git push --force-with-lease local "$CHAP" git push --force-with-lease origin "$CHAP" cd ../.. ================================================ FILE: tests/actual_manage_py_test.output ================================================ EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE..s.............EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE..............................EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE...............................EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE.EEEEEEEEEEEEEEEEEEEEE ====================================================================== ERROR: test_get_all_permissions (django.contrib.auth.tests.auth_backends.AnonymousUserBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_module_perms (django.contrib.auth.tests.auth_backends.AnonymousUserBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_perm (django.contrib.auth.tests.auth_backends.AnonymousUserBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_perms (django.contrib.auth.tests.auth_backends.AnonymousUserBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_message_attrs (django.contrib.auth.tests.context_processors.AuthContextProcessorTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_perm_in_perms_attrs (django.contrib.auth.tests.context_processors.AuthContextProcessorTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_perms_attrs (django.contrib.auth.tests.context_processors.AuthContextProcessorTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_is_accessed (django.contrib.auth.tests.context_processors.AuthContextProcessorTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_not_accessed (django.contrib.auth.tests.context_processors.AuthContextProcessorTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_user_attrs (django.contrib.auth.tests.context_processors.AuthContextProcessorTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_named_urls (django.contrib.auth.tests.views.AuthViewNamedURLTests) Named URLs should be reversible ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_inactive_user (django.contrib.auth.tests.forms.AuthenticationFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_inactive_user_i18n (django.contrib.auth.tests.forms.AuthenticationFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_invalid_username (django.contrib.auth.tests.forms.AuthenticationFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_success (django.contrib.auth.tests.forms.AuthenticationFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_username_field_label (django.contrib.auth.tests.forms.AuthenticationFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_anonymous_user (django.contrib.auth.tests.basic.BasicTestCase) Check the properties of the anonymous user ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_createsuperuser_management_command (django.contrib.auth.tests.basic.BasicTestCase) Check the operation of the createsuperuser management command ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_createsuperuser_nolocale (django.contrib.auth.tests.basic.BasicTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_createsuperuser_non_ascii_verbose_name (django.contrib.auth.tests.basic.BasicTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_user_model (django.contrib.auth.tests.basic.BasicTestCase) The current user model can be retrieved ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_superuser (django.contrib.auth.tests.basic.BasicTestCase) Check the creation and properties of a superuser ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_swappable_user (django.contrib.auth.tests.basic.BasicTestCase) The current user model can be swapped out for another ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_swappable_user_bad_setting (django.contrib.auth.tests.basic.BasicTestCase) The alternate user setting must point to something in the format app.model ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_swappable_user_nonexistent_model (django.contrib.auth.tests.basic.BasicTestCase) The current user model must point to an installed model ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_user (django.contrib.auth.tests.basic.BasicTestCase) Check that users can be created and can set their password ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_user_no_email (django.contrib.auth.tests.basic.BasicTestCase) Check that users can be created without an email ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_password_change_done_fails (django.contrib.auth.tests.views.ChangePasswordTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_password_change_done_succeeds (django.contrib.auth.tests.views.ChangePasswordTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_password_change_fails_with_invalid_old_password (django.contrib.auth.tests.views.ChangePasswordTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_password_change_fails_with_mismatched_passwords (django.contrib.auth.tests.views.ChangePasswordTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_password_change_succeeds (django.contrib.auth.tests.views.ChangePasswordTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_changelist_disallows_password_lookups (django.contrib.auth.tests.views.ChangelistTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_that_changepassword_command_changes_joes_password (django.contrib.auth.tests.management.ChangepasswordManagementCommandTestCase) Executing the changepassword management command should change joe's password ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_that_max_tries_exits_1 (django.contrib.auth.tests.management.ChangepasswordManagementCommandTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_createsuperuser (django.contrib.auth.tests.management.CreatesuperuserManagementCommandTestCase) Check the operation of the createsuperuser management command ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_email_in_username (django.contrib.auth.tests.management.CreatesuperuserManagementCommandTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_swappable_user (django.contrib.auth.tests.management.CreatesuperuserManagementCommandTestCase) A superuser can be created when a custom User model is in use ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_swappable_user_missing_required_field (django.contrib.auth.tests.management.CreatesuperuserManagementCommandTestCase) A Custom superuser won't be created when a required field isn't provided ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_verbosity_zero (django.contrib.auth.tests.management.CreatesuperuserManagementCommandTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_perms (django.contrib.auth.tests.auth_backends.CustomPermissionsUserModelBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_all_superuser_permissions (django.contrib.auth.tests.auth_backends.CustomPermissionsUserModelBackendTest) A superuser has all permissions. Refs #14795 ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_no_object_perm (django.contrib.auth.tests.auth_backends.CustomPermissionsUserModelBackendTest) Regressiontest for #12462 ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_perm (django.contrib.auth.tests.auth_backends.CustomPermissionsUserModelBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_authenticate (django.contrib.auth.tests.auth_backends.CustomUserModelBackendAuthenticateTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_username_non_unique (django.contrib.auth.tests.management.CustomUserModelValidationTestCase) A non-unique USERNAME_FIELD should raise a model validation error. ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_username_not_in_required_fields (django.contrib.auth.tests.management.CustomUserModelValidationTestCase) USERNAME_FIELD should not appear in REQUIRED_FIELDS. ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_confirm_valid_custom_user (django.contrib.auth.tests.views.CustomUserPasswordResetTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_perms (django.contrib.auth.tests.auth_backends.ExtensionUserModelBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_all_superuser_permissions (django.contrib.auth.tests.auth_backends.ExtensionUserModelBackendTest) A superuser has all permissions. Refs #14795 ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_no_object_perm (django.contrib.auth.tests.auth_backends.ExtensionUserModelBackendTest) Regressiontest for #12462 ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_perm (django.contrib.auth.tests.auth_backends.ExtensionUserModelBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_actual_implementation (django.contrib.auth.tests.management.GetDefaultUsernameTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing (django.contrib.auth.tests.management.GetDefaultUsernameTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_i18n (django.contrib.auth.tests.management.GetDefaultUsernameTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_simple (django.contrib.auth.tests.management.GetDefaultUsernameTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_module_perms (django.contrib.auth.tests.auth_backends.InActiveUserBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_perm (django.contrib.auth.tests.auth_backends.InActiveUserBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_builtin_user_isactive (django.contrib.auth.tests.models.IsActiveTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_is_active_field_default (django.contrib.auth.tests.models.IsActiveTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_user_is_created_and_added_to_group (django.contrib.auth.tests.models.LoadDataWithNaturalKeysTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_user_is_created_and_added_to_group (django.contrib.auth.tests.models.LoadDataWithoutNaturalKeysTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: testCallable (django.contrib.auth.tests.decorators.LoginRequiredTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: testLoginRequired (django.contrib.auth.tests.decorators.LoginRequiredTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: testLoginRequiredNextUrl (django.contrib.auth.tests.decorators.LoginRequiredTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: testView (django.contrib.auth.tests.decorators.LoginRequiredTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_current_site_in_context_after_login (django.contrib.auth.tests.views.LoginTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_security_check (django.contrib.auth.tests.views.LoginTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_https_login_url (django.contrib.auth.tests.views.LoginURLSettings) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_login_url_with_querystring (django.contrib.auth.tests.views.LoginURLSettings) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_remote_login_url (django.contrib.auth.tests.views.LoginURLSettings) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_remote_login_url_with_next_querystring (django.contrib.auth.tests.views.LoginURLSettings) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_standard_login_url (django.contrib.auth.tests.views.LoginURLSettings) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_14377 (django.contrib.auth.tests.views.LogoutTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_logout_default (django.contrib.auth.tests.views.LogoutTest) Logout without next_page option renders the default template ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_logout_with_custom_redirect_argument (django.contrib.auth.tests.views.LogoutTest) Logout with custom query string redirects to specified resource ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_logout_with_next_page_specified (django.contrib.auth.tests.views.LogoutTest) Logout with next_page option given redirects to specified resource ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_logout_with_overridden_redirect_url (django.contrib.auth.tests.views.LogoutTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_logout_with_redirect_argument (django.contrib.auth.tests.views.LogoutTest) Logout with query string redirects to specified resource ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_security_check (django.contrib.auth.tests.views.LogoutTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_check_password (django.contrib.auth.tests.handlers.ModWsgiHandlerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/contrib/auth/tests/handlers.py", line 21, in test_check_password User.objects.create_user('test', 'test@example.com', 'test') File "/usr/local/lib/python2.7/dist-packages/django/db/models/manager.py", line 256, in __get__ self.model._meta.object_name, self.model._meta.swapped AttributeError: Manager isn't available; User has been swapped for 'auth.ExtensionUser' ====================================================================== ERROR: test_check_password (django.contrib.auth.tests.handlers.ModWsgiHandlerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 268, in __call__ self._post_teardown() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 533, in _post_teardown self._fixture_teardown() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 553, in _fixture_teardown skip_validation=True, reset_sequences=False) File "/usr/local/lib/python2.7/dist-packages/django/core/management/__init__.py", line 161, in call_command return klass.execute(*args, **defaults) File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 255, in execute output = self.handle(*args, **options) File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 385, in handle return self.handle_noargs(**options) File "/usr/local/lib/python2.7/dist-packages/django/core/management/commands/flush.py", line 46, in handle_noargs sql_list = sql_flush(self.style, connection, only_django=True, reset_sequences=reset_sequences) File "/usr/local/lib/python2.7/dist-packages/django/core/management/sql.py", line 113, in sql_flush tables = connection.introspection.django_table_names(only_existing=True) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 995, in django_table_names existing_tables = self.table_names() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 965, in table_names cursor = self.connection.cursor() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_check_password_custom_user (django.contrib.auth.tests.handlers.ModWsgiHandlerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 220, in inner return test_func(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/django/contrib/auth/tests/handlers.py", line 45, in test_check_password_custom_user CustomUser._default_manager.create_user('test@example.com', '1990-01-01', 'test') File "/usr/local/lib/python2.7/dist-packages/django/contrib/auth/tests/custom_user.py", line 29, in create_user user.save(using=self._db) File "/usr/local/lib/python2.7/dist-packages/django/db/models/base.py", line 546, in save force_update=force_update, update_fields=update_fields) File "/usr/local/lib/python2.7/dist-packages/django/db/models/base.py", line 650, in save_base result = manager._insert([self], fields=fields, return_id=update_pk, using=using, raw=raw) File "/usr/local/lib/python2.7/dist-packages/django/db/models/manager.py", line 215, in _insert return insert_query(self.model, objs, fields, **kwargs) File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 1661, in insert_query return query.get_compiler(using=using).execute_sql(return_id) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 935, in execute_sql cursor = self.connection.cursor() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_check_password_custom_user (django.contrib.auth.tests.handlers.ModWsgiHandlerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 268, in __call__ self._post_teardown() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 533, in _post_teardown self._fixture_teardown() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 553, in _fixture_teardown skip_validation=True, reset_sequences=False) File "/usr/local/lib/python2.7/dist-packages/django/core/management/__init__.py", line 161, in call_command return klass.execute(*args, **defaults) File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 255, in execute output = self.handle(*args, **options) File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 385, in handle return self.handle_noargs(**options) File "/usr/local/lib/python2.7/dist-packages/django/core/management/commands/flush.py", line 46, in handle_noargs sql_list = sql_flush(self.style, connection, only_django=True, reset_sequences=reset_sequences) File "/usr/local/lib/python2.7/dist-packages/django/core/management/sql.py", line 113, in sql_flush tables = connection.introspection.django_table_names(only_existing=True) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 995, in django_table_names existing_tables = self.table_names() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 965, in table_names cursor = self.connection.cursor() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_groups_for_user (django.contrib.auth.tests.handlers.ModWsgiHandlerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/contrib/auth/tests/handlers.py", line 62, in test_groups_for_user user1 = User.objects.create_user('test', 'test@example.com', 'test') File "/usr/local/lib/python2.7/dist-packages/django/contrib/auth/models.py", line 186, in create_user user.save(using=self._db) File "/usr/local/lib/python2.7/dist-packages/django/db/models/base.py", line 546, in save force_update=force_update, update_fields=update_fields) File "/usr/local/lib/python2.7/dist-packages/django/db/models/base.py", line 650, in save_base result = manager._insert([self], fields=fields, return_id=update_pk, using=using, raw=raw) File "/usr/local/lib/python2.7/dist-packages/django/db/models/manager.py", line 215, in _insert return insert_query(self.model, objs, fields, **kwargs) File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 1661, in insert_query return query.get_compiler(using=using).execute_sql(return_id) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 935, in execute_sql cursor = self.connection.cursor() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_groups_for_user (django.contrib.auth.tests.handlers.ModWsgiHandlerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 268, in __call__ self._post_teardown() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 533, in _post_teardown self._fixture_teardown() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 553, in _fixture_teardown skip_validation=True, reset_sequences=False) File "/usr/local/lib/python2.7/dist-packages/django/core/management/__init__.py", line 161, in call_command return klass.execute(*args, **defaults) File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 255, in execute output = self.handle(*args, **options) File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 385, in handle return self.handle_noargs(**options) File "/usr/local/lib/python2.7/dist-packages/django/core/management/commands/flush.py", line 46, in handle_noargs sql_list = sql_flush(self.style, connection, only_django=True, reset_sequences=reset_sequences) File "/usr/local/lib/python2.7/dist-packages/django/core/management/sql.py", line 113, in sql_flush tables = connection.introspection.django_table_names(only_existing=True) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 995, in django_table_names existing_tables = self.table_names() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 965, in table_names cursor = self.connection.cursor() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_perms (django.contrib.auth.tests.auth_backends.ModelBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_all_superuser_permissions (django.contrib.auth.tests.auth_backends.ModelBackendTest) A superuser has all permissions. Refs #14795 ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_no_object_perm (django.contrib.auth.tests.auth_backends.ModelBackendTest) Regressiontest for #12462 ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_perm (django.contrib.auth.tests.auth_backends.ModelBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_group_natural_key (django.contrib.auth.tests.models.NaturalKeysTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_user_natural_key (django.contrib.auth.tests.models.NaturalKeysTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_raises_exception (django.contrib.auth.tests.auth_backends.NoBackendsTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_field_order (django.contrib.auth.tests.forms.PasswordChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_incorrect_password (django.contrib.auth.tests.forms.PasswordChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_password_verification (django.contrib.auth.tests.forms.PasswordChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_success (django.contrib.auth.tests.forms.PasswordChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_bug_5605 (django.contrib.auth.tests.forms.PasswordResetFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_cleaned_data (django.contrib.auth.tests.forms.PasswordResetFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_email_subject (django.contrib.auth.tests.forms.PasswordResetFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_inactive_user (django.contrib.auth.tests.forms.PasswordResetFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_invalid_email (django.contrib.auth.tests.forms.PasswordResetFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_nonexistant_email (django.contrib.auth.tests.forms.PasswordResetFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_unusable_password (django.contrib.auth.tests.forms.PasswordResetFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_admin_reset (django.contrib.auth.tests.views.PasswordResetTest) If the reset view is marked as being for admin, the HTTP_HOST header is used for a domain override. ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_confirm_complete (django.contrib.auth.tests.views.PasswordResetTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_confirm_different_passwords (django.contrib.auth.tests.views.PasswordResetTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_confirm_invalid (django.contrib.auth.tests.views.PasswordResetTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_confirm_invalid_post (django.contrib.auth.tests.views.PasswordResetTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_confirm_invalid_user (django.contrib.auth.tests.views.PasswordResetTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_confirm_overflow_user (django.contrib.auth.tests.views.PasswordResetTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_confirm_valid (django.contrib.auth.tests.views.PasswordResetTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_email_found (django.contrib.auth.tests.views.PasswordResetTest) Email is sent if a valid email address is provided for password reset ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_email_found_custom_from (django.contrib.auth.tests.views.PasswordResetTest) Email is sent if a valid email address is provided for password reset when a custom from_email is provided. ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_email_not_found (django.contrib.auth.tests.views.PasswordResetTest) Error is raised if the provided email address isn't currently registered ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_poisoned_http_host (django.contrib.auth.tests.views.PasswordResetTest) Poisoned HTTP_HOST headers can't be used for reset emails ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_poisoned_http_host_admin_site (django.contrib.auth.tests.views.PasswordResetTest) Poisoned HTTP_HOST headers can't be used for reset emails on admin views ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_permlookupdict_in (django.contrib.auth.tests.context_processors.PermWrapperTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_permwrapper_in (django.contrib.auth.tests.context_processors.PermWrapperTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_duplicated_permissions (django.contrib.auth.tests.management.PermissionDuplicationTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_site_profile_not_available (django.contrib.auth.tests.models.ProfileTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_bug_19349_render_with_none_value (django.contrib.auth.tests.forms.ReadOnlyPasswordHashWidgetTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_header_disappears (django.contrib.auth.tests.remote_user.RemoteUserCustomTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_known_user (django.contrib.auth.tests.remote_user.RemoteUserCustomTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_last_login (django.contrib.auth.tests.remote_user.RemoteUserCustomTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_no_remote_user (django.contrib.auth.tests.remote_user.RemoteUserCustomTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_unknown_user (django.contrib.auth.tests.remote_user.RemoteUserCustomTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_header_disappears (django.contrib.auth.tests.remote_user.RemoteUserNoCreateTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_known_user (django.contrib.auth.tests.remote_user.RemoteUserNoCreateTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_last_login (django.contrib.auth.tests.remote_user.RemoteUserNoCreateTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_no_remote_user (django.contrib.auth.tests.remote_user.RemoteUserNoCreateTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_unknown_user (django.contrib.auth.tests.remote_user.RemoteUserNoCreateTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_header_disappears (django.contrib.auth.tests.remote_user.RemoteUserTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_known_user (django.contrib.auth.tests.remote_user.RemoteUserTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_last_login (django.contrib.auth.tests.remote_user.RemoteUserTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_no_remote_user (django.contrib.auth.tests.remote_user.RemoteUserTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_unknown_user (django.contrib.auth.tests.remote_user.RemoteUserTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_all_permissions (django.contrib.auth.tests.auth_backends.RowlevelBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_group_permissions (django.contrib.auth.tests.auth_backends.RowlevelBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_perm (django.contrib.auth.tests.auth_backends.RowlevelBackendTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_password_verification (django.contrib.auth.tests.forms.SetPasswordFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_success (django.contrib.auth.tests.forms.SetPasswordFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_login (django.contrib.auth.tests.signals.SignalTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_logout (django.contrib.auth.tests.signals.SignalTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_logout_anonymous (django.contrib.auth.tests.signals.SignalTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_update_last_login (django.contrib.auth.tests.signals.SignalTestCase) Ensure that only `last_login` is updated in `update_last_login` ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_10265 (django.contrib.auth.tests.tokens.TokenGeneratorTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_date_length (django.contrib.auth.tests.tokens.TokenGeneratorTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_make_token (django.contrib.auth.tests.tokens.TokenGeneratorTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_timeout (django.contrib.auth.tests.tokens.TokenGeneratorTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_bug_14242 (django.contrib.auth.tests.forms.UserChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_bug_17944_empty_password (django.contrib.auth.tests.forms.UserChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_bug_17944_unknown_password_algorithm (django.contrib.auth.tests.forms.UserChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_bug_17944_unmanageable_password (django.contrib.auth.tests.forms.UserChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_bug_19133 (django.contrib.auth.tests.forms.UserChangeFormTest) The change form does not return the password value ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_bug_19349_bound_password_field (django.contrib.auth.tests.forms.UserChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_unsuable_password (django.contrib.auth.tests.forms.UserChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_username_validity (django.contrib.auth.tests.forms.UserChangeFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_both_passwords (django.contrib.auth.tests.forms.UserCreationFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_invalid_data (django.contrib.auth.tests.forms.UserCreationFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_password_verification (django.contrib.auth.tests.forms.UserCreationFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_success (django.contrib.auth.tests.forms.UserCreationFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_user_already_exists (django.contrib.auth.tests.forms.UserCreationFormTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_create_user (django.contrib.auth.tests.models.UserManagerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_create_user_email_domain_normalize (django.contrib.auth.tests.models.UserManagerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_create_user_email_domain_normalize_rfc3696 (django.contrib.auth.tests.models.UserManagerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_create_user_email_domain_normalize_with_whitespace (django.contrib.auth.tests.models.UserManagerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_empty_username (django.contrib.auth.tests.models.UserManagerTestCase) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_for_concrete_model (django.contrib.contenttypes.tests.ContentTypesTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_for_concrete_models (django.contrib.contenttypes.tests.ContentTypesTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_for_models_empty_cache (django.contrib.contenttypes.tests.ContentTypesTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_for_models_full_cache (django.contrib.contenttypes.tests.ContentTypesTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_for_models_partial_cache (django.contrib.contenttypes.tests.ContentTypesTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_lookup_cache (django.contrib.contenttypes.tests.ContentTypesTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_missing_model (django.contrib.contenttypes.tests.ContentTypesTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_shortcut_view (django.contrib.contenttypes.tests.ContentTypesTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_shortcut_view_with_broken_get_absolute_url (django.contrib.contenttypes.tests.ContentTypesTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_shortcut_view_without_get_absolute_url (django.contrib.contenttypes.tests.ContentTypesTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_actual_expiry (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_clear (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_datetime (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_reset (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_seconds (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_timedelta (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_cycle (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_decode (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_default_expiry (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_delete (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_exists_searches_cache_first (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_flush (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_empty (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_expire_at_browser_close (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_key (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_invalid_key (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_iteritems (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_iterkeys (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_itervalues (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_load_overlong_key (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_new_session (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_pop (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_pop_default (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_save (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_key_is_read_only (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_setdefault (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_store (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_update (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_values (django.contrib.sessions.tests.CacheDBSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_actual_expiry (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_clear (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_datetime (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_reset (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_seconds (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_timedelta (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_cycle (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_decode (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_default_expiry (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_delete (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_exists_searches_cache_first (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_flush (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_empty (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_expire_at_browser_close (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_key (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_invalid_key (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_iteritems (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_iterkeys (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_itervalues (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_load_overlong_key (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_new_session (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_pop (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_pop_default (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_save (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_key_is_read_only (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_setdefault (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_store (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_update (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_values (django.contrib.sessions.tests.CacheDBSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_actual_expiry (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_clear (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_datetime (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_reset (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_seconds (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_timedelta (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_cycle (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_decode (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_default_expiry (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_delete (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_flush (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_empty (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_expire_at_browser_close (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_key (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_invalid_key (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_iteritems (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_iterkeys (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_itervalues (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_new_session (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_pop (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_pop_default (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_save (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_key_is_read_only (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_setdefault (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_store (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_update (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_values (django.contrib.sessions.tests.CookieSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_actual_expiry (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_clear (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_clearsessions_command (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_datetime (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_reset (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_seconds (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_timedelta (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_cycle (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_decode (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_default_expiry (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_delete (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_flush (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_empty (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_expire_at_browser_close (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_key (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_invalid_key (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_iteritems (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_iterkeys (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_itervalues (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_new_session (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_pop (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_pop_default (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_save (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_get_decoded (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_key_is_read_only (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_sessionmanager_save (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_setdefault (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_store (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_update (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_values (django.contrib.sessions.tests.DatabaseSessionTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_actual_expiry (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_clear (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_clearsessions_command (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_datetime (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_reset (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_seconds (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_expiry_timedelta (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_cycle (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_decode (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_default_expiry (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_delete (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_flush (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_empty (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_expire_at_browser_close (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_has_key (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_invalid_key (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_iteritems (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_iterkeys (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_itervalues (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_new_session (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_pop (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_pop_default (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_save (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_get_decoded (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_key_is_read_only (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_sessionmanager_save (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_setdefault (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_store (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_update (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_values (django.contrib.sessions.tests.DatabaseSessionWithTimeZoneTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_httponly_session_cookie (django.contrib.sessions.tests.SessionMiddlewareTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 220, in inner return test_func(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/tests.py", line 504, in test_httponly_session_cookie response = middleware.process_response(request, response) File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/middleware.py", line 38, in process_response request.session.save() File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/db.py", line 50, in save session_key=self._get_or_create_session_key(), File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/base.py", line 148, in _get_or_create_session_key self._session_key = self._get_new_session_key() File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/base.py", line 142, in _get_new_session_key if not self.exists(session_key): File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/db.py", line 26, in exists return Session.objects.filter(session_key=session_key).exists() File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 596, in exists return self.query.has_results(using=self.db) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/query.py", line 442, in has_results return bool(compiler.execute_sql(SINGLE)) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 830, in execute_sql sql, params = self.as_sql() File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 74, in as_sql out_cols = self.get_columns(with_col_aliases) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 174, in get_columns result = ['(%s) AS %s' % (col[0], qn2(alias)) for alias, col in six.iteritems(self.query.extra_select)] File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_no_httponly_session_cookie (django.contrib.sessions.tests.SessionMiddlewareTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 220, in inner return test_func(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/tests.py", line 521, in test_no_httponly_session_cookie response = middleware.process_response(request, response) File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/middleware.py", line 38, in process_response request.session.save() File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/db.py", line 50, in save session_key=self._get_or_create_session_key(), File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/base.py", line 148, in _get_or_create_session_key self._session_key = self._get_new_session_key() File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/base.py", line 142, in _get_new_session_key if not self.exists(session_key): File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/db.py", line 26, in exists return Session.objects.filter(session_key=session_key).exists() File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 596, in exists return self.query.has_results(using=self.db) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/query.py", line 442, in has_results return bool(compiler.execute_sql(SINGLE)) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 830, in execute_sql sql, params = self.as_sql() File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 74, in as_sql out_cols = self.get_columns(with_col_aliases) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 174, in get_columns result = ['(%s) AS %s' % (col[0], qn2(alias)) for alias, col in six.iteritems(self.query.extra_select)] File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_secure_session_cookie (django.contrib.sessions.tests.SessionMiddlewareTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 220, in inner return test_func(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/tests.py", line 489, in test_secure_session_cookie response = middleware.process_response(request, response) File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/middleware.py", line 38, in process_response request.session.save() File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/db.py", line 50, in save session_key=self._get_or_create_session_key(), File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/base.py", line 148, in _get_or_create_session_key self._session_key = self._get_new_session_key() File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/base.py", line 142, in _get_new_session_key if not self.exists(session_key): File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/db.py", line 26, in exists return Session.objects.filter(session_key=session_key).exists() File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 596, in exists return self.query.has_results(using=self.db) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/query.py", line 442, in has_results return bool(compiler.execute_sql(SINGLE)) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 830, in execute_sql sql, params = self.as_sql() File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 74, in as_sql out_cols = self.get_columns(with_col_aliases) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 174, in get_columns result = ['(%s) AS %s' % (col[0], qn2(alias)) for alias, col in six.iteritems(self.query.extra_select)] File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_save_on_500 (django.contrib.sessions.tests.SessionMiddlewareTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/tests.py", line 541, in test_session_save_on_500 self.assertNotIn('hello', request.session.load()) File "/usr/local/lib/python2.7/dist-packages/django/contrib/sessions/backends/db.py", line 18, in load expire_date__gt=timezone.now() File "/usr/local/lib/python2.7/dist-packages/django/db/models/manager.py", line 143, in get return self.get_query_set().get(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 382, in get num = len(clone) File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 90, in __len__ self._result_cache = list(self.iterator()) File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 301, in iterator for row in compiler.results_iter(): File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 775, in results_iter for rows in self.execute_sql(MULTI): File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 830, in execute_sql sql, params = self.as_sql() File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 74, in as_sql out_cols = self.get_columns(with_col_aliases) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 212, in get_columns col_aliases) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 299, in get_default_columns r = '%s.%s' % (qn(alias), qn2(field.column)) File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/compiler.py", line 52, in quote_name_unless_alias r = self.connection.ops.quote_name(name) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_current_site (django.contrib.sites.tests.SitesFrameworkTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_save_another (django.contrib.sites.tests.SitesFrameworkTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_site_cache (django.contrib.sites.tests.SitesFrameworkTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_site_manager (django.contrib.sites.tests.SitesFrameworkTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_add (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_add_lazy_translation (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_add_update (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_tags (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_default_level (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_domain (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_add (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_add_read_update (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_read (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_read_add_update (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_full_request_response_cycle (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_bad_cookie (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_high_level (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_json_encoder_decoder (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_low_level (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_max_cookie_length (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_middleware_disabled (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_middleware_disabled_fail_silently (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_multiple_posts (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_no_update (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_safedata (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_settings_level (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_tags (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_with_template_response (django.contrib.messages.tests.cookie.CookieTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/utils.py", line 208, in _pre_setup original_pre_setup(innerself) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_add (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_add_lazy_translation (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_add_update (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_tags (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_default_level (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_add (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_add_read_update (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_read (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_read_add_update (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_flush_used_backends (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_full_request_response_cycle (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_empty (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_fallback (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get_fallback_only (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_high_level (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_low_level (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_middleware_disabled (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_middleware_disabled_fail_silently (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_multiple_posts (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_no_fallback (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_no_update (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_fallback (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_session_fallback_only (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_settings_level (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_tags (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_with_template_response (django.contrib.messages.tests.fallback.FallbackTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_add (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_add_lazy_translation (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_add_update (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_custom_tags (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_default_level (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_add (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_add_read_update (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_read (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_existing_read_add_update (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_full_request_response_cycle (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_get (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_high_level (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_low_level (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_middleware_disabled (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_middleware_disabled_fail_silently (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_multiple_posts (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_no_update (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_safedata (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_settings_level (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_tags (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ====================================================================== ERROR: test_with_template_response (django.contrib.messages.tests.session.SessionTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "/usr/local/lib/python2.7/dist-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "/usr/local/lib/python2.7/dist-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. ---------------------------------------------------------------------- Ran 85 tests in 0.785s FAILED (errors=404, skipped=1) AttributeError: _original_allowed_hosts ================================================ FILE: tests/book_parser.py ================================================ #!/usr/bin/env python3 import re COMMIT_REF_FINDER = r"ch\d\dl\d\d\d-?\d?" class CodeListing: COMMIT_REF_FINDER = r"^(.+) \((" + COMMIT_REF_FINDER + r")\)$" def __init__(self, filename, contents): self.is_server_listing = False if re.match(CodeListing.COMMIT_REF_FINDER, filename): self.filename = re.match(CodeListing.COMMIT_REF_FINDER, filename).group(1) self.commit_ref = re.match(CodeListing.COMMIT_REF_FINDER, filename).group(2) elif filename.startswith("server: "): self.filename = filename.replace("server: ", "") self.commit_ref = None self.is_server_listing = True else: self.filename = filename self.commit_ref = None self.contents = contents self.was_written = False self.skip = False self.currentcontents = False self.against_server = False self.pause_first = False def is_diff(self): lines = self.contents.split("\n") if any(l.count("@@") > 1 for l in lines): return True if len([l for l in lines if l.startswith("+") or l.startswith("-")]) > 2: return True @property def type(self): if self.is_server_listing: return "server code listing" elif self.currentcontents: return "code listing currentcontents" elif self.commit_ref: return "code listing with git ref" elif self.is_diff(): return "diff" else: return "code listing" def __repr__(self): return "" % (self.filename, self.contents.split("\n")[0]) class Command(str): def __init__(self, a_string): self.was_run = False self.skip = False self.ignore_errors = False self.server_command = False self.against_server = False self.pause_first = False self.dofirst = None str.__init__(a_string) @property def type(self): if self.server_command: return "server command" for git_cmd in ("git diff", "git status", "git commit"): if git_cmd in self: return git_cmd if self.startswith("python") and "test" in self: return "test" if self == "python manage.py behave": return "bdd test" if self == "python manage.py migrate": return "interactive manage.py" if self == "python manage.py makemigrations": return "interactive manage.py" if self == "python manage.py collectstatic": return "interactive manage.py" if "docker run" in self and "-it" in self: return "docker run tty" if "docker exec" in self and "container-id-or-name" in self: return "docker exec" if "ottg.co.uk" in self: return "against staging" return "other command" def __repr__(self): return f"" class Output(str): def __init__(self, a_string): self.was_checked = False self.skip = False self.dofirst = None self.jasmine_output = False self.against_server = False self.pause_first = False str.__init__(a_string) @property def type(self) -> str: if self.jasmine_output: return "jasmine output" if "├" in self: return "tree" else: return "output" def fix_newlines(text): if text is None: return "" return text.replace("\r\n", "\n").replace("\\\n", "").strip("\n") def parse_output(listing): text = fix_newlines(listing.text_content().strip()) commands = listing.cssselect("pre strong") if not commands: return [Output(text)] outputs = [] output_before = fix_newlines(listing.text.strip()) if listing.text else "" for command in commands: if "$" in output_before and "\n" in output_before: last_cr = output_before.rfind("\n") previous_lines = output_before[:last_cr] if previous_lines: outputs.append(Output(previous_lines)) elif output_before and "$" not in output_before: outputs.append(Output(output_before)) command_text = fix_newlines(command.text) if output_before.strip().startswith("(virtualenv)"): command_text = "source ./.venv/bin/activate && " + command_text outputs.append(Command(command_text)) output_before = fix_newlines(command.tail) if output_before: outputs.append(Output(output_before)) return outputs def _strip_callouts(content): callout_at_end = r"\s+\(\d+\)$" counts = 0 while re.search(callout_at_end, content, re.MULTILINE): content = re.sub(callout_at_end, "", content, flags=re.MULTILINE) counts += 1 return content def parse_listing(listing): # noqa: PLR0912 classes = listing.get("class").split() skip = "skipme" in classes dofirst_classes = [c for c in classes if c.startswith("dofirst")] if dofirst_classes: dofirst = re.findall(COMMIT_REF_FINDER, dofirst_classes[0])[0] else: dofirst = None if "sourcecode" in classes: try: filename = listing.cssselect(".title")[0].text_content().strip() except IndexError: raise Exception( f"could not find title for listing {listing.text_content()}" ) contents = ( listing.cssselect(".content")[0] .text_content() .replace("\r\n", "\n") .strip("\n") ) contents = _strip_callouts(contents) listing = CodeListing(filename, contents) listing.skip = skip listing.dofirst = dofirst if "currentcontents" in classes: listing.currentcontents = True return [listing] elif "jasmine-output" in classes: contents = ( listing.cssselect(".content")[0] .text_content() .replace("\r\n", "\n") .strip("\n") ) output = Output(contents) output.jasmine_output = True output.skip = skip output.dofirst = dofirst return [output] if "server-commands" in classes: listing = listing.cssselect("div.content")[0] outputs = parse_output(listing) if skip: for listing in outputs: listing.skip = True if dofirst: outputs[0].dofirst = dofirst if "ignore-errors" in classes: for listing in outputs: if isinstance(listing, Command): listing.ignore_errors = True if "server-commands" in classes: for listing in outputs: if isinstance(listing, Command): listing.server_command = True if "against-server" in classes: for listing in outputs: listing.against_server = True if "pause-first" in classes: for listing in outputs: listing.pause_first = True return outputs def get_commands(node): return [ el.text_content().replace("\\\n", "") for el in node.cssselect("pre code strong") ] ================================================ FILE: tests/book_tester.py ================================================ import os import re import subprocess import sys import tempfile import time import unittest from pathlib import Path from textwrap import wrap from book_parser import ( CodeListing, Command, Output, parse_listing, ) from lxml import html from sourcetree import Commit, SourceTree from update_source_repo import update_sources_for_chapter from write_to_file import write_to_file JASMINE_RUNNER = Path(__file__).parent / "run-js-spec.py" # DO_SERVER_COMMANDS = True # if os.environ.get("CI") or os.environ.get("NO_SERVER_COMMANDS"): DO_SERVER_COMMANDS = False def contains(inseq, subseq): return any( inseq[pos : pos + len(subseq)] == subseq for pos in range(0, len(inseq) - len(subseq) + 1) ) def wrap_long_lines(text): paragraphs = text.split("\n") return "\n".join( "\n".join(wrap(p, 79, break_long_words=True, break_on_hyphens=False)) for p in paragraphs ) def split_blocks(text): return [ block.strip() for block in re.split(r"\n\n+|^.*\[\.\.\..*$", text, flags=re.MULTILINE) ] def fix_test_dashes(output): return output.replace(" " + "-" * 69, "-" * 70) def strip_mock_ids(output): strip_mocks_with_names = re.sub( r"Mock name='(.+)' id='(\d+)'>", r"Mock name='\1' id='XX'>", output, ) strip_all_mocks = re.sub( r"Mock id='(\d+)'>", r"Mock id='XX'>", strip_mocks_with_names, ) return strip_all_mocks def strip_object_ids(output): return re.sub("0x([0-9a-f]+)>", "0x123abc22>", output) def strip_migration_timestamps(output): return re.sub(r"00(\d\d)_auto_20\d{6}_\d{4}", r"00\1_auto_20XXXXXX_XXXX", output) def strip_localhost_port(output): lh_fixed = re.sub(r"localhost:\d\d\d\d\d?", r"localhost:XXXX", output) ipaddr_fixed = re.sub(r"127.0.0.1:\d\d\d\d\d?", r"127.0.0.1:XXXX", lh_fixed) return ipaddr_fixed def strip_selenium_trace_ids(output): fixing = re.sub( r"\d{13}(\s+)geckodriver", r"1234567890111\1geckodriver", output, ) fixing = re.sub( r"\d{13}(\s+)webdriver", r"1234567890112\1webdriver", fixing, ) fixing = re.sub( r"\d{13}(\s+)mozrunner", r"1234567890113\1mozrunner", fixing, ) fixing = re.sub( r"\d{13}(\s+)Marionette", r"1234567890114\1Marionette", fixing, ) return fixing def fix_firefox_esr_version(output): return re.sub(r"(\d\d\d\.\d+\.?\d*)esr", r"128.10.1esr", output) def strip_session_ids(output): return re.sub(r"^[a-z0-9]{32}$", r"xxx_session_id_xxx", output) def standardise_assertionerror_none(output): return output.replace("AssertionError: None", "AssertionError") def standardise_git_init_msg(output): return output.replace( "Initialized empty Git repository", "Initialised empty Git repository" ) def strip_git_hashes(output): fixed_indexes = re.sub( r"index .......\.\........ 100644", r"index XXXXXXX\.\.XXXXXXX 100644", output, ) fixed_diff_commits = re.sub( r"^[a-f0-9]{7} ", r"XXXXXXX ", fixed_indexes, flags=re.MULTILINE, ) fixed_git_log_commits = re.sub( r"\* [a-f0-9]{7} ", r"* abc123d ", fixed_diff_commits, flags=re.MULTILINE, ) return fixed_git_log_commits def strip_callouts(output): minus_old_callouts = re.sub( r"^(.+) <\d+>$", r"\1", output, flags=re.MULTILINE, ) minus_new_callouts = re.sub( r"^(.+) \(\d+\)$", r"\1", minus_old_callouts, flags=re.MULTILINE, ) return minus_new_callouts def standardise_library_paths(output): return re.sub( r'(File ").+packages/', r"\1.../", output, flags=re.MULTILINE, ) def standardise_geckodriver_tracebacks(output): return re.sub( r"@chrome://remote/(.+):(\d+:\d+)$", r"@chrome://\1:XXX:XXX", output, flags=re.MULTILINE, ) def strip_test_speed(output): return re.sub( r"Ran (\d+) tests? in \d+\.\d\d\ds", r"Ran \1 tests in X.Xs", output, ) def strip_js_test_speed(output): return re.sub( r"Took \d+ms to run (\d+) tests. (\d+) passed, (\d+) failed.", r"Took XXms to run \1 tests. \2 passed, \3 failed.", output, ) def strip_bdd_test_speed(output): return re.sub( r"features/steps/(\w+).py:(\d+) \d+.\d\d\ds", r"features/steps/\1.py:\2 XX.XXXs", output, ) def strip_screenshot_timestamps(output): fixed = re.sub( r"-(20\d\d-\d\d-\d\dT\d\d\.\d\d\.\d?\d?)", r"-20XX-XX-XXTXX.XX", output, ) # this last is very specific to one listing in 19... fixed = re.sub(r"^\d\d\.html$", "XX.html", fixed, flags=re.MULTILINE) return fixed def strip_docker_image_ids_and_creation_times(output): fixed = re.sub( r"superlists\s+latest\s+\w+\s+\d+ \w+ ago\s+164MB", r"superlists latest someidorother X time ago 164MB", output, ) return fixed def fix_curl_stuff(output): fixed = re.sub( r"User-Agent: curl/\d\.\d+\.\d*", r"User-Agent: curl/8.6.0", output, ) fixed = re.sub( r"Trying \[::1\]:(\d\d\d\d)...", r"Trying ::1:\1...", fixed, ) fixed = re.sub( r"Closing connection 0", r"Closing connection", fixed, ) fixed = re.sub( r"Connected to localhost \(127.0.0.1\) port (\d\d\d\d) \(#0\)", r"Connected to localhost (127.0.0.1) port \1", fixed, ) return fixed def fix_curl_linebreak_after_download(output): return re.sub( r"\*(\s+)Trying", "\n*\\1Trying", output, ) SQLITE_MESSAGES = { "django.db.utils.IntegrityError: lists_item.list_id may not be NULL": "django.db.utils.IntegrityError: NOT NULL constraint failed: lists_item.list_id", "django.db.utils.IntegrityError: columns list_id, text are not unique": "django.db.utils.IntegrityError: UNIQUE constraint failed: lists_item.list_id,\nlists_item.text", "sqlite3.IntegrityError: columns list_id, text are not unique": "sqlite3.IntegrityError: UNIQUE constraint failed: lists_item.list_id,\nlists_item.text", } def fix_sqlite_messages(actual_text): fixed_text = actual_text for old_version, new_version in SQLITE_MESSAGES.items(): fixed_text = fixed_text.replace(old_version, new_version) return fixed_text def standardize_layout_test_pixelsize(actual_text): return re.sub( r"10\d.\d+ != 512 within 10 delta \(40\d.\d+", r"102.5 != 512 within 10 delta (409.5", actual_text, ) def fix_creating_database_line(actual_text): creating_db = "Creating test database for alias 'default'..." actual_lines = actual_text.split("\n") if creating_db in actual_lines: actual_lines.remove(creating_db) actual_lines.insert(0, creating_db) actual_text = "\n".join(actual_lines) return actual_text def fix_interactive_managepy_stuff(actual_text): return actual_text.replace( "Select an option: ", "Select an option:\n", ).replace( ">>> ", ">>>\n", ) class ChapterTest(unittest.TestCase): chapter_name = "override me" maxDiff = None def setUp(self): self.sourcetree = SourceTree() self.tempdir = self.sourcetree.tempdir self.processes = [] self.pos = 0 self.dev_server_running = False self.current_server_cd = None self.current_server_exports = {} def tearDown(self): print(f"finished running test in {self.sourcetree.tempdir}") print("writing tmpdir out to", f".tmpdir.test_{self.chapter_name}") with open(f".tmpdir.test_{self.chapter_name}", "w") as f: f.write(str(self.sourcetree.tempdir)) self.sourcetree.cleanup() def parse_listings(self): base_dir = os.path.split(os.path.abspath(os.path.dirname(__file__)))[0] filename = self.chapter_name + ".html" with open(os.path.join(base_dir, filename), encoding="utf-8") as f: raw_html = f.read() parsed_html = html.fromstring(raw_html) all_nodes = parsed_html.cssselect( ".exampleblock.sourcecode, div:not(.sourcecode) div.listingblock" ) listing_nodes = [] for ix, node in enumerate(all_nodes): prev = all_nodes[ix - 1] if node not in list(prev.iterdescendants()): listing_nodes.append(node) self.listings = [p for n in listing_nodes for p in parse_listing(n)] def check_final_diff(self, ignore=None, diff=None): if diff is None: diff = self.run_command(Command(f"git diff -w repo/{self.chapter_name}")) print("checking final diff", diff) self.assertNotIn("fatal:", diff) start_marker = "diff --git a/\n" commit = Commit.from_diff(start_marker + diff) if ignore is None: if commit.lines_to_add: self.fail(f"Found lines to add in diff:\n{commit.lines_to_add}") if commit.lines_to_remove: self.fail(f"Found lines to remove in diff:\n{commit.lines_to_remove}") return if "moves" in ignore: ignore.remove("moves") difference_lines = commit.deleted_lines + commit.new_lines else: difference_lines = commit.lines_to_add + commit.lines_to_remove for line in difference_lines: if any(ignorable in line for ignorable in ignore): continue self.fail(f"Found divergent line in diff:\n{line}") def start_with_checkout(self): update_sources_for_chapter(self.chapter_name, self.previous_chapter) self.sourcetree.start_with_checkout(self.chapter_name, self.previous_chapter) def write_to_file(self, codelisting): self.assertEqual( type(codelisting), CodeListing, "passed a non-Codelisting to write_to_file:\n%s" % (codelisting,), ) print("writing to file", codelisting.filename) write_to_file(codelisting, self.tempdir) def apply_patch(self, codelisting): tf = tempfile.NamedTemporaryFile(delete=False) tf.write(codelisting.contents.encode("utf8")) tf.write(b"\n") tf.close() print("patch:\n", codelisting.contents) patch_output = self.run_command( Command( "patch --fuzz=3 --no-backup-if-mismatch %s %s" % (codelisting.filename, tf.name) ) ) print(patch_output) self.assertNotIn("malformed", patch_output) self.assertNotIn("failed", patch_output.lower()) codelisting.was_checked = True with open(os.path.join(self.tempdir, codelisting.filename)) as f: print(f.read()) os.remove(tf.name) self.pos += 1 codelisting.was_written = True def run_command(self, command, cwd=None, user_input=None, ignore_errors=False): assert isinstance(command, Command), ( f"passed a non-Command to run-command:\n{command}" ) if command == "git push": command.was_run = True return if command.startswith("curl"): cmd_to_run = command.replace("curl", "curl --silent --show-error") if command.startswith("grep") and sys.platform == "darwin": cmd_to_run = command.replace("grep", "ggrep") else: cmd_to_run = str(command) print(f"running command {cmd_to_run} with {ignore_errors=}") output = self.sourcetree.run_command( cmd_to_run, cwd=cwd, user_input=user_input, ignore_errors=ignore_errors ) command.was_run = True return output def prep_virtualenv(self): virtualenv_path = self.tempdir / ".venv" if not virtualenv_path.exists(): print("preparing virtualenv") self.sourcetree.run_command("uv venv -p 3.14 .venv") os.environ["VIRTUAL_ENV"] = str(virtualenv_path) os.environ["PATH"] = ":".join( [f"{virtualenv_path}/bin"] + os.environ["PATH"].split(":") ) if (self.tempdir / "requirements.txt").exists(): self.sourcetree.run_command("uv pip install -r requirements.txt") else: self.sourcetree.run_command('uv pip install "django<6" selenium') self.sourcetree.run_command("uv pip install pip") def prep_database(self): self.sourcetree.run_command(f"python {self._manage_py()} migrate --noinput") def assertLineIn(self, line, lines): if "\t" in line or "\t" in "\n".join(lines): print("tabz") if line not in lines: raise AssertionError( f"{repr(line)} not found in:\n" + "\n".join(repr(l) for l in lines) ) def assert_console_output_correct(self, actual, expected, ls=False): print("checking expected output\n", expected) print("against actual\n", actual) self.assertEqual( type(expected), Output, f"passed a non-Output to run-command:\n{expected}", ) if str(self.tempdir) in actual: actual = actual.replace(str(self.tempdir), "...goat-book") if sys.platform == "darwin": # for some reason macos does full paths to virtualenvs # when linux doesnt actual = actual.replace("...goat-book/.venv", "./.venv") actual = actual.replace("/private", "") # macos thing if ls: actual = actual.strip() self.assertCountEqual(actual.split("\n"), expected.split()) expected.was_checked = True return actual_fixed = standardise_library_paths(actual) actual_fixed = standardise_geckodriver_tracebacks(actual_fixed) actual_fixed = standardize_layout_test_pixelsize(actual_fixed) actual_fixed = strip_test_speed(actual_fixed) actual_fixed = strip_js_test_speed(actual_fixed) actual_fixed = strip_bdd_test_speed(actual_fixed) actual_fixed = strip_git_hashes(actual_fixed) actual_fixed = strip_mock_ids(actual_fixed) actual_fixed = strip_object_ids(actual_fixed) actual_fixed = strip_migration_timestamps(actual_fixed) actual_fixed = strip_session_ids(actual_fixed) actual_fixed = strip_docker_image_ids_and_creation_times(actual_fixed) actual_fixed = fix_curl_stuff(actual_fixed) actual_fixed = fix_curl_linebreak_after_download(actual_fixed) actual_fixed = strip_localhost_port(actual_fixed) actual_fixed = strip_selenium_trace_ids(actual_fixed) actual_fixed = fix_firefox_esr_version(actual_fixed) actual_fixed = strip_screenshot_timestamps(actual_fixed) actual_fixed = fix_sqlite_messages(actual_fixed) actual_fixed = fix_creating_database_line(actual_fixed) actual_fixed = fix_interactive_managepy_stuff(actual_fixed) actual_fixed = standardise_assertionerror_none(actual_fixed) actual_fixed = standardise_git_init_msg(actual_fixed) actual_fixed = wrap_long_lines(actual_fixed) expected_fixed = standardise_library_paths(expected) expected_fixed = standardise_geckodriver_tracebacks(expected_fixed) expected_fixed = fix_test_dashes(expected_fixed) expected_fixed = strip_test_speed(expected_fixed) expected_fixed = strip_js_test_speed(expected_fixed) expected_fixed = strip_bdd_test_speed(expected_fixed) expected_fixed = strip_git_hashes(expected_fixed) expected_fixed = strip_mock_ids(expected_fixed) expected_fixed = strip_docker_image_ids_and_creation_times(expected_fixed) expected_fixed = fix_curl_stuff(expected_fixed) expected_fixed = strip_object_ids(expected_fixed) expected_fixed = strip_migration_timestamps(expected_fixed) expected_fixed = strip_session_ids(expected_fixed) expected_fixed = strip_localhost_port(expected_fixed) expected_fixed = strip_selenium_trace_ids(expected_fixed) expected_fixed = fix_firefox_esr_version(expected_fixed) expected_fixed = strip_screenshot_timestamps(expected_fixed) expected_fixed = strip_callouts(expected_fixed) expected_fixed = standardise_assertionerror_none(expected_fixed) actual_fixed = actual_fixed.replace("\xa0", " ") expected_fixed = expected_fixed.replace("\xa0", " ") if "\t" in actual_fixed: print("fixing tabs") actual_fixed = re.sub(r"\s+", " ", actual_fixed) expected_fixed = re.sub(r"\s+", " ", expected_fixed) actual_lines = actual_fixed.split("\n") expected_lines = expected_fixed.split("\n") for line in expected_lines: if line.startswith("[..."): continue if line.endswith("[...]"): line = line.rsplit("[...]")[0].rstrip() self.assertLineIn(line, [l[: len(line)] for l in actual_lines]) elif line.startswith(" "): self.assertLineIn(line, actual_lines) else: self.assertLineIn(line.rstrip(), [l.strip() for l in actual_lines]) if ( len(expected_lines) > 4 and "[..." not in expected_fixed and expected.type != "qunit output" ): self.assertMultiLineEqual(actual_fixed.strip(), expected_fixed.strip()) expected.was_checked = True def find_with_check(self, pos, expected_content): listing = self.listings[pos] listing_text = lambda l: getattr(l, "contents", l) first_match = next( ( (ix, l) for ix, l in enumerate(self.listings) if expected_content in listing_text(l) ), None, ) all_listings = "\n".join(str(t) for t in enumerate(self.listings)) error = f'Could not find {expected_content} at pos {pos}: ("{listing}"). ' + ( f"Did you mean {first_match}?" if first_match else f"Listings were:\n{all_listings}" ) if hasattr(listing, "contents"): if expected_content not in listing.contents: raise Exception(error) elif expected_content not in listing: raise Exception(error) return listing def skip_with_check(self, pos, expected_content): listing = self.find_with_check(pos, expected_content) listing.skip = True def replace_command_with_check(self, pos, old, new): listing = self.listings[pos] all_listings = "\n".join(str(t) for t in enumerate(self.listings)) error = f'Could not find {old} at pos {pos}: "{listing}". Listings were:\n{all_listings}' if old not in listing: raise Exception(error) assert type(listing) == Command new_listing = Command(listing.replace(old, new)) for attr, val in vars(listing).items(): setattr(new_listing, attr, val) self.listings[pos] = new_listing def skip_forward_if_skipto_set(self) -> None: if target_listing := os.environ.get("SKIPTO"): self.sourcetree.run_command("uv pip install gunicorn whitenoise") commit_spec = self.sourcetree.get_commit_spec(target_listing) while True: listing = self.listings[self.pos] found = False self.pos += 1 if getattr(listing, "commit_ref", None) == target_listing: found = True print("Skipping to pos", self.pos) self.sourcetree.run_command(f"git checkout {commit_spec}") break if not found: raise Exception(f"Could not find {target_listing}") def _run_tree(self, target="", no_report=False): return self.sourcetree.run_command( f"tree -v -I __pycache__ {'--noreport' if no_report else ''} {target}" ) def assert_directory_tree_correct(self, expected_tree): actual_tree = self._run_tree(no_report=True) self.assert_console_output_correct(actual_tree, expected_tree) def assert_all_listings_checked(self, listings, exceptions=[]): for i, listing in enumerate(listings): if i in exceptions: continue if listing.skip: continue if type(listing) == CodeListing: self.assertTrue( listing.was_written, "Listing %d not written:\n%s" % (i, listing) ) if type(listing) == Command: self.assertTrue( listing.was_run, "Command %d not run:\n%s" % (i, listing) ) if type(listing) == Output: self.assertTrue( listing.was_checked, "Output %d not checked:\n%s" % (i, listing) ) def check_test_code_cycle(self, pos, test_command_in_listings=True, ft=False): self.write_to_file(self.listings[pos]) if test_command_in_listings: pos += 1 self.assertIn("test", self.listings[pos]) test_run = self.run_command(self.listings[pos]) elif ft: test_run = self.run_command(Command("python functional_tests.py")) else: test_run = self.run_command( Command(f"python {self._manage_py()} test lists") ) pos += 1 self.assert_console_output_correct(test_run, self.listings[pos]) def unset_PYTHONDONTWRITEBYTECODE(self): # so any references to __pycache__ in the book work if "PYTHONDONTWRITEBYTECODE" in os.environ: del os.environ["PYTHONDONTWRITEBYTECODE"] def run_test_and_check_result(self, bdd=False): if bdd: self.assertIn("behave", self.listings[self.pos]) else: self.assertIn("test", self.listings[self.pos]) if bdd: test_run = self.run_command(self.listings[self.pos], ignore_errors=True) else: test_run = self.run_command(self.listings[self.pos]) self.assert_console_output_correct(test_run, self.listings[self.pos + 1]) self.pos += 2 def run_js_tests(self, tests_path: Path): p = subprocess.run( ["python", str(JASMINE_RUNNER), str(tests_path)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, # env={**os.environ, "OPENSSL_CONF": "/dev/null"}, check=False, ) return p.stdout.decode() def check_jasmine_output(self, expected_output): lists_tests = Path(self.tempdir) / "src/lists/static/tests/SpecRunner.html" assert lists_tests.exists() lists_run = self.run_js_tests(lists_tests) self.assert_console_output_correct(lists_run, expected_output) def check_current_contents(self, listing, actual_contents): print("CHECK CURRENT CONTENTS") stripped_actual_lines = [l.strip() for l in actual_contents.split("\n")] listing_contents = re.sub(r" +#$", "", listing.contents, flags=re.MULTILINE) for block in split_blocks(listing_contents): stripped_block = [line.strip() for line in block.strip().split("\n")] for line in stripped_block: self.assertIn( line, stripped_actual_lines, f"{line!r} not found in\n" + "\n".join(repr(l) for l in stripped_actual_lines), ) self.assertTrue( contains(stripped_actual_lines, stripped_block), "\n{}\n\nnot found in\n\n{}".format( "\n".join(stripped_block), "\n".join(stripped_actual_lines) ), ) listing.was_written = True def check_commit(self, pos): if self.listings[pos].endswith("commit -a"): self.listings[pos] = Command( self.listings[pos] + 'm "commit for listing %d"' % (self.pos,) ) elif self.listings[pos].endswith("commit"): self.listings[pos] = Command( self.listings[pos] + ' -am "commit for listing %d"' % (self.pos,) ) commit = self.run_command(self.listings[pos]) assert "insertion" in commit or "changed" in commit self.pos += 1 def check_diff_or_status(self, pos): LIKELY_FILES = [ "urls.py", "tests.py", "views.py", "functional_tests.py", "settings.py", "home.html", "list.html", "base.html", "test_", "base.py", "test_my_lists.py", "deploy-playbook.yaml", ] self.assertTrue("diff" in self.listings[pos] or "status" in self.listings[pos]) git_output = self.run_command(self.listings[pos]) self.pos += 1 comment = self.listings[pos + 1] if comment.skip: comment.was_checked = True self.pos += 1 return if comment.type != "output": return if not any("/" + l in git_output for l in LIKELY_FILES): if not any(f in git_output for f in ("lists/", "functional_tests.py")): self.fail("no likely files in diff output %s" % (git_output,)) for expected_file in LIKELY_FILES: if "/" + expected_file in git_output: if expected_file not in comment: self.fail( "could not find %s in comment %r given git output\n%s" % (expected_file, comment, git_output) ) self.listings[pos + 1].was_checked = True comment.was_checked = True self.pos += 1 def _manage_py(self): if (self.tempdir / "src/manage.py").exists(): # if we're later in the book, # we've moved everything into an src folder return "src/manage.py" return "manage.py" def start_dev_server(self): self.run_command(Command("python manage.py runserver")) self.dev_server_running = True time.sleep(1) def restart_dev_server(self): print("restarting dev server") self.run_command(Command("pkill -f runserver")) time.sleep(1) self.start_dev_server() time.sleep(1) def run_unit_tests(self): if (self.tempdir / "src/accounts/tests").exists(): return self.run_command( Command(f"python {self._manage_py()} test lists accounts") ) else: return self.run_command(Command(f"python {self._manage_py()} test lists")) def run_fts(self): if (self.tempdir / "functional_tests").exists(): return self.run_command( Command(f"python {self._manage_py()} test functional_tests") ) if (self.tempdir / "src/functional_tests").exists(): return self.run_command( Command(f"python {self._manage_py()} test functional_tests") ) else: return self.run_command(Command("python functional_tests.py")) def run_interactive_manage_py(self, listing): output_before = self.listings[self.pos + 1] assert isinstance(output_before, Output) LIKELY_INPUTS = ("yes", "no", "1", "2", "''") user_input = self.listings[self.pos + 2] if isinstance(user_input, Command) and user_input in LIKELY_INPUTS: if user_input == "yes": print("yes case") # in this case there is moar output after the yes output_after = self.listings[self.pos + 3] assert isinstance(output_after, Output) expected_output = Output( wrap_long_lines(output_before + " " + output_after.lstrip()) ) next_output = None elif user_input == "1": print("migrations 1 case") # in this case there is another hop output_after = self.listings[self.pos + 3] assert isinstance(output_after, Output) first_input = user_input next_input = self.listings[self.pos + 4] assert isinstance(next_input, Command) next_output = self.listings[self.pos + 5] expected_output = Output( wrap_long_lines( output_before + "\n" + output_after + "\n" + next_output ) ) user_input = Command(first_input + "\n" + next_input) else: expected_output = output_before output_after = None next_output = None if user_input == "2": ignore_errors = True else: ignore_errors = False else: user_input = None expected_output = output_before output_after = None ignore_errors = True next_output = None output = self.run_command( listing, user_input=user_input, ignore_errors=ignore_errors ) self.assert_console_output_correct(output, expected_output) listing.was_checked = True output_before.was_checked = True self.pos += 2 if user_input is not None: user_input.was_run = True self.pos += 1 if output_after is not None: output_after.was_checked = True self.pos += 1 if next_output is not None: self.pos += 2 next_output.was_checked = True first_input.was_run = True next_input.was_run = True def recognise_listing_and_process_it(self): listing = self.listings[self.pos] if listing.pause_first: print("pausing first") time.sleep(2) if listing.dofirst: print("DOFIRST", listing.dofirst) self.sourcetree.patch_from_commit( listing.dofirst, ) if listing.skip: print("SKIP") listing.was_checked = True listing.was_written = True self.pos += 1 elif listing.against_server and not DO_SERVER_COMMANDS: print("SKIP AGAINST SERVER") listing.was_checked = True listing.was_run = True self.pos += 1 elif listing.type == "test": print("TEST RUN") self.run_test_and_check_result() elif listing.type == "bdd test": print("BDD TEST RUN") self.run_test_and_check_result(bdd=True) elif listing.type == "git diff": print("GIT DIFF") self.check_diff_or_status(self.pos) elif listing.type == "git status": print("STATUS") self.check_diff_or_status(self.pos) elif listing.type == "git commit": print("COMMIT") self.check_commit(self.pos) elif listing.type == "interactive manage.py": print("INTERACTIVE MANAGE.PY") self.run_interactive_manage_py(listing) elif listing.type == "tree": print("TREE") self.assert_directory_tree_correct(listing) self.pos += 1 elif listing.type == "server command": if DO_SERVER_COMMANDS: assert 0, "re-implement" server_output = self.run_server_command(listing) listing.was_run = True self.pos += 1 next_listing = self.listings[self.pos] if next_listing.type == "output" and not next_listing.skip: if DO_SERVER_COMMANDS: for line in next_listing.split("\n"): line = line.split("[...]")[0].strip() line = re.sub(r"\s+", " ", line) server_output = re.sub(r"\s+", " ", server_output) self.assertIn(line, server_output) next_listing.was_checked = True self.pos += 1 elif listing.type == "against staging": print("AGAINST STAGING") next_listing = self.listings[self.pos + 1] if DO_SERVER_COMMANDS: output = self.run_command(listing, ignore_errors=listing.ignore_errors) listing.was_checked = True else: listing.skip = True if next_listing.type == "output" and not next_listing.skip: if DO_SERVER_COMMANDS: self.assert_console_output_correct(output, next_listing) next_listing.was_checked = True else: next_listing.skip = True self.pos += 2 elif listing.type == "docker run tty": self.sourcetree.run_command( "docker kill $(docker ps -q)", ignore_errors=True, silent=True ) fixed = Command(listing.replace(" -it ", " -t ")) if "docker run --platform=linux/amd64 -t debug-ci" in fixed: fixed = Command( fixed.replace( "docker run --platform=linux/amd64 -t debug-ci", "docker run -e PYTHON_COLORS=0 --platform=linux/amd64 -t debug-ci", ) ) next_listing = self.listings[self.pos + 1] if next_listing.type == "output" and not next_listing.skip: output = self.run_command(fixed, ignore_errors=listing.ignore_errors) listing.was_run = True self.assert_console_output_correct(output, next_listing) next_listing.was_checked = True self.pos += 2 else: self.run_command(fixed, ignore_errors=listing.ignore_errors) listing.was_run = True listing.was_checked = True self.pos += 1 elif listing.type == "docker exec": container_id = self.sourcetree.run_command( "docker ps --filter=ancestor=superlists -q" ).strip() fixed = Command(listing.replace("container-id-or-name", container_id)) next_listing = self.listings[self.pos + 1] if next_listing.type == "output" and not next_listing.skip: output = self.run_command(fixed, ignore_errors=listing.ignore_errors) listing.was_run = True self.assert_console_output_correct(output, next_listing) next_listing.was_checked = True self.pos += 2 else: self.run_command(fixed, ignore_errors=listing.ignore_errors) listing.was_run = True listing.was_checked = True self.pos += 1 elif listing.type == "other command": print("A COMMAND") next_listing = self.listings[self.pos + 1] if next_listing.type == "output" and not next_listing.skip: output = self.run_command(listing, ignore_errors=listing.ignore_errors) ls = listing.startswith("ls") self.assert_console_output_correct(output, next_listing, ls=ls) next_listing.was_checked = True self.pos += 2 elif "tree" in listing and next_listing.type == "tree": assert listing.startswith("tree") _, _, target = listing.partition("tree") output = self._run_tree(target=target) listing.was_run = True self.assert_console_output_correct(output, next_listing) next_listing.was_checked = True self.pos += 2 else: self.run_command(listing, ignore_errors=listing.ignore_errors) listing.was_checked = True self.pos += 1 elif listing.type == "diff": print("DIFF") self.apply_patch(listing) elif listing.type == "code listing currentcontents": actual_contents = self.sourcetree.get_contents(listing.filename) self.check_current_contents(listing, actual_contents) self.pos += 1 elif listing.type == "code listing": print("CODE") self.write_to_file(listing) self.pos += 1 elif listing.type == "code listing with git ref": print("CODE FROM GIT REF") self.sourcetree.apply_listing_from_commit(listing) self.pos += 1 elif listing.type == "server code listing": assert 0, "reimplement" elif listing.type == "jasmine output": self.check_jasmine_output(listing) self.pos += 1 elif listing.type == "output": test_run = self.run_unit_tests() if "OK" in test_run.splitlines() and "OK" not in listing.splitlines(): print("unit tests pass, must be an FT:\n", test_run) test_run = self.run_fts() try: self.assert_console_output_correct(test_run, listing) except AssertionError as e: if "OK" in test_run.splitlines() and "OK" in listing.splitlines(): print("got error when checking unit tests", e) test_run = self.run_fts() self.assert_console_output_correct(test_run, listing) else: raise self.pos += 1 else: self.fail("not implemented for " + str(listing)) ================================================ FILE: tests/chapters.py ================================================ CHAPTERS = [ # part 1 "chapter_01", "chapter_02_unittest", "chapter_03_unit_test_first_view", "chapter_04_philosophy_and_refactoring", "chapter_05_post_and_database", "chapter_06_explicit_waits_1", "chapter_07_working_incrementally", # part 2: deploy "chapter_08_prettification", "chapter_09_docker", "chapter_10_production_readiness", "chapter_11_server_prep", "chapter_12_ansible", # part 3: validation "chapter_13_organising_test_files", "chapter_14_database_layer_validation", "chapter_15_simple_form", "chapter_16_advanced_forms", # part 4: spiking and mocking "chapter_17_javascript", "chapter_18_second_deploy", "chapter_19_spiking_custom_auth", "chapter_20_mocking_1", "chapter_21_mocking_2", "chapter_22_fixtures_and_wait_decorator", "chapter_23_debugging_prod", "chapter_24_outside_in", "chapter_25_CI", "chapter_26_page_pattern", "chapter_27_hot_lava", ] ================================================ FILE: tests/check_links.py ================================================ from lxml import html import requests with open('book.html') as f: node = html.fromstring(f.read()) all_hrefs = [e.get('href') for e in node.cssselect('a')] urls = [l for l in all_hrefs if l and l.startswith('h')] for l in urls: try: response = requests.get(l) if response.status_code != 200: print(l) else: print('.', end="", flush=True) except requests.RequestException: print(l) ================================================ FILE: tests/conftest.py ================================================ import pytest pytest.register_assert_rewrite('sourcetree') ================================================ FILE: tests/examples.py ================================================ CODE_LISTING_WITH_CAPTION = """
functional_tests.py
from selenium import webdriver

browser = webdriver.Firefox()
browser.get('http://localhost:8000')

assert 'Django' in browser.title
""" CODE_LISTING_WITH_CAPTION_AND_GIT_COMMIT_REF = """
functional_tests/tests.py (ch06l001)
from django.test import LiveServerTestCase
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time


class NewVisitorTest(LiveServerTestCase):

    def setUp(self):
        [...]
""" SERVER_COMMAND = """
elspeth@server:$ sudo do stuff
""" COMMANDS_WITH_VIRTUALENV = """
$ source ../.venv/bin/activate
(virtualenv)$ python manage.py test lists
[...]
ImportError: No module named django
""" CODE_LISTING_WITH_DIFF_FORMATING_AND_COMMIT_REF = """
lists/tests/test_models.py (ch09l010)
diff --git a/lists/tests/test_views.py b/lists/tests/test_views.py
index fc1eb64..9305bf8 100644
--- a/lists/tests/test_views.py
+++ b/lists/tests/test_views.py
@@ -81,33 +81,3 @@ class ListViewTest(TestCase):
         self.assertTemplateUsed(response, 'list.html')
         self.assertEqual(response.context['list'], list)

-
-
-class ListAndItemModelsTest(TestCase):
-
-    def test_saving_and_retrieving_items(self):
[...]
""" COMMAND_MADE_WITH_ATS = """ 
$ grep id_new_item functional_tests/tests/test*
""" OUTPUT_WITH_SKIPME = """ 
try:
    item.save()
    self.fail('The full_clean should have raised an exception')
except ValidationError:
    pass
""" CODE_LISTING_WITH_SKIPME = """
lists/functional_tests/test_list_item_validation.py
    def DONTtest_cannot_add_empty_list_items(self):
""" OUTPUTS_WITH_DOFIRST = """ 
$ grep -r id_new_item lists/

lists/static/base.css:#id_new_item {
lists/templates/list.html:        <input name="item_text" id="id_new_item"
placeholder="Enter a to-do item" />
""" OUTPUTS_WITH_CURRENTCONTENTS = """
superlists/urls.py
from django.conf.urls import patterns, include, url

from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'superlists.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),

    url(r'^admin/', include(admin.site.urls)),
)
from django.conf.urls import patterns, include, url
""" JASMINE_OUTPUT = """ 
2 specs, 0 failures, randomized with seed 12345        finished in 0.01s

Superlists tests
  * check we know how to hide things
  * sense check our html fixture
""" OUTPUT_WITH_CONTINUATION = """ 
$ wget -O bootstrap.zip https://github.com/twbs/bootstrap/releases/download/\
v3.1.0/bootstrap-3.1.0-dist.zip
$ unzip bootstrap.zip
$ mkdir lists/static
$ mv dist lists/static/bootstrap
$ rm bootstrap.zip
""" OUTPUT_WITH_COMMANDS_INLINE = """ 
$ python manage.py makemigrations
You are trying to add a non-nullable field 'list' to item without a default;
we can't do that (the database needs something to populate existing rows).
Please select a fix:
 1) Provide a one-off default now (will be set on all existing rows)
 2) Quit, and let me add a default in models.py
Select an option: 1
Please enter the default value now, as valid Python
The datetime module is available, so you can do e.g. datetime.date.today()
>>> ''
Migrations for 'lists':
  0003_item_list.py:
    - Add field list to item
""" CODE_LISTING_WITH_ASCIIDOCTOR_CALLOUTS = """
src/lists/templates/base.html (ch16l004)
    </div>

    <script>
      const textInput = document.querySelector("#id_text");  (1)
      textInput.oninput = () => {  (2) (3)
        const errorMsg = document.querySelector(".invalid-feedback");
        errorMsg.style.display = "none";  (4)
      }
    </script>
""" OUTPUT_WITH_CALLOUTS = """
$ python manage.py test functional_tests.test_list_item_validation
Creating test database for alias 'default'...
E
======================================================================
ERROR: test_cannot_add_empty_list_items
(functional_tests.test_list_item_validation.ItemValidationTest)
 ---------------------------------------------------------------------
Traceback (most recent call last):
  File "...goat-book/functional_tests/test_list_item_validation.py", line
15, in test_cannot_add_empty_list_items
    self.wait_for(lambda: self.assertEqual(  (1)
  File "...goat-book/functional_tests/base.py", line 37, in wait_for
    raise e  (2)
  File "...goat-book/functional_tests/base.py", line 34, in wait_for
    return fn()  (2)
  File "...goat-book/functional_tests/test_list_item_validation.py", line
16, in <lambda>  (3)
    self.browser.find_element_by_css_selector('.has-error').text,  (3)
[...]
selenium.common.exceptions.NoSuchElementException: Message: Unable to locate
element: .has-error


 ---------------------------------------------------------------------
Ran 1 test in 10.575s

FAILED (errors=1)
""" EXAMPLE_DIFF_LISTING = """
lists/templates/home.html (ch07l018)
   <body>
-    <h1>Your To-Do list</h1>
+    <h1>Start a new To-Do list</h1>
     <form method="POST" action="/">
       <input name="item_text" id="id_new_item" placeholder="Enter a to-do item" />
       {% csrf_token %}
     </form>
-    <table id="id_list_table">
-      {% for item in items %}
-        <tr><td>{{ forloop.counter }}: {{ item.text }}</td></tr>
-      {% endfor %}
-    </table>
   </body>
""" ================================================ FILE: tests/my-phantomjs-qunit-runner.js ================================================ /*global require, phantom */ var system = require('system'); if (!system.args[1]){ console.log('Pass path to test file as second arg'); phantom.exit(); } var path = system.args[1]; if (path.indexOf('/') !== 0) { path = system.env.PWD + '/' + path; } var page = require('webpage').create(); var logs = ''; page.onConsoleMessage = function (msg) { logs += msg + '\n'; }; page.open('file://' + path, function () { setTimeout(function() { var output = page.evaluate( function () { var results = ''; var headline = $('#qunit-testresult').text().split('.')[1] + '.'; results += headline + '\n'; var testCounter = 0; $('#qunit-tests li').each(function() { var li = $(this); if (li.prop('id').indexOf('qunit-test-output') !== -1){ testCounter += 1; var resultLine = ''; resultLine += testCounter + '. '; if (li.find('.module-name').length > 0) { resultLine += li.find('.module-name').text() + ': '; } resultLine += li.find('.test-name').text(); resultLine += ' ' + li.find('.counts').text(); resultLine = resultLine.replace('Rerun', ''); var fails = li.find('.fail'); if (fails.text()) { li.find('.qunit-assert-list li').each(function (assertCounter) { var assert = $(this); resultLine += '\n'; resultLine += ' ' + (assertCounter + 1) + '. '; resultLine += assert.find('.test-message').text(); if (assert.find('.fail')) { assert.find('tr').each(function () { resultLine += '\n'; resultLine += ' ' + $(this).text(); }); } }); } results += resultLine + '\n'; } }); return results; }); console.log(output); console.log(logs); phantom.exit(); }, 100); }); ================================================ FILE: tests/run-js-spec.py ================================================ #!python import re import sys import time from pathlib import Path from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement def sub_book_path(text: str) -> str: return re.sub( r"@file://(.+)/superlists/src/", ".../goat-book/src/", text, ) def run(path: Path): assert path.exists() options = webdriver.FirefoxOptions() options.add_argument("--headless") browser = webdriver.Firefox(options=options) # options = webdriver.ChromeOptions() # options.add_argument("--headless=new") # options.binary_location = "/Applications/Vivaldi.app/Contents/MacOS/Vivaldi" # browser = webdriver.Chrome(options=options) failed = False def _el_text(sel, node: webdriver.Remote | WebElement = browser): raw = "\n".join(el.text for el in node.find_elements(By.CSS_SELECTOR, sel)) return re.sub(r" 0\.\d+s", " 0.005s", raw) try: browser.get(f"file:///{path}?seed=12345") time.sleep(0.2) # for entry in browser.get_log('browser'): # print(entry) print( f"{_el_text('.jasmine-overall-result')} {_el_text('.jasmine-duration')}" ) print(_el_text(".jasmine-bar.jasmine-errored")) print(_el_text(".jasmine-menu.jasmine-failure-list")) for failures_el in browser.find_elements(By.CSS_SELECTOR, ".jasmine-failures"): for spec_failure in failures_el.find_elements( By.CSS_SELECTOR, ".jasmine-spec-detail.jasmine-failed" ): failed = True print() print(_el_text(".jasmine-description", spec_failure)) print(_el_text(".jasmine-messages", spec_failure)) for success_el in browser.find_elements(By.CSS_SELECTOR, ".jasmine-summary"): for suite_el in success_el.find_elements(By.CSS_SELECTOR, ".jasmine-suite"): if suite_el.is_displayed(): print(_el_text("li.jasmine-suite-detail", suite_el)) for spec_el in suite_el.find_elements( By.CSS_SELECTOR, "ul.jasmine-specs li.jasmine-passed" ): print(" * " + spec_el.text) finally: browser.quit() return failed if __name__ == "__main__": _, fn, *__ = sys.argv if fn.endswith("Spec.js"): fn = fn.replace("Spec.js", "SpecRunner.html") failed = run(Path(fn).resolve()) sys.exit(1 if failed is True else 0) ================================================ FILE: tests/slimerjs-0.9.0/LICENSE ================================================ The files SlimerJS are licensed under the MPL 2.0 (http://mozilla.org/MPL/2.0/), with the exception of the sources file listed below (zipped into the omni.ja file in the distributed version of SlimerJS), which are made available by their authors under the licenses listed alongside. * modules/addon-sdk/* are released under the Mozilla Public License, v. 2.0 see http://mozilla.org/MPL/2.0/ These files comes from Firefox source code and Mozilla Addon-SDK source code http://mxr.mozilla.org/mozilla-central/source/toolkit/addon-sdk/ https://github.com/laurentj/addon-sdk/tree/master/lib/sdk/io * components/httpd.js * modules/httpUtils.jsm is released under the Mozilla Public License, v. 2.0 see http://mozilla.org/MPL/2.0/ These files comes from Mozilla source code http://mxr.mozilla.org/mozilla-central/source/netwerk/test/httpserver/ * components/ConsoleAPI.js * components/nsPrompter.js is released under the Mozilla Public License, v. 2.0 see http://mozilla.org/MPL/2.0/ This file comes originally from the source code of Firefox http://mxr.mozilla.org/mozilla-central/source/dom/base/ConsoleAPI.js http://mxr.mozilla.org/mozilla-central/source/toolkit/components/prompts/src/nsPrompter.js * modules/slimer-sdk/net-log.js is released under the Mozilla Public License, v. 2.0 see http://mozilla.org/MPL/2.0/ The author is Olivier Meunier and Laurent Jouanneau and this file comes from https://github.com/olivier-m/jetpack-net-log/ * modules/slimer-sdk/webpage.js is released under the Mozilla Public License, v. 2.0 see http://mozilla.org/MPL/2.0/ Some pieces of code come from webpage.js of the project https://github.com/olivier-m/jetpack-webpage/ The author of these pieces of code is Olivier Meunier * modules/slPhantomJSKeyCode.jsm content of this file is a part of webpage.js from the PhantomJS project from Ofi Labs. Copyright (C) 2011 Ariya Hidayat Copyright (C) 2011 Ivan De Marino Copyright (C) 2011 James Roe Copyright (C) 2011 execjosh, http://execjosh.blogspot.com Copyright (C) 2012 James M. Greene Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: tests/slimerjs-0.9.0/README.md ================================================ # SlimerJS SlimerJS is a scriptable browser. It allows you to manipulate a web page with a Javascript script: opening a webpage, clicking on links, modifying the content... It is useful to do functional tests, page automaton, network monitoring, screen capture etc. Go to [http://slimerjs.org] to know more and to access to the documentation # Install - Install [Firefox](http://getfirefox.com), or [XulRunner](http://ftp.mozilla.org/pub/mozilla.org/xulrunner/releases/19.0.2/runtimes/) (both version 18 or more) - [Download the latest package](http://download.slimerjs.org/slimerjs-0.5RC1.zip) or [the source code of SlimerJS](https://github.com/laurentj/slimerjs/archive/master.zip) if you didn't it yet - On windows, a .bat is provided, but you can also launch slimer from a "true" console. In this case, you should install [Cygwin](http://www.cygwin.com/) or any other unix environment to launch slimerjs. - SlimerJS needs to know where Firefox or XulRunner is stored. It tries to discover itself the path but can fail. You must then set the environment variable SLIMERJSLAUNCHER, which should contain the full path to the firefox binary: - On linux: ```export SLIMERJSLAUNCHER=/usr/bin/firefox``` - on Windows: ```SET SLIMERJSLAUNCHER="c:\Program Files\Mozilla Firefox\firefox.exe``` - On windows with cygwin : ```export SLIMERJSLAUNCHER="/cygdrive/c/program files/mozilla firefox/firefox.exe"``` - On MacOS: ```export SLIMERJSLAUNCHER=/Applications/Firefox.app/Contents/MacOS/firefox``` - You can of course set this variable in your .bashrc, .profile or in the computer properties on Windows. # Launching SlimerJS Open a terminal and go to the directory of SlimerJS (src/ if you downloaded the source code). Then launch: ``` ./slimerjs myscript.js ``` In the Windows commands console: ``` slimerjs.bat myscript.js ``` The given script myscripts.js is then executed in a window. If your script is short, you probably won't see this window. You can for example launch some tests if you execute SlimerJS from the source code: ``` ./slimerjs ../test/initial-tests.js ``` # Launching a headless SlimerJS There is a tool called xvfb, available on Linux and MacOS. It allows to launch any "graphical" programs without the need of X-Windows environment. Windows of the application won't be shown and will be drawn only in memory. Install it from your prefered repository (```sudo apt-get install xvfb``` with debian/ubuntu). Then launch SlimerJS like this: ``` xvfb-run ./slimerjs myscript.js ``` You won't see any windows. If you have any problems with xvfb, see its documentation. # Getting help - Ask your questions on the dedicated [mailing list](https://groups.google.com/forum/#!forum/slimerjs). - Discuss with us on IRC: channel #slimerjs on irc.mozilla.org. - Read the faq [on the website](http://slimerjs.org/faq.html). - Read [the documentation](http://docs.slimerjs.org/current/) ================================================ FILE: tests/slimerjs-0.9.0/application.ini ================================================ [App] Vendor=Innophi Name=SlimerJS Version=0.9.0 BuildID=20131211 ID=slimerjs@slimerjs.org Copyright=Copyright 2012-2013 Laurent Jouanneau & Innophi [Gecko] MinVersion=17.0.0 MaxVersion=27.* ================================================ FILE: tests/slimerjs-0.9.0/slimerjs ================================================ #!/bin/bash #retrieve full path of the current script # symlinks are resolved, so application.ini could be found # this code comes from http://stackoverflow.com/questions/59895/ SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink SLIMERDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$SLIMERDIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done SLIMERDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SYSTEM=`uname -o 2>&1` if [ "$SYSTEM" == "Cygwin" ] then IN_CYGWIN=1 SLIMERDIR=`cygpath -w $SLIMERDIR` else IN_CYGWIN= fi # retrieve the path of a gecko launcher if [ "$SLIMERJSLAUNCHER" == "" ] then if [ -f "$SLIMERDIR/xulrunner/xulrunner" ] then SLIMERJSLAUNCHER="$SLIMERDIR/xulrunner/xulrunner" else if [ -f "$SLIMERDIR/xulrunner/xulrunner.exe" ] then SLIMERJSLAUNCHER="$SLIMERDIR/xulrunner/xulrunner.exe" else SLIMERJSLAUNCHER=`command -v firefox` if [ "$SLIMERJSLAUNCHER" == "" ] then SLIMERJSLAUNCHER=`command -v xulrunner` if [ "$SLIMERJSLAUNCHER" == "" ] then echo "SLIMERJSLAUNCHER environment variable is missing. Set it with the path to Firefox or XulRunner" exit 1 fi fi fi fi fi if [ ! -x "$SLIMERJSLAUNCHER" ] then echo "SLIMERJSLAUNCHER environment variable does not contain an executable path. Set it with the path to Firefox" exit 1 fi function showHelp() { echo " --config= Load the given configuration file" echo " (JSON formated)" echo " --debug=[yes|no] Prints additional warning and debug message" echo " (default is no)" echo " --disk-cache=[yes|no] Enables disk cache (default is no)." echo " --help or -h Show this help" #echo " --ignore-ssl-errors=[yes|no] Ignores SSL errors (default is no)." echo " --load-images=[yes|no] Loads all inlined images (default is yes)" echo " --local-storage-quota= Sets the maximum size of the offline" echo " local storage (in KB)" #echo " --local-to-remote-url-access=[yes|no] Allows local content to access remote" #echo " URL (default is no)" echo " --max-disk-cache-size= Limits the size of the disk cache (in KB)" #echo " --output-encoding= Sets the encoding for the terminal output" #echo " (default is 'utf8')" #echo " --remote-debugger-port= Starts the script in a debug harness and" #echo " listens on the specified port" #echo " --remote-debugger-autorun=[yes|no] Runs the script in the debugger immediately" #echo " (default is no)" echo " --proxy= Sets the proxy server" echo " --proxy-auth= Provides authentication information for the" echo " proxy" echo " --proxy-type=[http|socks5|none|auto|system|config-url] Specifies the proxy type (default is http)" #echo " --script-encoding= Sets the encoding used for the starting" #echo " script (default is utf8)" #echo " --web-security=[yes|no] Enables web security (default is yes)" echo " --version or v Prints out SlimerJS version" #echo " --webdriver or --wd or -w Starts in 'Remote WebDriver mode' (embedded" #echo " GhostDriver) '127.0.0.1:8910'" #echo " --webdriver=[:] Starts in 'Remote WebDriver mode' in the" #echo " specified network interface" #echo " --webdriver-logfile= File where to write the WebDriver's Log " #echo " (default 'none') (NOTE: needs '--webdriver')" #echo " --webdriver-loglevel=[ERROR|WARN|INFO|DEBUG] WebDriver Logging Level " #echo " (default is 'INFO') (NOTE: needs '--webdriver')" #echo " --webdriver-selenium-grid-hub= URL to the Selenium Grid HUB (default is" #echo " 'none') (NOTE: needs '--webdriver') " echo " --error-log-file= Log all javascript errors in a file" echo " -jsconsole Open a window to view all javascript errors" echo " during the execution" echo "" echo "*** About profiles: see details of these Mozilla options at" echo "https://developer.mozilla.org/en-US/docs/Mozilla/Command_Line_Options#User_Profile" echo "" echo " --createprofile name Create a new profile and exit" echo " -P name Use the specified profile to execute the script" echo " -profile path Use the profile stored in the specified" echo " directory, to execute the script" echo "By default, SlimerJS use a temporary profile" echo "" } # retrieve list of existing environment variable, because Mozilla doesn't provide an API to get this # list LISTVAR="" ENVVAR=`env`; for v in $ENVVAR; do IFS='=' read -a var <<< "$v" LISTVAR="$LISTVAR,${var[0]}" done # check arguments. CREATE_TEMP='Y' HIDE_ERRORS='Y' shopt -s nocasematch for i in $*; do case "$i" in --help|-h) showHelp exit 0 ;; -reset-profile|-profile|-p|-createprofile|-profilemanager) CREATE_TEMP='' ;; --reset-profile|--profile|--p|--createprofile|--profilemanager) CREATE_TEMP='' ;; "--debug=true") HIDE_ERRORS='N' esac if [[ $i == --debug* ]] && [[ "$i" == *errors* ]]; then HIDE_ERRORS='N' fi done shopt -u nocasematch # If profile parameters, don't create a temporary profile PROFILE="" PROFILE_DIR="" if [ "$CREATE_TEMP" == "Y" ] then PROFILE_DIR=`mktemp -d -q /tmp/slimerjs.XXXXXXXX` if [ "$PROFILE_DIR" == "" ]; then echo "Error: cannot generate temp profile" exit 1 fi if [ "$IN_CYGWIN" == 1 ]; then PROFILE_DIR=`cygpath -w $PROFILE_DIR` fi PROFILE="--profile $PROFILE_DIR" else PROFILE="-purgecaches" fi # put all arguments in a variable, to have original arguments before their transformation # by Mozilla export __SLIMER_ARGS="$@" export __SLIMER_ENV="$LISTVAR" # launch slimerjs with firefox/xulrunner if [ "$HIDE_ERRORS" == "Y" ]; then "$SLIMERJSLAUNCHER" -app $SLIMERDIR/application.ini $PROFILE -no-remote "$@" 2> /dev/null else "$SLIMERJSLAUNCHER" -app $SLIMERDIR/application.ini $PROFILE -no-remote "$@" fi EXITCODE=$? if [ "$PROFILE_DIR" != "" ]; then rm -rf $PROFILE_DIR fi exit $EXITCODE ================================================ FILE: tests/slimerjs-0.9.0/slimerjs.bat ================================================ @echo off SET SLIMERJSLAUNCHER="%SLIMERJSLAUNCHER%" REM % ~ d[rive] p[ath] 0[script name] is the absolute path to this bat file, without quotes, always. REM ~ strips quotes from the argument SET SLIMERDIR=%~dp0 REM %* is every argument passed to this script. SET __SLIMER_ARGS=%* SET __SLIMER_ENV= SET CREATETEMP=Y SET HIDE_ERRORS=Y REM check arguments FOR %%A IN (%*) DO ( if ["%%A"]==["/?"] ( call :helpMessage goto :eof ) if /I ["%%A"]==["--help"] ( call :helpMessage goto :eof ) if /I ["%%A"]==["-h"] ( call :helpMessage goto :eof ) if /I ["%%A"]==["/h"] ( call :helpMessage goto :eof ) if /I ["%%A"]==["-reset-profile"] ( SET CREATETEMP= ) if /I ["%%A"]==["-profile"] ( SET CREATETEMP= ) if /I ["%%A"]==["-p"] ( SET CREATETEMP= ) if /I ["%%A"]==["-createprofile"] ( SET CREATETEMP= ) if /I ["%%A"]==["-profilemanager"] ( SET CREATETEMP= ) if /I ["%%A"]==["--reset-profile"] ( SET CREATETEMP= ) if /I ["%%A"]==["--profile"] ( SET CREATETEMP= ) if /I ["%%A"]==["--p"] ( SET CREATETEMP= ) if /I ["%%A"]==["--createprofile"] ( SET CREATETEMP= ) if /I ["%%A"]==["--profilemanager"] ( SET CREATETEMP= ) if /I ["%%A"]==["/debug"] ( SET HIDE_ERRORS= ) ) if not exist %SLIMERJSLAUNCHER% ( if exist "%SLIMERDIR%\xulrunner\xulrunner.exe" ( SET SLIMERJSLAUNCHER="%SLIMERDIR%\xulrunner\xulrunner.exe" ) ) if not exist %SLIMERJSLAUNCHER% ( call :findFirefox ) if not exist %SLIMERJSLAUNCHER% ( echo SLIMERJSLAUNCHER environment variable is missing or the path is invalid. echo Set it with the path to Firefox or xulrunner. echo The current value of SLIMERJSLAUNCHER is: %SLIMERJSLAUNCHER% REM %% escapes the percent sign so it displays literally echo SET SLIMERJSLAUNCHER="%%programfiles%%\Mozilla Firefox\firefox.exe" echo SET SLIMERJSLAUNCHER="%%programfiles%%\XULRunner\xulrunner.exe" pause exit 1 ) SETLOCAL EnableDelayedExpansion REM store environment variable into __SLIMER_ENV for SlimerJS FOR /F "usebackq delims==" %%i IN (`set`) DO set __SLIMER_ENV=!__SLIMER_ENV!,%%i REM let's create a temporary dir for the profile, if needed if ["%CREATETEMP%"]==[""] ( SET PROFILEDIR= SET PROFILE=-purgecaches goto callexec ) :createdirname SET PROFILEDIR=%Temp%\slimerjs-!Random!!Random!!Random! IF EXIST "%PROFILEDIR%" ( GOTO createdirname ) mkdir %PROFILEDIR% SET PROFILE=-profile %PROFILEDIR% :callexec if ["%HIDE_ERRORS%"]==[""] ( %SLIMERJSLAUNCHER% -app "%SLIMERDIR%application.ini" %PROFILE% -attach-console -no-remote %__SLIMER_ARGS% ) ELSE ( %SLIMERJSLAUNCHER% -app "%SLIMERDIR%application.ini" %PROFILE% -attach-console -no-remote %__SLIMER_ARGS% 2>NUL ) if ["%CREATETEMP%"]==["Y"] ( rmdir /S /Q %PROFILEDIR% ) ENDLOCAL goto :eof :helpMessage REM in echo statements the escape character is ^ REM escape < > | and & REM the character % is escaped by doubling it to %% REM if delayed variable expansion is turned on then the character ! needs to be escaped as ^^! echo Available options are: echo. echo --config=^ Load the given configuration file echo (JSON formated) echo --debug=[yes^|no] Prints additional warning and debug message echo (default is no) echo --disk-cache=[yes^|no] Enables disk cache (default is no). echo --help or -h Show this help REM echo --ignore-ssl-errors=[yes^|no] Ignores SSL errors (default is no). echo --load-images=[yes^|no] Loads all inlined images (default is yes) echo --local-storage-quota=^ Sets the maximum size of the offline echo local storage (in KB) REM echo --local-to-remote-url-access=[yes^|no] Allows local content to access remote REM echo URL (default is no) echo --max-disk-cache-size=^ Limits the size of the disk cache (in KB) REM echo --output-encoding=^ Sets the encoding for the terminal output REM echo (default is 'utf8') REM echo --remote-debugger-port=^ Starts the script in a debug harness and REM echo listens on the specified port REM echo --remote-debugger-autorun=[yes^|no] Runs the script in the debugger immediately REM echo (default is no) echo --proxy=^ Sets the proxy server echo --proxy-auth=^ Provides authentication information for the echo proxy echo --proxy-type=[http^|socks5^|none^|auto^|system^|config-url] Specifies the proxy type (default is http) REM echo --script-encoding=^ Sets the encoding used for the starting REM echo script (default is utf8) REM echo --web-security=[yes^|no] Enables web security (default is yes) echo --version or v Prints out SlimerJS version REM echo --webdriver or --wd or -w Starts in 'Remote WebDriver mode' (embedded REM echo GhostDriver) '127.0.0.1:8910' REM echo --webdriver=[^:]^ Starts in 'Remote WebDriver mode' in the REM echo specified network interface REM echo --webdriver-logfile=^ File where to write the WebDriver's Log REM echo (default 'none') (NOTE: needs '--webdriver') REM echo --webdriver-loglevel=[ERROR^|WARN^|INFO^|DEBUG^|] WebDriver Logging Level REM echo (default is 'INFO') (NOTE: needs '--webdriver') REM echo --webdriver-selenium-grid-hub=^ URL to the Selenium Grid HUB (default is REM echo 'none') (NOTE: needs '--webdriver') echo --error-log-file= Log all javascript errors in a file echo -jsconsole Open a window to view all javascript errors echo during the execution echo. echo *** About profiles: see details of these Mozilla options at echo https://developer.mozilla.org/en-US/docs/Mozilla/Command_Line_Options#User_Profile echo. echo --createprofile name Create a new profile and exit echo -P name Use the specified profile to execute the script echo -profile path Use the profile stored in the specified echo directory, to execute the script echo By default, SlimerJS use a temporary profile echo. goto :eof :findFirefox if exist "%programfiles%\Mozilla Firefox\firefox.exe" ( SET SLIMERJSLAUNCHER="%programfiles%\Mozilla Firefox\firefox.exe" ) if exist "%programfiles% (x86)\Mozilla Firefox\firefox.exe" ( SET SLIMERJSLAUNCHER="%programfiles% (x86)\Mozilla Firefox\firefox.exe" ) echo SLIMERJSLAUNCHER is set to %SLIMERJSLAUNCHER% goto :eof ================================================ FILE: tests/slimerjs-0.9.0/slimerjs.py ================================================ #!/usr/bin/env python import os import sys import tempfile import shutil import string import subprocess def resolve(path): if os.path.islink(path): path = os.path.join(os.path.dirname(path), os.readlink(path)) return resolve(path) return path def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) def which(program): fpath, fname = os.path.split(program) if fpath: if is_exe(program): return program else: for path in os.environ["PATH"].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, program) if is_exe(exe_file): return exe_file return None # retrieve full path of the current script # symlinks are resolved, so application.ini could be found SLIMERJS_PATH = os.path.abspath(os.path.dirname(resolve(__file__))) SYS_ARGS = sys.argv[1:] SLIMERJSLAUNCHER = os.environ.get("SLIMERJSLAUNCHER", ""); if SLIMERJSLAUNCHER == "": POSSIBLE_PATH = [] if sys.platform == "linux" or sys.platform == "linux2" or sys.platform == "darwin": POSSIBLE_PATH.append(os.path.join(SLIMERJS_PATH, "xulrunner", "xulrunner")) path = which('firefox') if path != None: POSSIBLE_PATH.append(path) path = which('xulrunner') if path != None: POSSIBLE_PATH.append(path) elif sys.platform == "win32": POSSIBLE_PATH.append(os.path.join(SLIMERJS_PATH, "xulrunner", "xulrunner.exe")) path = which('firefox.exe') if path != None: POSSIBLE_PATH.append(path) path = which('xulrunner.exe') if path != None: POSSIBLE_PATH.append(path) POSSIBLE_PATH.append(os.path.join(os.environ.get('programfiles'), "Mozilla Firefox", "firefox.exe")) POSSIBLE_PATH.append("%s (x86)" % os.path.join(os.environ.get('programfiles'), "Mozilla Firefox", "firefox.exe")) for path in POSSIBLE_PATH: if is_exe(path): SLIMERJSLAUNCHER = path break if SLIMERJSLAUNCHER == "": print('SLIMERJSLAUNCHER environment variable is missing and I don\'t find XulRunner or Firefox') print('Set SLIMERJSLAUNCHER with the path to Firefox or XulRunner') sys.exit(1) else: if not os.path.exists(SLIMERJSLAUNCHER): print("SLIMERJSLAUNCHER environment variable does not contain an executable path: %s. Set it with the path to Firefox" % SLIMERJSLAUNCHER) sys.exit(1) def showHelp(): print(" --config= Load the given configuration file") print(" (JSON formated)") print(" --debug=[yes|no] Prints additional warning and debug message") print(" (default is no)") print(" --disk-cache=[yes|no] Enables disk cache (default is no).") print(" --help or -h Show this help") #print(" --ignore-ssl-errors=[yes|no] Ignores SSL errors (default is no).") print(" --load-images=[yes|no] Loads all inlined images (default is yes)") print(" --local-storage-quota= Sets the maximum size of the offline") print(" local storage (in KB)") #print(" --local-to-remote-url-access=[yes|no] Allows local content to access remote") #print(" URL (default is no)") print(" --max-disk-cache-size= Limits the size of the disk cache (in KB)") #print(" --output-encoding= Sets the encoding for the terminal output") #print(" (default is 'utf8')") #print(" --remote-debugger-port= Starts the script in a debug harness and") #print(" listens on the specified port") #print(" --remote-debugger-autorun=[yes|no] Runs the script in the debugger immediately") #print(" (default is no)") print(" --proxy= Sets the proxy server") print(" --proxy-auth= Provides authentication information for the") print(" proxy") print(" --proxy-type=[http|socks5|none|auto|system|config-url] Specifies the proxy type (default is http)") #print(" --script-encoding= Sets the encoding used for the starting") #print(" script (default is utf8)") #print(" --web-security=[yes|no] Enables web security (default is yes)") print(" --version or v Prints out SlimerJS version") #print(" --webdriver or --wd or -w Starts in 'Remote WebDriver mode' (embedded") #print(" GhostDriver) '127.0.0.1:8910'") #print(" --webdriver=[:] Starts in 'Remote WebDriver mode' in the") #print(" specified network interface") #print(" --webdriver-logfile= File where to write the WebDriver's Log ") #print(" (default 'none') (NOTE: needs '--webdriver')") #print(" --webdriver-loglevel=[ERROR|WARN|INFO|DEBUG] WebDriver Logging Level ") #print(" (default is 'INFO') (NOTE: needs '--webdriver')") #print(" --webdriver-selenium-grid-hub= URL to the Selenium Grid HUB (default is") #print(" 'none') (NOTE: needs '--webdriver') ") print(" --error-log-file= Log all javascript errors in a file") print(" -jsconsole Open a window to view all javascript errors") print(" during the execution") print("") print("*** About profiles: see details of these Mozilla options at") print("https://developer.mozilla.org/en-US/docs/Mozilla/Command_Line_Options#User_Profile") print("") print(" --createprofile name Create a new profile and exit") print(" -P name Use the specified profile to execute the script") print(" -profile path Use the profile stored in the specified") print(" directory, to execute the script") print("By default, SlimerJS use a temporary profile") print("") # retrieve list of existing environment variable, #because Mozilla doesn't provide an API to get this # list LISTVAR="" for env in os.environ.data: LISTVAR = "%s,%s" % (LISTVAR, env) # check arguments. HIDE_ERRORS=True CREATE_TEMP=True NO_TEMP_PROFILE_OPTIONS = [ "-reset-profile","-profile","-p","-createprofile","-profilemanager", "--reset-profile","--profile","--p","--createprofile","--profilemanager", ] for arg in SYS_ARGS: if arg == '--help' or arg == "-h": showHelp() sys.exit(0) # If profile parameters, don't create a temporary profile if arg.lower() in NO_TEMP_PROFILE_OPTIONS: CREATE_TEMP=False if arg == '--debug=true' or (arg.startswith('--debug=') and arg.find("errors") != -1): HIDE_ERRORS=False PROFILE=[] PROFILE_DIR="" if CREATE_TEMP: PROFILE_DIR = tempfile.mkdtemp('', 'slimerjs.') PROFILE=['--profile', PROFILE_DIR] else: PROFILE=["-purgecaches"] # put all arguments in a variable, to have original arguments before their transformation # by Mozilla os.environ.data['__SLIMER_ENV'] = LISTVAR os.environ.data['__SLIMER_ARGS'] = string.join(SYS_ARGS,' ') # launch slimerjs with firefox/xulrunner SLCMD = [ SLIMERJSLAUNCHER ] SLCMD.extend(["-app", os.path.join(SLIMERJS_PATH, "application.ini"), "-no-remote"]) if sys.platform == "win32": SLCMD.extend(["-attach-console"]) SLCMD.extend(PROFILE) SLCMD.extend(SYS_ARGS) exitCode = 0 try: if HIDE_ERRORS: try: from subprocess import DEVNULL # py3k except ImportError: DEVNULL = open(os.devnull, 'wb') exitCode = subprocess.call(SLCMD, stderr=DEVNULL) else: exitCode = subprocess.call(SLCMD) except OSError as err: print('Fatal: %s. Are you sure %s exists?' % (err, SLIMERJSLAUNCHER)) sys.exit(1) if CREATE_TEMP: shutil.rmtree(PROFILE_DIR) sys.exit(exitCode) ================================================ FILE: tests/source_updater.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import ast from collections import OrderedDict import os import re from textwrap import dedent VIEW_FINDER = re.compile(r'^def (\w+)\(request.*\):$') class SourceUpdateError(Exception): pass def get_indent(line): return (len(line) - len(line.lstrip())) * " " class Block(object): def __init__(self, node, source): self.name = node.name self.node = node self.full_source = source self.start_line = self.node.lineno - 1 self.full_line = self.full_source.split('\n')[self.start_line] self.source = '\n'.join( self.full_source.split('\n')[self.start_line:self.last_line + 1] ) @property def is_view(self): return bool(VIEW_FINDER.match(self.full_line)) @property def last_line(self): last_line_no = max( getattr(n, 'lineno', -1) for n in ast.walk(self.node) ) lines = self.full_source.split('\n') if len(lines) > last_line_no: for line in lines[last_line_no:]: if line.strip() == '': break last_line_no += 1 return last_line_no - 1 class Source(object): def __init__(self): self.contents = '' @classmethod def from_path(kls, path): source = Source() if os.path.exists(path): with open(path) as f: source.contents = f.read() source.path = path return source @classmethod def _from_contents(kls, contents): source = Source() source.contents = contents return source @property def lines(self): return self.contents.split('\n') @property def functions(self): if not hasattr(self, '_functions'): self._functions = OrderedDict() for node in self.ast: if isinstance(node, ast.FunctionDef): block = Block(node, self.contents) self._functions[block.name] = block return self._functions @property def views(self): return OrderedDict((f.name, f) for f in self.functions.values() if f.is_view) @property def ast(self): try: return list(ast.walk(ast.parse(self.contents))) except SyntaxError: return [] @property def classes(self): if not hasattr(self, '_classes'): self._classes = OrderedDict() for node in self.ast: if isinstance(node, ast.ClassDef): block = Block(node, self.contents) self._classes[block.name] = block return self._classes @property def _import_nodes(self): for node in self.ast: if isinstance(node, (ast.Import, ast.ImportFrom)): node.full_line = self.lines[node.lineno - 1] yield node @property def _deduped_import_nodes(self): from_imports = {} other_imports = [] for node in self._import_nodes: if isinstance(node, ast.Import): other_imports.append(node) else: if node.module in from_imports: if len(node.names) > len(from_imports[node.module].names): from_imports[node.module] = node else: from_imports[node.module] = node return other_imports + list(from_imports.values()) @property def imports(self): for node in self._deduped_import_nodes: yield node.full_line @property def django_imports(self): return [i for i in self.imports if i.startswith('from django')] @property def project_imports(self): return [i for i in self.imports if i.startswith('from lists')] @property def general_imports(self): return [i for i in self.imports if i not in self.django_imports and i not in self.project_imports] @property def fixed_imports(self): import_sections = [] if self.general_imports: import_sections.append('\n'.join(sorted(self.general_imports))) if self.django_imports: import_sections.append('\n'.join(sorted(self.django_imports))) if self.project_imports: import_sections.append('\n'.join(sorted(self.project_imports))) fixed_imports = '\n\n'.join(import_sections) if fixed_imports and not fixed_imports.endswith('\n'): fixed_imports += '\n' return fixed_imports def find_first_nonimport_line(self): try: first_nonimport = next(l for l in self.lines if l and l not in self.imports) except StopIteration: return len(self.lines) pos = self.lines.index(first_nonimport) if self._import_nodes: if pos < max(n.lineno for n in self._import_nodes): raise SourceUpdateError('first nonimport (%s) was before end of imports (%s)' % ( first_nonimport, max(n.lineno for n in self._import_nodes)) ) return pos def replace_function(self, new_lines): function_name = re.search(r'def (\w+)\(.*\):', new_lines[0].strip()).group(1) print('replacing function', function_name) old_function = self.functions[function_name] indent = get_indent(old_function.full_line) self.contents = '\n'.join( self.lines[:old_function.start_line] + [indent + l for l in new_lines] + self.lines[old_function.last_line + 1:] ) return self.contents def remove_function(self, function_name): print('removing function %s' % (function_name,)) function = self.functions[function_name] self.contents = '\n'.join( self.lines[:function.start_line] + self.lines[function.last_line + 1:] ) self.contents = re.sub(r'\n\n\n\n+', r'\n\n\n', self.contents) return self.contents def find_start_line(self, new_lines): if not new_lines: raise SourceUpdateError() start_line = new_lines[0].strip() if start_line == '': raise SourceUpdateError() try: return [l.strip() for l in self.lines].index(start_line.strip()) except ValueError: print('no start line match for', start_line) def add_to_class(self, classname, new_lines): new_lines = dedent('\n'.join(new_lines)).strip().split('\n') klass = self.classes[classname] lines_before_class = '\n'.join(self.lines[:klass.start_line]) print('lines before\n', lines_before_class) lines_after_class = '\n'.join(self.lines[klass.last_line + 1:]) print('lines after\n', lines_after_class) new_class = klass.source + '\n\n\n' + '\n'.join( ' ' + l for l in new_lines ) print('new class\n', new_class) self.contents = lines_before_class + '\n' + new_class + '\n' + lines_after_class def find_end_line(self, new_lines): if not new_lines: raise SourceUpdateError() end_line = new_lines[-1].strip() if end_line == '': raise SourceUpdateError() start_line = self.find_start_line(new_lines) try: from_start = [l.strip() for l in self.lines[start_line:]].index(end_line.strip()) return start_line + from_start except ValueError: print('no end line match for', end_line) def add_imports(self, imports): post_import_lines = self.lines[self.find_first_nonimport_line():] self.contents = '\n'.join(imports + self.lines) self.contents = ( self.fixed_imports + '\n' + '\n'.join(post_import_lines) ) def update(self, new_contents): self.contents = new_contents def get_updated_contents(self): if not self.contents.endswith('\n'): self.contents += '\n' return self.contents def write(self): with open(self.path, 'w') as f: f.write(self.get_updated_contents()) ================================================ FILE: tests/sourcetree.py ================================================ import io import os import re import shutil import signal import subprocess import tempfile import time from dataclasses import dataclass from pathlib import Path def strip_comments(line): match_python = re.match(r"^(.+\S) +#$", line) if match_python: print("match python") return match_python.group(1) match_js = re.match(r"^(.+\S) +//$", line) if match_js: return match_js.group(1) return line BOOTSTRAP_WGET = "wget -O bootstrap.zip https://github.com/twbs/bootstrap/releases/download/v3.3.4/bootstrap-3.3.4-dist.zip" @dataclass class Commit: info: str @staticmethod def from_diff(commit_info): return Commit(info=commit_info) @property def all_lines(self): return self.info.split("\n") @property def lines_to_add(self): return [ l[1:] for l in self.all_lines if l.startswith("+") and l[1:].strip() and l[1] != "+" ] @property def lines_to_remove(self): return [ l[1:] for l in self.all_lines if l.startswith("-") and l[1:].strip() and l[1] != "-" ] @property def moved_lines(self): return [l for l in self.lines_to_add if l in self.lines_to_remove] @property def deleted_lines(self): return [l for l in self.lines_to_remove if l not in self.lines_to_add] @property def new_lines(self): return [l for l in self.lines_to_add if l not in self.lines_to_remove] @property def stripped_lines_to_add(self): return [l.strip() for l in self.lines_to_add] class ApplyCommitException(Exception): pass class SourceTree: def __init__(self): self.tempdir = Path(tempfile.mkdtemp()) self.processes = [] self.dev_server_running = False def get_contents(self, path): with open(os.path.join(self.tempdir, path)) as f: return f.read() def cleanup(self): for process in self.processes: try: os.killpg(process.pid, signal.SIGTERM) except OSError: pass if os.environ.get("TMPDIR_CLEANUP") not in ("0", "false"): shutil.rmtree(self.tempdir) def run_command( self, command, cwd=None, user_input=None, ignore_errors=False, silent=False ): if cwd is None: cwd = self.tempdir env = os.environ.copy() if "manage.py test" in command: # prevent stdout and stderr from appearing to come out in wrong order env["PYTHONUNBUFFERED"] = "1" process = subprocess.Popen( command, shell=True, cwd=cwd, executable="/bin/bash", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, # preexec_fn=os.setsid, # disabled to get passwordless sudo to work universal_newlines=True, env=env, ) process._command = command self.processes.append(process) if "runserver" in command: # can't read output, stdout.read just hangs. # TODO: readline? see below. # TODO: could also try UNBUFFERED? return if "docker run" in command and "superlists" in command and not ignore_errors: output = "" while True: line = process.stdout.readline() print(f"\t{line}", end="") output += line if "Quit the server with CONTROL-C." in output: # go any further and we hang. print("brief sleep to allow docker server to become available") time.sleep(2) return output if "Booting worker with pid" in output: # gunicorn startup, also hangs: print("brief sleep to allow docker server to become available") time.sleep(2) return output if "ERROR: failed to solve" in output: # docker build error, bail out break if user_input and not user_input.endswith("\n"): user_input += "\n" if user_input: print(f"sending user input: {user_input}") output, _ = process.communicate(user_input) if process.returncode and not ignore_errors: if ( " test" in command or "functional_tests" in command or "diff" in command or "migrate" in command ): return output print( "process %s return a non-zero code (%s)" % (command, process.returncode) ) print("output:\n", output) raise Exception( "process %s return a non-zero code (%s)" % (command, process.returncode) ) if not silent: try: print(output) except io.BlockingIOError as e: print(e) pass return output def get_local_repo_path(self, chapter_name): return os.path.abspath( os.path.join( os.path.dirname(__file__), "../source/{}/superlists".format(chapter_name), ) ) def start_with_checkout(self, chapter, previous_chapter): print("starting with checkout") self.run_command("git init .") self.run_command(f'git remote add repo "{self.get_local_repo_path(chapter)}"') self.run_command("git fetch repo") # NB - relies on previous_chapter existing as a branch in the chapter local_repo_path self.run_command("git reset --hard repo/{}".format(previous_chapter)) print(self.run_command("git status")) self.chapter = chapter def get_commit_spec(self, commit_ref): return f"repo/{self.chapter}^{{/--{commit_ref}--}}" def get_files_from_commit_spec(self, commit_spec): return self.run_command( f"git diff-tree --no-commit-id --name-only --find-renames -r {commit_spec}" ).split() def show_future_version(self, commit_spec, path): return self.run_command("git show {}:{}".format(commit_spec, path), silent=True) def patch_from_commit(self, commit_ref, path=None): commit_spec = self.get_commit_spec(commit_ref) self.run_command( #'git diff {commit}^ {commit} | patch'.format(commit=commit_spec) "git show -M {commit} | patch -p1 --fuzz=3 --no-backup-if-mismatch".format( commit=commit_spec ) ) # self.run_command('git reset') def tidy_up_after_patches(self): # tidy up any .origs from patches self.run_command('find . -name "*.orig" -exec rm {} \\;') def apply_listing_from_commit(self, listing): commit_spec = self.get_commit_spec(listing.commit_ref) print("Applying listing from commit.\nListing:\n" + listing.contents) files = self.get_files_from_commit_spec(commit_spec) if files != [listing.filename]: raise ApplyCommitException( f"wrong files in listing {listing.commit_ref}: {listing.filename!r} should have been {files}" ) future_contents = self.show_future_version(commit_spec, listing.filename) self._check_listing_matches_commit(listing, commit_spec, future_contents) self.patch_from_commit(listing.commit_ref, listing.filename) listing.was_written = True print("applied commit.") def _check_listing_matches_commit(self, listing, commit_spec, future_contents): commit = Commit.from_diff(self.run_command(f"git show -w {commit_spec}")) if listing.is_diff(): diff = Commit.from_diff(listing.contents) if diff.new_lines == commit.new_lines: return commit_withwhitespace = Commit.from_diff( self.run_command(f"git show {commit_spec}") ) if diff.new_lines == commit_withwhitespace.new_lines: return raise ApplyCommitException( f"diff new lines did not match.\n" f"{diff.new_lines}\n!=\n{commit.new_lines}" ) listing_lines = [strip_comments(l) for l in listing.contents.split("\n")] stripped_listing_lines = [l.strip() for l in listing_lines] for new_line in commit.new_lines: if new_line.strip() not in stripped_listing_lines: # print('stripped_listing_lines', stripped_listing_lines) raise ApplyCommitException( f"could not find commit new line {new_line!r} in listing {listing.commit_ref}:\n{listing.contents}" ) check_chunks_against_future_contents("\n".join(listing_lines), future_contents) def check_chunks_against_future_contents(listing_contents, future_contents): future_lines = future_contents.split("\n") for chunk in split_into_chunks(listing_contents): reindented_chunk = reindent_to_match(chunk, future_lines) if reindented_chunk not in future_contents: missing_lines = [ l for l in reindented_chunk.splitlines() if l not in future_lines ] if missing_lines: print("missing lines:\n" + "\n".join(repr(l) for l in missing_lines)) raise ApplyCommitException( f"{len(missing_lines)} lines did not match future contents" ) else: print("reindented listing") print("\n".join(repr(l) for l in reindented_chunk.splitlines())) print("future contents") print("\n".join(repr(l) for l in future_contents.splitlines())) tdir = Path(tempfile.mkdtemp()) print("saving to", tdir) (tdir / "listing.txt").write_text(reindented_chunk) (tdir / "future.txt").write_text(future_contents) raise ApplyCommitException( "Commit lines in wrong order, or listing is missing a [...] (?)" ) def get_offset(lines, future_lines): for line in lines: if line == "": continue if line in future_lines: return "" else: for future_line in future_lines: if future_line.endswith(line): return future_line[: -len(line)] raise Exception(f"not match found to determine offset {lines[0]!r}") def reindent_to_match(code, future_lines): offset = get_offset(code.splitlines(), future_lines) return "\n".join((offset + line) if line else "" for line in code.splitlines()) def split_into_chunks(code): chunk = "" for line in code.splitlines(): if line.strip() == "[...]": if chunk: yield chunk chunk = "" elif line.startswith("[...]"): if chunk: yield chunk chunk = "" elif line.endswith("[...]"): linestart, _, _ = line.partition("[...]") chunk += linestart yield chunk chunk = "" else: if chunk: chunk = f"{chunk}\n{line}" else: chunk = line if chunk: yield chunk ================================================ FILE: tests/test_appendix_DjangoRestFramework.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class AppendixVIITest(ChapterTest): chapter_name = 'appendix_DjangoRestFramework' previous_chapter = 'appendix_rest_api' def test_listings_and_commands_and_output(self): self.parse_listings() self.start_with_checkout() self.prep_virtualenv() # sanity checks self.assertEqual(self.listings[0].type, 'other command') self.assertEqual(self.listings[1].type, 'code listing') # skips #self.skip_with_check(22, 'switch back to master') # comment # hack fast-forward skip = False if skip: self.pos = 40 self.sourcetree.run_command('git switch {}'.format( self.sourcetree.get_commit_spec('ch36l027') )) while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) # TODO: # self.sourcetree.patch_from_commit('ch37l015') # self.sourcetree.patch_from_commit('ch37l017') # self.sourcetree.run_command( # 'git add . && git commit -m"final commit in rest api chapter"' #) if __name__ == '__main__': unittest.main() ================================================ FILE: tests/test_appendix_Django_Class-Based_Views.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class AppendixIITest(ChapterTest): chapter_name = 'appendix_Django_Class-Based_Views' previous_chapter = 'chapter_16_advanced_forms' def test_listings_and_commands_and_output(self): self.parse_listings() self.start_with_checkout() #self.prep_virtualenv() # sanity checks self.assertEqual(self.listings[0].type, 'code listing currentcontents') self.assertEqual(self.listings[1].type, 'code listing with git ref') self.assertEqual(self.listings[2].type, 'code listing with git ref') # skips #self.skip_with_check(22, 'switch back to master') # comment # hack fast-forward skip = False if skip: self.pos = 27 self.sourcetree.run_command('git switch {0}'.format( self.sourcetree.get_commit_spec('ch20l015') )) while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff(ignore=["moves"]) if __name__ == '__main__': unittest.main() ================================================ FILE: tests/test_appendix_bdd.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class AppendixVTest(ChapterTest): chapter_name = 'appendix_bdd' previous_chapter = 'chapter_23_debugging_prod' def test_listings_and_commands_and_output(self): self.parse_listings() self.start_with_checkout() #self.prep_virtualenv() # sanity checks self.assertEqual(self.listings[0].type, 'other command') self.assertEqual(self.listings[4].type, 'tree') self.assertEqual(self.listings[6].type, 'diff') self.assertEqual(self.listings[7].type, 'bdd test') # skips #self.skip_with_check(22, 'switch back to master') # comment # hack fast-forward skip = False if skip: self.pos = 27 self.sourcetree.run_command('git switch {0}'.format( self.sourcetree.get_commit_spec('ch20l015') )) while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.sourcetree.run_command('git add . && git commit -m"final commit in bdd chapter"') self.check_final_diff() if __name__ == '__main__': unittest.main() ================================================ FILE: tests/test_appendix_purist_unit_tests.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter20Test(ChapterTest): chapter_name = 'appendix_purist_unit_tests' previous_chapter = 'chapter_24_outside_in' def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, 'other command') self.assertEqual(self.listings[1].type, 'output') self.assertEqual(self.listings[4].type, 'code listing currentcontents') # skips self.skip_with_check(1, '# a branch') # comment self.skip_with_check(109, '# optional backup') # comment self.skip_with_check(112, '# reset master') # comment # prep self.start_with_checkout() # hack fast-forward skip = False if skip: self.pos = 75 self.sourcetree.run_command('git switch {0}'.format( self.sourcetree.get_commit_spec('ch19l041') )) while self.pos < len(self.listings): print(self.pos, self.listings[self.pos].type) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff(ignore=["moves"]) if __name__ == '__main__': unittest.main() ================================================ FILE: tests/test_appendix_rest_api.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class AppendixVITest(ChapterTest): chapter_name = 'appendix_rest_api' previous_chapter = 'chapter_26_page_pattern' def test_listings_and_commands_and_output(self): self.parse_listings() self.start_with_checkout() # self.prep_virtualenv() # sanity checks self.assertEqual(self.listings[0].type, 'code listing') self.assertEqual(self.listings[1].type, 'code listing') # skips #self.skip_with_check(22, 'switch back to master') # comment # hack fast-forward skip = False if skip: self.pos = 40 self.sourcetree.run_command('git switch {}'.format( self.sourcetree.get_commit_spec('ch36l027') )) while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.sourcetree.run_command( 'git add . && git commit -m"final commit in rest api chapter"' ) self.check_final_diff() if __name__ == '__main__': unittest.main() ================================================ FILE: tests/test_book_parser.py ================================================ #!/usr/bin/env python from lxml import html import re from textwrap import dedent import unittest from book_parser import ( COMMIT_REF_FINDER, CodeListing, Command, Output, get_commands, parse_listing, _strip_callouts, ) import examples class CodeListingTest(unittest.TestCase): def test_stringify(self): c = CodeListing(filename="a.py", contents="abc\ndef") assert "abc" in str(c) assert "a.py" in str(c) assert c.is_server_listing is False def test_server_codelisting(self): c = CodeListing(filename="server: a_filename.py", contents="foo") assert c.contents == "foo" assert c.filename == "a_filename.py" assert c.is_server_listing is True class CommitRefFinderTest(unittest.TestCase): def test_base_finder(self): assert re.search(COMMIT_REF_FINDER, "bla bla ch09l027-2") assert re.findall(COMMIT_REF_FINDER, "bla bla ch09l027-2") == ["ch09l027-2"] assert not re.search(COMMIT_REF_FINDER, "bla bla 09l6666") def test_finder_on_codelisting(self): matches = re.match( CodeListing.COMMIT_REF_FINDER, "some_filename.txt (ch09l027-2)" ) assert matches.group(1) == "some_filename.txt" assert matches.group(2) == "ch09l027-2" class ParseCodeListingTest(unittest.TestCase): def test_recognises_code_listings(self): code_html = examples.CODE_LISTING_WITH_CAPTION.replace("\n", "\r\n") node = html.fromstring(code_html) listings = parse_listing(node) self.assertEqual(len(listings), 1) listing = listings[0] self.assertEqual(type(listing), CodeListing) self.assertEqual(listing.filename, "functional_tests.py") self.assertEqual( listing.contents, dedent( """ from selenium import webdriver browser = webdriver.Firefox() browser.get('http://localhost:8000') assert 'Django' in browser.title """ ).strip(), ) self.assertFalse("\r" in listing.contents) self.assertEqual(listing.commit_ref, None) def test_recognises_git_commit_refs(self): code_html = examples.CODE_LISTING_WITH_CAPTION_AND_GIT_COMMIT_REF.replace( "\n", "\r\n" ) node = html.fromstring(code_html) listings = parse_listing(node) self.assertEqual(len(listings), 1) listing = listings[0] self.assertEqual(type(listing), CodeListing) self.assertEqual(listing.filename, "functional_tests/tests.py") self.assertEqual(listing.commit_ref, "ch06l001") self.assertEqual(listing.type, "code listing with git ref") def test_recognises_git_commit_refs_even_if_formatted_as_diffs(self): code_html = examples.CODE_LISTING_WITH_DIFF_FORMATING_AND_COMMIT_REF.replace( "\n", "\r\n" ) node = html.fromstring(code_html) listings = parse_listing(node) self.assertEqual(len(listings), 1) listing = listings[0] self.assertEqual(type(listing), CodeListing) self.assertEqual(listing.filename, "lists/tests/test_models.py") self.assertEqual(listing.commit_ref, "ch09l010") self.assertEqual(listing.type, "code listing with git ref") self.assertTrue(listing.is_diff()) def test_recognises_diffs_even_if_they_dont_have_atat(self): code_html = examples.EXAMPLE_DIFF_LISTING.replace("\n", "\r\n") node = html.fromstring(code_html) [listing] = parse_listing(node) self.assertEqual(listing.type, "code listing with git ref") self.assertTrue(listing.is_diff()) def test_recognises_skipme_tag_on_unmarked_code_listing(self): code_html = examples.OUTPUT_WITH_SKIPME.replace("\n", "\r\n") node = html.fromstring(code_html) listings = parse_listing(node) self.assertEqual(len(listings), 1) listing = listings[0] self.assertEqual(listing.skip, True) def test_recognises_skipme_tag_on_code_listing(self): code_html = examples.CODE_LISTING_WITH_SKIPME.replace("\n", "\r\n") node = html.fromstring(code_html) listings = parse_listing(node) self.assertEqual(len(listings), 1) listing = listings[0] self.assertEqual(listing.skip, True) def test_recognises_currentcontents_tag(self): code_html = examples.OUTPUTS_WITH_CURRENTCONTENTS.replace("\n", "\r\n") node = html.fromstring(code_html) listings = parse_listing(node) self.assertEqual(len(listings), 1) listing = listings[0] assert listing.currentcontents is True assert listing.type == "code listing currentcontents" def test_recognises_dofirst_tag(self): code_html = examples.OUTPUTS_WITH_DOFIRST.replace("\n", "\r\n") node = html.fromstring(code_html) listings = parse_listing(node) listing = listings[0] assert listing.dofirst == "ch09l058" self.assertEqual(len(listings), 2) def test_recognises_jasmine_tag(self): code_html = examples.JASMINE_OUTPUT.replace("\n", "\r\n") node = html.fromstring(code_html) listings = parse_listing(node) self.assertEqual(len(listings), 1) listing = listings[0] assert listing.type == "jasmine output" assert ( listing == dedent( """ 2 specs, 0 failures, randomized with seed 12345 finished in 0.01s Superlists tests * check we know how to hide things * sense check our html fixture """ ).strip() ) def test_recognises_server_commands(self): code_html = examples.SERVER_COMMAND.replace("\n", "\r\n") node = html.fromstring(code_html) listings = parse_listing(node) print(listings) self.assertEqual(len(listings), 1) listing = listings[0] self.assertEqual(listing.type, "server command") self.assertEqual(listing, "sudo do stuff") def test_recognises_virtualenv_commands(self): code_html = examples.COMMANDS_WITH_VIRTUALENV.replace("\n", "\r\n") node = html.fromstring(code_html) listings = parse_listing(node) print(listings) virtualenv_command = listings[1] self.assertEqual( virtualenv_command, "source ./.venv/bin/activate && python manage.py test lists", ) self.assertEqual(len(listings), 3) def test_recognises_command_with_ats(self): code_html = examples.COMMAND_MADE_WITH_ATS.replace("\n", "\r\n") node = html.fromstring(code_html) listings = parse_listing(node) print(listings) self.assertEqual(len(listings), 1) command = listings[0] self.assertEqual(command, "grep id_new_item functional_tests/tests/test*") self.assertEqual(command.type, "other command") def test_can_extract_one_command_and_its_output(self): listing = html.fromstring( '
\r\n' '
\r\n' "
$ python functional_tests.py\r\n"
            "Traceback (most recent call last):\r\n"
            '  File "functional_tests.py", line 6, in <module>\r\n'
            "    assert 'Django' in browser.title\r\n"
            "AssertionError
\r\n" "
\n" ) parsed_listings = parse_listing(listing) self.assertEqual( parsed_listings, [ "python functional_tests.py", "Traceback (most recent call last):\n" ' File "functional_tests.py", line 6, in \n' " assert 'Django' in browser.title\n" "AssertionError", ], ) self.assertEqual(type(parsed_listings[0]), Command) self.assertEqual(type(parsed_listings[1]), Output) def test_extracting_multiple(self): listing = html.fromstring( '
\r\n' '
\r\n' "
$ ls\r\n"
            "superlists          functional_tests.py\r\n"
            "$ mv functional_tests.py superlists/\r\n"
            "$ cd superlists\r\n"
            "$ git init .\r\n"
            "Initialized empty Git repository in /chapter_1/superlists/.git/
\r\n" "
\n" ) parsed_listings = parse_listing(listing) self.assertEqual( parsed_listings, [ "ls", "superlists functional_tests.py", "mv functional_tests.py superlists/", "cd superlists", "git init .", "Initialized empty Git repository in /chapter_1/superlists/.git/", ], ) self.assertEqual(type(parsed_listings[0]), Command) self.assertEqual(type(parsed_listings[1]), Output) self.assertEqual(type(parsed_listings[2]), Command) self.assertEqual(type(parsed_listings[3]), Command) self.assertEqual(type(parsed_listings[4]), Command) self.assertEqual(type(parsed_listings[5]), Output) def test_post_command_comment_with_multiple_spaces(self): listing = html.fromstring( '
' '
' "
$ git diff  # should show changes to functional_tests.py\n"
            '$ git commit -am "Functional test now checks we can input a to-do item"
' "
" ) commands = get_commands(listing) self.assertEqual( commands, [ "git diff", 'git commit -am "Functional test now checks we can input a to-do item"', ], ) parsed_listings = parse_listing(listing) self.assertEqual( parsed_listings, [ "git diff", " # should show changes to functional_tests.py", 'git commit -am "Functional test now checks we can input a to-do item"', ], ) self.assertEqual(type(parsed_listings[0]), Command) self.assertEqual(type(parsed_listings[1]), Output) self.assertEqual(type(parsed_listings[2]), Command) def test_catches_command_with_trailing_comment(self): listing = html.fromstring( dedent(""" 
$ git diff --staged # will show you the diff that you're about to commit
""") ) parsed_listings = parse_listing(listing) self.assertEqual( parsed_listings, [ "git diff --staged", " # will show you the diff that you're about to commit", ], ) self.assertEqual(type(parsed_listings[0]), Command) self.assertEqual(type(parsed_listings[1]), Output) def test_handles_multiline_commands(self): listing = html.fromstring( dedent( """ 
$ do something\\
            that continues on this line
            OK
""" ) ) commands = get_commands(listing) assert len(commands) == 1 # assert commands[0] == 'do something\\\nthat continues on this line' assert commands[0] == "do somethingthat continues on this line" # too hard for now parsed_listings = parse_listing(listing) print(parsed_listings) self.assertEqual(type(parsed_listings[0]), Command) self.assertEqual(parsed_listings[0], commands[0]) def test_handles_inline_inputs(self): listing = html.fromstring(examples.OUTPUT_WITH_COMMANDS_INLINE) commands = get_commands(listing) self.assertEqual( [str(c) for c in commands], [ "python manage.py makemigrations", "1", "''", ], ) # too hard for now parsed_listings = parse_listing(listing) print(parsed_listings) self.assertEqual(type(parsed_listings[0]), Command) self.assertEqual(parsed_listings[0], commands[0]) print(parsed_listings[1]) self.assertIn("Select an option:", parsed_listings[1]) self.assertTrue(parsed_listings[1].endswith("Select an option: ")) def test_strips_asciidoctor_callouts_from_code(self): code_html = examples.CODE_LISTING_WITH_ASCIIDOCTOR_CALLOUTS.replace( "\n", "\r\n" ) node = html.fromstring(code_html) listings = parse_listing(node) listing = listings[0] self.assertEqual(type(listing), CodeListing) self.assertNotIn("(1)", listing.contents) self.assertNotIn("(2)", listing.contents) self.assertNotIn("(3)", listing.contents) self.assertNotIn("(4)", listing.contents) self.assertNotIn("(7)", listing.contents) def test_strips_asciidoctor_callouts_from_output(self): listing_html = examples.OUTPUT_WITH_CALLOUTS.replace("\n", "\r\n") node = html.fromstring(listing_html) listings = parse_listing(node) output = listings[1] self.assertEqual(type(output), Output) self.assertNotIn("(1)", output) # self.assertIn('assertEqual(\n', output) ## TODO: re-enable def test_strip_callouts_helper(self): self.assertEqual(_strip_callouts("foo (1)"), "foo") self.assertEqual(_strip_callouts("foo (1)"), "foo") self.assertEqual(_strip_callouts("foo (112)"), "foo") self.assertEqual( _strip_callouts("line1\nline2 (2)\nline3"), "line1\nline2\nline3" ) self.assertEqual(_strip_callouts("foo (ya know) (2)"), "foo (ya know)") self.assertEqual(_strip_callouts("foo (1)\n bar (7)"), "foo\n bar") self.assertEqual(_strip_callouts("foo (1)\n bar (7)\n"), "foo\n bar\n") self.assertEqual( _strip_callouts("foo (hi)"), "foo (hi)", ) self.assertEqual( _strip_callouts("this (4) foo"), "this (4) foo", ) self.assertEqual( _strip_callouts("foo(1)"), "foo(1)", ) self.assertEqual(_strip_callouts("foo (1) (2)"), "foo") self.assertEqual(_strip_callouts(" (1)"), "") class GetCommandsTest(unittest.TestCase): def test_extracting_one_command(self): listing = html.fromstring( '
\r\n
\r\n
$ python functional_tests.py\r\nTraceback (most recent call last):\r\n  File "functional_tests.py", line 6, in <module>\r\n    assert \'Django\' in browser.title\r\nAssertionError
\r\n
\n' # noqa ) self.assertEqual(get_commands(listing), ["python functional_tests.py"]) def test_extracting_multiple(self): listing = html.fromstring( '
\r\n
\r\n
$ ls\r\nsuperlists          functional_tests.py\r\n$ mv functional_tests.py superlists/\r\n$ cd superlists\r\n$ git init .\r\nInitialized empty Git repository in /chapter_1/superlists/.git/
\r\n
\n' # noqa ) self.assertEqual( get_commands(listing), [ "ls", "mv functional_tests.py superlists/", "cd superlists", "git init .", ], ) ================================================ FILE: tests/test_book_tester.py ================================================ import os import shutil import subprocess import sys import unittest from textwrap import dedent from unittest.mock import Mock import pytest from book_parser import ( CodeListing, Command, Output, ) from book_tester import ( JASMINE_RUNNER, ChapterTest, contains, split_blocks, wrap_long_lines, ) class WrapLongLineTest(unittest.TestCase): def test_wrap_long_lines_with_words(self): self.assertEqual(wrap_long_lines("normal line"), "normal line") text = ( "This is a short line\n" "This is a long line which should wrap just before the word that " "takes it over 79 chars in length\n" "This line is fine though." ) expected_text = ( "This is a short line\n" "This is a long line which should wrap just before the word that " "takes it over\n" "79 chars in length\n" "This line is fine though." ) self.assertMultiLineEqual(wrap_long_lines(text), expected_text) def test_wrap_long_lines_with_words_2(self): text = "ViewDoesNotExist: Could not import superlists.views.home. Parent module superlists.views does not exist." expected_text = "ViewDoesNotExist: Could not import superlists.views.home. Parent module\nsuperlists.views does not exist." self.assertMultiLineEqual(wrap_long_lines(text), expected_text) def test_wrap_long_lines_with_words_3(self): text = ' File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py", line 442, in supports_transactions' expected_text = ' File "/usr/local/lib/python2.7/dist-packages/django/db/backends/__init__.py",\nline 442, in supports_transactions' self.assertMultiLineEqual(wrap_long_lines(text), expected_text) def test_wrap_long_lines_doesnt_swallow_spaces(self): text = "A really long line that uses multiple spaces to go over 80 chars by a country mile" expected_text = "A really long line that uses multiple spaces to go over 80 chars\nby a country mile" # TODO: handle trailing space corner case? self.assertMultiLineEqual(wrap_long_lines(text), expected_text) def test_wrap_long_lines_with_unbroken_chars(self): text = "." * 479 # fmt: off expected_text = ( "." * 79 + "\n" + "." * 79 + "\n" + "." * 79 + "\n" + "." * 79 + "\n" + "." * 79 + "\n" + "." * 79 + "\n" + "....." ) # fmt: on self.assertMultiLineEqual(wrap_long_lines(text), expected_text) def test_wrap_long_lines_with_unbroken_chars_2(self): text = ( "E\n" "======================================================================\n" "ERROR: test_root_url_resolves_to_home_page_view (lists.tests.HomePageTest)" ) expected_text = ( "E\n" "======================================================================\n" "ERROR: test_root_url_resolves_to_home_page_view (lists.tests.HomePageTest)" ) self.assertMultiLineEqual(wrap_long_lines(text), expected_text) def test_wrap_long_lines_with_indent(self): text = ( "This is a short line\n" " This is a long line with an indent which should wrap just " "before the word that takes it over 79 chars in length\n" " This is a short indented line\n" "This is a long line which should wrap just before the word that " "takes it over 79 chars in length" ) expected_text = ( "This is a short line\n" " This is a long line with an indent which should wrap just " "before the word\n" "that takes it over 79 chars in length\n" " This is a short indented line\n" "This is a long line which should wrap just before the word that " "takes it over\n" "79 chars in length" ) self.assertMultiLineEqual(wrap_long_lines(text), expected_text) class RunCommandTest(ChapterTest): def test_calls_sourcetree_run_command_and_marks_as_run(self): self.sourcetree.run_command = Mock() cmd = Command("foo") output = self.run_command(cmd, cwd="bar", user_input="thing") assert output == self.sourcetree.run_command.return_value self.sourcetree.run_command.assert_called_with( "foo", cwd="bar", user_input="thing", ignore_errors=False, ) assert cmd.was_run def test_raises_if_not_command(self): with self.assertRaises(AssertionError): self.run_command("foo") class GetListingsTest(ChapterTest): chapter_name = "chapter_01" def test_get_listings_gets_exampleblock_code_listings_and_regular_listings(self): self.parse_listings() self.assertEqual(self.listings[0].type, "code listing") self.assertEqual( self.listings[0].contents.split()[:3], ["from", "selenium", "import"] ) self.assertEqual(self.listings[1], "python functional_tests.py") self.assertEqual(self.listings[1].type, "test") self.assertEqual(self.listings[2].type, "output") class AssertConsoleOutputCorrectTest(ChapterTest): def test_simple_case(self): actual = "foo" expected = Output("foo") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_test_run_times_and_test_dashes(self): actual = dedent( """ bla bla bla ---------------------------------------------------------------------- Ran 1 test in 1.343s """ ).strip() expected = Output( dedent( """ bla bla bla --------------------------------------------------------------------- Ran 1 test in 1.456s """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_handles_elipsis(self): actual = dedent( """ bla bla bla loads more stuff """ ).strip() expected = Output( dedent( """ bla bla bla [...] """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_handles_elipsis_at_end_of_line_where_theres_actually_a_linebreak(self): actual = dedent( """ bla bla bla loads more stuff """ ).strip() expected = Output( dedent( """ bla bla bla [...] """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_with_start_elipsis_and_OK(self): actual = dedent( """ bla OK and some epilogue """ ).strip() expected = Output( dedent( """ [...] OK """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_with_elipsis_finds_assertionerrors(self): actual = dedent( """ bla bla bla self.assertSomething(burgle) AssertionError: nope and then there's some stuff afterwards we don't care about """ ).strip() expected = Output( dedent( """ [...] self.assertSomething(burgle) AssertionError: nope """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_with_start_elipsis_and_end_longline_elipsis(self): actual = dedent( """ bla bla bla loads more stuff raise MyException('eek') MyException: a really long exception, which will eventually wrap into multiple lines, so much so that it just gets boring after a while and we just stop caring... and then there's some stuff afterwards we don't care about """ ).strip() # noqa expected = Output( dedent( """ [...] MyException: a really long exception, which will eventually wrap into multiple lines, so much so that it just gets boring after a while and [...] """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_with_start_elipsis_and_end_longline_elipsis_with_assertionerror(self): actual = dedent( """ bla self.assertSomething(bla) AssertionError: a really long exception, which will eventually wrap into multiple lines, so much so that it gets boring after a while... and then there's some stuff afterwards we don't care about """ ).strip() expected = Output( dedent( """ [...] AssertionError: a really long exception, which will eventually [...] """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_for_short_expected_with_trailing_elipsis(self): actual = dedent( """ bla bla bla self.assertSomething(burgle) AssertionError: a long assertion error which ends up wrapping so we have to have it across two lines but then it really goes on and on and on, so much so that it gets boring and we chop it off """ # noqa ).strip() expected = Output( dedent( """ AssertionError: a long assertion error which ends up wrapping so we have to have it across two lines but then it really goes on and on [...] """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_elipsis_lines_still_checked(self): actual = dedent( """ AssertionError: a long assertion error which ends up wrapping so we have to have it across two lines but then it changes and ends up saying something different from what was expected so we shoulf fail """ # noqa ).strip() expected = Output( dedent( """ AssertionError: a long assertion error which ends up wrapping so we have to have it across two lines but then it really goes on and on [...] """ ).strip() ) with self.assertRaises(AssertionError): self.assert_console_output_correct(actual, expected) def test_with_middle_elipsis(self): actual = dedent( """ bla bla bla ERROR: the first line some more blurg something else an indented penultimate line KeyError: something more stuff happens later """ ).strip() expected = Output( dedent( """ ERROR: the first line [...] an indented penultimate line KeyError: something """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ls(self): expected = Output("superlists functional_tests.py") actual = "functional_tests.py\nsuperlists\n" self.assert_console_output_correct(actual, expected, ls=True) self.assertTrue(expected.was_checked) def test_working_directory_substitution(self): expected = Output("bla bla ...goat-book/foo stuff") actual = f"bla bla {self.tempdir}/foo stuff" self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_tabs(self): expected = Output("# bla bla") actual = "#\tbla bla" self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_diff_indexes(self): actual = dedent( """ diff --git a/functional_tests.py b/functional_tests.py index d333591..1f55409 100644 --- a/functional_tests.py """ ).strip() expected = Output( dedent( """ diff --git a/functional_tests.py b/functional_tests.py index d333591..b0f22dc 100644 --- a/functional_tests.py """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_callouts(self): actual = dedent( """ bla bla stuff """ ).strip() expected = Output( dedent( """ bla bla <12> stuff """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_asciidoctor_callouts(self): actual = dedent( """ bla bla stuff """ ).strip() expected = Output( dedent( """ bla bla (12) stuff """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_git_commit_numers_in_logs(self): actual = dedent( """ ea82222 Basic view now returns minimal HTML 7159049 First unit test and url mapping, dummy view edba758 Add app for lists, with deliberately failing unit test """ ).strip() expected = Output( dedent( """ a6e6cc9 Basic view now returns minimal HTML 450c0f3 First unit test and url mapping, dummy view ea2b037 Add app for lists, with deliberately failing unit test """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) actual = dedent( """ abc Basic view now returns minimal HTML 123 First unit test and url mapping, dummy view """ ).strip() expected = Output( dedent( """ bad Basic view now returns minimal HTML 456 First unit test and url mapping, dummy view """ ).strip() ) with self.assertRaises(AssertionError): self.assert_console_output_correct(actual, expected) def test_ignores_geckodriver_stacktrace_line_numbers(self): actual = dedent( """ Stacktrace: RemoteError@chrome://remote/content/shared/RemoteError.sys.mjs:8:8 WebDriverError@chrome://remote/content/shared/webdriver/Errors.sys.mjs:188:3 """ ).rstrip() expected = Output( dedent( """ Stacktrace: RemoteError@chrome://remote/content/shared/RemoteError.sys.mjs:9:8 WebDriverError@chrome://remote/content/shared/webdriver/Errors.sys.mjs:180:6 """ ).rstrip() ) self.assert_console_output_correct(actual, expected) def test_ignores_mock_ids(self): actual = dedent( """ self.assertEqual(user, mock_user) AssertionError: None != """ ).rstrip() expected = Output( dedent( """ self.assertEqual(user, mock_user) AssertionError: None != """ ).rstrip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_mock_ids_when_they_dont_have_names(self): actual = dedent( """ self.assertEqual(user, mock_user) AssertionError: None != """ ).rstrip() expected = Output( dedent( """ self.assertEqual(user, mock_user) AssertionError: None != """ ).rstrip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_phantomjs_run_times(self): actual = "Took 24ms to run 2 tests. 2 passed, 0 failed." expected = Output("Took 15ms to run 2 tests. 2 passed, 0 failed.") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_bdd_run_times(self): actual = "features/steps/my_lists.py:19 0.187s" expected = Output("features/steps/my_lists.py:19 0.261s") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_object_ids(self): actual = "" expected = Output("") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_migration_timestamps(self): actual = " 0005_auto_20140414_2038.py:" expected = Output(" 0005_auto_20140414_2108.py:") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_session_ids(self): actual = "qnslckvp2aga7tm6xuivyb0ob1akzzwl" expected = Output("jvhzc8kj2mkh06xooqq9iciptead20qq") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_3_5_x_AssertionError_None_thing(self): actual = "AssertionError" expected = Output("AssertionError: None") self.assert_console_output_correct(actual, expected) actual2 = "AssertionError: something" with self.assertRaises(AssertionError): self.assert_console_output_correct(actual2, expected) def test_ignores_localhost_server_port_4digits(self): actual = "//localhost:2021/my-url is a thing" expected = Output("//localhost:3339/my-url is a thing") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_localhost_server_port_5_digits(self): actual = "//localhost:40433/my-url is a thing" expected = Output("//localhost:8081/my-url is a thing") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_ignores_127_0_0_1_server_port_4digits(self): actual = "//127.0.0.1:2021/my-url is a thing" expected = Output("//127.0.0.1:3339/my-url is a thing") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_only_ignores_exactly_32_char_strings_no_whitespace(self): actual = "qnslckvp2aga7tm6xuivyb0ob1akzzwl" expected = Output("jvhzc8kj2mkh06xooqq9iciptead20qq") with self.assertRaises(AssertionError): self.assert_console_output_correct(actual[:-1], expected[:-1]) self.assert_console_output_correct(actual + "1", expected + "a") self.assert_console_output_correct(" " + actual, " " + expected) def test_ignores_selenium_trace_log_ids(self): actual = dedent( """ 1739977878464 geckodriver INFO Listening on 127.0.0.1:59905 1739977878481 webdriver::server DEBUG -> POST /session """ ) expected = dedent( """ 1739977878465 geckodriver INFO Listening on 127.0.0.1:59905 1739987878488 webdriver::server DEBUG -> POST /session """ ) self.assert_console_output_correct(actual, Output(expected)) with self.assertRaises(AssertionError): self.assert_console_output_correct( actual.replace("geckodriver", "foo"), expected.replace("geckodriver", "foo"), ) self.assert_console_output_correct( actual.replace("webdriver", "foo"), expected.replace("webdriver", "foo"), ) def test_ignores_firefox_esr_version(self): expected = "1234567890111 geckodriver::capabilities DEBUG Found version\n128.10esr" actual = "1747863999574 geckodriver::capabilities DEBUG Found version 128.10.1esr" self.assert_console_output_correct(actual, Output(expected)) with self.assertRaises(AssertionError): self.assert_console_output_correct( actual.replace("128.10.1esr", "1234abc"), Output(expected), ) actual2 = "1234567890111 geckodriver::capabilities DEBUG Found version 140.3esr" self.assert_console_output_correct(actual2, Output(expected)) def test_ignores_docker_image_ids_and_creation_time(self): actual = "superlists latest 522824a399de 2 weeks ago 164MB" expected = Output("superlists latest 522824a399de 2 minutes ago 164MB") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) with self.assertRaises(AssertionError): bad_actual = "geoff latest 522824a399de 2 weeks ago 164MB" self.assert_console_output_correct(bad_actual, expected) def test_ignores_minor_differences_in_curl_output1(self): actual = "* Trying ::1:8888..." expected = Output("* Trying [::1]:8888...") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) with self.assertRaises(AssertionError): bad_actual = "* Trying ::1:9999..." self.assert_console_output_correct(bad_actual, expected) bad_actual = "* Trying [::]1:9999..." self.assert_console_output_correct(bad_actual, expected) def test_ignores_minor_differences_in_curl_output2(self): actual = "* Closing connection" expected = Output("* Closing connection 0") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) with self.assertRaises(AssertionError): bad_actual = "Closing Geoff" self.assert_console_output_correct(bad_actual, expected) def test_ignores_minor_differences_in_curl_output3(self): actual = "* Connected to localhost (127.0.0.1) port 8888 (#0)" expected = Output("* Connected to localhost (127.0.0.1) port 8888") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) with self.assertRaises(AssertionError): bad_actual = "* Connected to localhost (127.0.0.1) port 8889 (#0)" self.assert_console_output_correct(bad_actual, expected) def test_ignores_minor_differences_in_curl_output4(self): actual = "*> User-Agent: curl/7.81.0" expected = Output("*> User-Agent: curl/8.6.0") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) with self.assertRaises(AssertionError): bad_actual = "Closing Geoff" self.assert_console_output_correct(bad_actual, expected) def test_ignores_minor_differences_in_curl_output5(self): actual = "0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying [::1]:8888..." expected = Output("* Trying ::1:8888...") self.assert_console_output_correct(actual, expected) with self.assertRaises(AssertionError): bad_actual = "10* Hi" expected = Output("*Hi") self.assert_console_output_correct(bad_actual, expected) def test_ignores_git_localisation_uk_vs_usa(self): actual = "Initialized empty Git repository in somewhere/.git/" expected = Output("Initialised empty Git repository in somewhere/.git/") self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) with self.assertRaises(AssertionError): bad_actual = "Error initializing Git repo" self.assert_console_output_correct(bad_actual, expected) def test_ignores_screenshot_times(self): actual = ( "screenshotting to ...goat-book/functional_tests/screendumps/MyListsTes\n" "t.test_logged_in_users_lists_are_saved_as_my_lists-window0-2024-03-09T11.39.38.\n" "png\n" "dumping page HTML to ...goat-book/functional_tests/screendumps/MyLists\n" "Test.test_logged_in_users_lists_are_saved_as_my_lists-window0-2024-03-09T11.39.\n" "38.html\n" ) expected = Output( "screenshotting to ...goat-book/functional_tests/screendumps/MyListsTes\n" "t.test_logged_in_users_lists_are_saved_as_my_lists-window0-2013-04-09T13.40.39.\n" "png\n" "dumping page HTML to ...goat-book/functional_tests/screendumps/MyLists\n" "Test.test_logged_in_users_lists_are_saved_as_my_lists-window0-2024-04-04T12.43.\n" "42.html\n" ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_matches_system_vs_virtualenv_install_paths(self): actual = dedent( """ File "/home/harry/.virtualenvs/Book/lib/python3.4/site-packages/django/core/urlresolvers.py", line 521, in resolve return get_resolver(urlconf).resolve(path) """ ).rstrip() expected = Output( dedent( """ File "...-packages/django/core/urlresolvers.py", line 521, in resolve return get_resolver(urlconf).resolve(path) """ ).rstrip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) incorrect_actual = dedent( """ File "/home/harry/.virtualenvs/Book/lib/python3.4/site-packages/django/core/urlresolvers.py", line 522, in resolve return get_resolver(urlconf).resolve(path) """ ).rstrip() with self.assertRaises(AssertionError): self.assert_console_output_correct(incorrect_actual, expected) incorrect_actual = dedent( """ File "/home/harry/.virtualenvs/Book/lib/python3.4/site-packages/django/core/another_file.py", line 521, in resolve return get_resolver(urlconf).resolve(path) """ ).rstrip() with self.assertRaises(AssertionError): self.assert_console_output_correct(incorrect_actual, expected) def test_fixes_stdout_stderr_for_creating_db(self): actual = dedent( """ ====================================================================== FAIL: test_basic_addition (lists.tests.SimpleTest) ---------------------------------------------------------------------- Traceback etc ---------------------------------------------------------------------- Ran 1 tests in X.Xs FAILED (failures=1) Creating test database for alias 'default'... Destroying test database for alias 'default' """ ).strip() expected = Output( dedent( """ Creating test database for alias 'default'... ====================================================================== FAIL: test_basic_addition (lists.tests.SimpleTest) ---------------------------------------------------------------------- Traceback etc ---------------------------------------------------------------------- Ran 1 tests in X.Xs FAILED (failures=1) Destroying test database for alias 'default' """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_handles_long_lines(self): actual = dedent( """ A normal line An indented line, that's longer than 80 chars. it goes on for a while you see. a normal indented line """ ).strip() expected = Output( dedent( """ A normal line An indented line, that's longer than 80 chars. it goes on for a while you see. a normal indented line """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_for_minimal_expected(self): actual = dedent( """ Creating test database for alias 'default'... E ====================================================================== ERROR: test_root_url_resolves_to_home_page_view (lists.tests.HomePageTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "...goat-book/lists/tests.py", line 8, in test_root_url_resolves_to_home_page_view found = resolve('/') File "/usr/local/lib/python2.7/dist-packages/django/core/urlresolvers.py", line 440, in resolve return get_resolver(urlconf).resolve(path) File "/usr/local/lib/python2.7/dist-packages/django/core/urlresolvers.py", line 104, in get_callable (lookup_view, mod_name)) ViewDoesNotExist: Could not import superlists.views.home. Parent module superlists.views does not exist. ---------------------------------------------------------------------- Ran 1 tests in X.Xs FAILED (errors=1) Destroying test database for alias 'default'... """ ).strip() expected = Output( dedent( """ ViewDoesNotExist: Could not import superlists.views.home. Parent module superlists.views does not exist. """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) def test_for_long_traceback(self): with open( os.path.join(os.path.dirname(__file__), "actual_manage_py_test.output") ) as f: actual = f.read().strip() expected = Output( dedent( """ [... lots and lots of traceback] Traceback (most recent call last): File "[...]-packages/django/test/testcases.py", line 259, in __call__ self._pre_setup() File "[...]-packages/django/test/testcases.py", line 479, in _pre_setup self._fixture_setup() File "[...]-packages/django/test/testcases.py", line 829, in _fixture_setup if not connections_support_transactions(): File "[...]-packages/django/test/testcases.py", line 816, in connections_support_transactions for conn in connections.all()) File "[...]-packages/django/test/testcases.py", line 816, in for conn in connections.all()) File "[...]-packages/django/utils/functional.py", line 43, in __get__ res = instance.__dict__[self.func.__name__] = self.func(instance) File "[...]-packages/django/db/backends/__init__.py", line 442, in supports_transactions self.connection.enter_transaction_management() File "[...]-packages/django/db/backends/dummy/base.py", line 15, in complain raise ImproperlyConfigured("settings.DATABASES is improperly configured. " ImproperlyConfigured: settings.DATABASES is improperly configured. Please supply the ENGINE value. Check settings documentation for more details. --------------------------------------------------------------------- Ran 85 tests in 0.788s FAILED (errors=404, skipped=1) AttributeError: _original_allowed_hosts """ ).strip() ) self.assert_console_output_correct(actual, expected) self.assertTrue(expected.was_checked) class CurrentContentsTest(ChapterTest): def test_ok_for_correct_current_contents(self): actual_contents = dedent( """ line 0 line 1 line 2 line 3 line 4 """ ) listing = CodeListing( filename="file2.txt", contents=dedent( """ line 1 line 2 line 3 """ ).lstrip(), ) self.check_current_contents(listing, actual_contents) # should not raise def test_raises_for_any_line_not_in_actual_contents(self): actual_contents = dedent( """ line 0 line 1 line 2 line 3 line 4 """ ) listing = CodeListing( filename="file2.txt", contents=dedent( """ line 3 line 4 line 5 """ ).lstrip(), ) with self.assertRaises(AssertionError): self.check_current_contents(listing, actual_contents) def test_indentation_is_ignored(self): actual_contents = dedent( """ line 0 line 1 line 2 line 3 """ ) listing = CodeListing( filename="file2.txt", contents=dedent( """ line 1 line 2 line 3 """ ).lstrip(), ) self.check_current_contents(listing, actual_contents) def test_raises_if_lines_not_in_order(self): actual_contents = dedent( """ line 1 line 2 line 3 line 4 """ ) listing = CodeListing( filename="file2.txt", contents=dedent( """ line 1 line 3 line 2 """ ).lstrip(), ) listing.currentcontents = True with self.assertRaises(AssertionError): self.check_current_contents(listing, actual_contents) def test_checks_elipsis_blocks_separately(self): actual_contents = dedent( """ line 1 line 2 line 3 line 4 line 5 """ ) listing = CodeListing( filename="file2.txt", contents=dedent( """ line 1 line 2 [...] line 4 """ ).lstrip(), ) listing.currentcontents = True self.check_current_contents(listing, actual_contents) # should not raise def test_checks_ignores_blank_lines(self): actual_contents = dedent( """ line 1 line 2 line 3 line 4 line 5 """ ) listing = CodeListing( filename="file2.txt", contents=dedent( """ line 1 line 2 line 3 line 4 """ ).lstrip(), ) listing.currentcontents = True self.check_current_contents(listing, actual_contents) # should not raise listing2 = CodeListing( filename="file2.txt", contents=dedent( """ line 1 line 2 line 3 line 4 """ ).lstrip(), ) with self.assertRaises(AssertionError): self.check_current_contents(listing2, actual_contents) class SplitBlocksTest(unittest.TestCase): def test_splits_on_multi_newlines(self): assert split_blocks( dedent( """ this is block 1 this is block 2 """ ) ) == ["this\nis block 1", "this is block 2"] def test_splits_on_elipsis(self): assert split_blocks( dedent( """ this is block 1 [...] this is block 2 """ ) ) == ["this\nis block 1", "this is block 2"] class TestContains: def test_smoketest(self): assert contains([1, 2, 3, 4], [1, 2]) def test_contains_end_seq(self): assert contains([1, 2, 3, 4], [3, 4]) def test_contains_middle_seq(self): assert contains([1, 2, 3, 4, 5], [3, 4]) def test_contains_oversized_seq(self): assert contains([1, 2, 3, 4, 4], [1, 2, 3, 4]) def test_contains_iteslf(self): assert contains([1, 2, 3], [1, 2, 3]) @pytest.mark.skipif(not shutil.which("phantomjs"), reason="PhantomJS not available") class CheckQunitOuptutTest(ChapterTest): def test_partial_listing_passes(self): self.chapter_name = "chapter_17_javascript" self.sourcetree.start_with_checkout( "chapter_18_second_deploy", "chapter_17_javascript" ) expected = Output("2 assertions of 2 passed, 0 failed.") self.check_qunit_output(expected) # should pass assert expected.was_checked def test_fails_if_lists_fail_and_no_accounts(self): self.chapter_name = "chapter_17_javascript" self.sourcetree.start_with_checkout( "chapter_18_second_deploy", "chapter_17_javascript" ) with self.assertRaises(AssertionError): self.check_qunit_output(Output("arg")) def TODOtest_runs_phantomjs_runner_against_lists_tests(self): self.chapter_name = "chapter_17_javascript" self.sourcetree.start_with_checkout( "chapter_18_second_deploy", "chapter_17_javascript" ) lists_tests = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../source/chapter_17_javascript/superlists/lists/static/tests/tests.html", ) manual_run = subprocess.check_output( ["python", JASMINE_RUNNER, lists_tests], ) expected = Output(manual_run.strip().decode()) self.check_jasmine_output(expected) # should pass class CheckFinalDiffTest(ChapterTest): chapter_name = "chapter_01" def test_empty_passes(self): self.run_command = lambda _: "" self.check_final_diff() # should pass def test_diff_fails(self): diff = dedent( """ + a missing line - a line that was wrong bla """ ) self.run_command = lambda _: diff with self.assertRaises(AssertionError): self.check_final_diff() def test_blank_lines_ignored(self): diff = dedent( """ + - bla """ ) self.run_command = lambda _: diff self.check_final_diff() # should pass def test_ignore_moves(self): diff = dedent( """ + some + block stuff - some - block bla """ ) self.run_command = lambda _: diff with self.assertRaises(AssertionError): self.check_final_diff() self.check_final_diff(ignore=["moves"]) # should pass with self.assertRaises(AssertionError): diff += "\n+a genuinely different line" self.check_final_diff(ignore=["moves"]) def test_ignore_secret_key_and_generated_by_django(self): diff = dedent( """ diff --git a/superlists/settings.py b/superlists/settings.py index 7463a4c..6eb4bde 100644 --- a/superlists/settings.py +++ b/superlists/settings.py @@ -17,7 +17,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(__file__)) # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '!x8-9w9o%s#c8u(4^zb9n2g(xy4q*@c^$9axl2o48wkz(v%_!*' +SECRET_KEY = 'y)exet(6z6z6)(b!v1m8it$a0q^e=b^#*r8a2o5er1u(=sl=7f' # SECURITY WARNING: don't run with debug turned on in production! -# Generated by Django 1.10.3 on 2016-12-01 21:11 +# Generated by Django 1.10.3 on 2016-12-02 10:19 from __future__ import unicode_literals """ ) self.run_command = lambda _: diff with self.assertRaises(AssertionError): self.check_final_diff() self.check_final_diff( ignore=["SECRET_KEY", "Generated by Django 1.10"] ) # should pass with self.assertRaises(AssertionError): diff += "\n+a genuinely different line" self.check_final_diff(ignore=["SECRET_KEY", "Generated by Django 1.10"]) def test_ignore_moves_and_custom(self): diff = dedent( """ + some + block stuff - some - block bla + ignore me """ ) self.run_command = lambda _: diff with self.assertRaises(AssertionError): self.check_final_diff() self.check_final_diff(ignore=["moves", "ignore me"]) # should pass with self.assertRaises(AssertionError): diff += "\n+a genuinely different line" self.check_final_diff(ignore=["moves", "ignore me"]) ================================================ FILE: tests/test_chapter_01.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import unittest from book_parser import Output from book_tester import ChapterTest, CodeListing, write_to_file from update_source_repo import update_sources_for_chapter os.environ["LC_ALL"] = "en_GB.UTF-8" os.environ["LANG"] = "en_GB.UTF-8" os.environ["LANGUAGE"] = "en_GB.UTF-8" class Chapter1Test(ChapterTest): chapter_name = "chapter_01" def write_to_file(self, codelisting): # override write to file, in this chapter cwd is root tempdir print("writing to file", codelisting.filename) write_to_file(codelisting, os.path.join(self.tempdir)) print("wrote", open(os.path.join(self.tempdir, codelisting.filename)).read()) def test_listings_and_commands_and_output(self): update_sources_for_chapter(self.chapter_name, previous_chapter=None) self.parse_listings() # self.fail('\n'.join(f'{l.type}: {l}' for l in self.listings)) # sanity checks self.assertEqual(type(self.listings[0]), CodeListing) self.skip_with_check(6, "Performing system checks...") # after runserver self.listings[8] = Output(str(self.listings[8]).replace("$", "")) # prep folder as it would be self.sourcetree.run_command("mkdir -p .venv/bin") self.sourcetree.run_command("mkdir -p .venv/lib") self.unset_PYTHONDONTWRITEBYTECODE() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) # manually add repo, we didn't do it at the beginning local_repo_path = os.path.abspath( os.path.join(os.path.dirname(__file__), "../source/chapter_01/superlists") ) self.sourcetree.run_command('git remote add repo "{}"'.format(local_repo_path)) self.sourcetree.run_command("git fetch repo") self.check_final_diff( ignore=[ "SECRET_KEY", "Generated by 'django-admin startproject' using Django 5.2.", ] ) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_02_unittest.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import unittest from book_tester import ( ChapterTest, CodeListing, Command, ) class Chapter2Test(ChapterTest): chapter_name = 'chapter_02_unittest' previous_chapter = 'chapter_01' def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(type(self.listings[0]), CodeListing) self.assertEqual(type(self.listings[2]), Command) self.start_with_checkout() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff() if __name__ == '__main__': unittest.main() ================================================ FILE: tests/test_chapter_03_unit_test_first_view.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import unittest import time from book_tester import ( ChapterTest, CodeListing, Command, Output, ) class Chapter3Test(ChapterTest): chapter_name = 'chapter_03_unit_test_first_view' previous_chapter = 'chapter_02_unittest' def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(type(self.listings[0]), Command) self.assertEqual(type(self.listings[1]), Output) self.assertEqual(type(self.listings[2]), CodeListing) self.skip_with_check(10, 'will show you') final_ft = 43 self.assertIn('Finish the test', self.listings[final_ft + 1]) self.start_with_checkout() self.start_dev_server() self.unset_PYTHONDONTWRITEBYTECODE() print(self.pos) assert 'manage.py startapp lists' in self.listings[self.pos] self.recognise_listing_and_process_it() time.sleep(1) # voodoo sleep, otherwise db.sqlite3 doesnt appear in CI sometimes while self.pos < final_ft: print(self.pos) self.recognise_listing_and_process_it() self.restart_dev_server() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff() if __name__ == '__main__': unittest.main() ================================================ FILE: tests/test_chapter_04_philosophy_and_refactoring.py ================================================ #!/usr/bin/env python3 import time import unittest from book_tester import ( ChapterTest, CodeListing, Command, Output, ) class Chapter4Test(ChapterTest): chapter_name = "chapter_04_philosophy_and_refactoring" previous_chapter = "chapter_03_unit_test_first_view" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(type(self.listings[0]), Command) self.assertEqual(type(self.listings[1]), Output) self.assertEqual(type(self.listings[2]), CodeListing) self.start_with_checkout() self.start_dev_server() self.skip_with_check(38, "add the untracked templates folder") self.skip_with_check(40, "review the changes") while self.pos < len(self.listings): print(self.pos, self.listings[self.pos].type) time.sleep(0.5) # let runserver fs watcher catch up self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff() if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_05_post_and_database.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ( ChapterTest, CodeListing, Command, Output, ) class Chapter5Test(ChapterTest): chapter_name = "chapter_05_post_and_database" previous_chapter = "chapter_04_philosophy_and_refactoring" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(type(self.listings[0]), Output) self.assertEqual(type(self.listings[1]), CodeListing) self.assertEqual(type(self.listings[3]), Command) views_pos = 21 self.find_with_check(views_pos, "def home_page") nutemplate_pos = 95 nl = self.find_with_check(nutemplate_pos, '{"items": items}') print(nl) migrate_pos = 99 ml = self.find_with_check(migrate_pos, "migrate") assert ml.type == "interactive manage.py" self.start_with_checkout() self.start_dev_server() self.unset_PYTHONDONTWRITEBYTECODE() restarted_after_views = False restarted_after_migrate = False restarted_after_nutemplate = False while self.pos < len(self.listings): print(self.pos) if self.pos > views_pos and not restarted_after_views: self.restart_dev_server() restarted_after_views = True if self.pos > migrate_pos and not restarted_after_migrate: self.restart_dev_server() restarted_after_migrate = True if self.pos > nutemplate_pos and not restarted_after_nutemplate: self.restart_dev_server() restarted_after_nutemplate = True self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff( ignore=[ "moves", "Generated by Django 5.", ] ) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_06_explicit_waits_1.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ( ChapterTest, Command, ) class Chapter6Test(ChapterTest): chapter_name = "chapter_06_explicit_waits_1" previous_chapter = "chapter_05_post_and_database" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(type(self.listings[0]), Command) self.assertEqual(type(self.listings[1]), Command) self.assertEqual(type(self.listings[2]), Command) # skips self.skip_with_check(15, "msg eg") # git # other prep self.start_with_checkout() self.unset_PYTHONDONTWRITEBYTECODE() self.run_command(Command("python3 manage.py migrate --noinput")) # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff() if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_07_working_incrementally.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ( ChapterTest, Command, ) class Chapter7Test(ChapterTest): chapter_name = "chapter_07_working_incrementally" previous_chapter = "chapter_06_explicit_waits_1" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "output") self.assertEqual(self.listings[1].type, "output") # skips self.skip_with_check(61, "should show 4 changed files") # git self.skip_with_check(66, "add a message summarising") # git self.skip_with_check(87, "5 changed files") # git self.skip_with_check(89, "forms x2") # git self.skip_with_check(116, "3 changed files") # git # other prep self.start_with_checkout() self.run_command(Command("python3 manage.py migrate --noinput")) # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.check_final_diff(ignore=["moves", "Generated by Django 5."]) self.assert_all_listings_checked(self.listings) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_08_prettification.py ================================================ #!/usr/bin/env python3 import unittest from book_parser import Command, Output from book_tester import ChapterTest class Chapter8Test(ChapterTest): chapter_name = "chapter_08_prettification" previous_chapter = "chapter_07_working_incrementally" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(type(self.listings[1]), Command) self.assertEqual(type(self.listings[2]), Output) self.start_with_checkout() # other prep self.sourcetree.run_command("python3 manage.py migrate --noinput") # self.unset_PYTHONDONTWRITEBYTECODE() self.prep_virtualenv() self.sourcetree.run_command("uv pip install pip") # skips self.skip_with_check(24, "the -w means ignore whitespace") self.skip_with_check(27, "leave static, for now") self.skip_with_check(52, "will now show all the bootstrap") # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff(ignore=["moves"]) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_09_docker.py ================================================ #!/usr/bin/env python3 import os import unittest from book_tester import ChapterTest class Chapter9Test(ChapterTest): chapter_name = "chapter_09_docker" previous_chapter = "chapter_08_prettification" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "test") # skips: # docker build output, we want to run the 'docker build' # but not check output self.skip_with_check(29, "naming to docker.io/library/superlists") self.skip_with_check(36, "naming to docker.io/library/superlists") self.skip_with_check(36, "naming to docker.io/library/superlists") # normal git one self.skip_with_check(82, "add Dockerfile, .dockerignore, .gitignore") self.start_with_checkout() # simulate having a db.sqlite3 and a static folder from previous chaps self.sourcetree.run_command("./manage.py migrate --noinput") self.sourcetree.run_command("./manage.py collectstatic --noinput") # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): listing = self.listings[self.pos] print(self.pos, listing.type, repr(listing)) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff() if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_10_production_readiness.py ================================================ #!/usr/bin/env python3 import unittest from pathlib import Path from book_tester import ChapterTest THIS_DIR = Path(__file__).parent class Chapter10Test(ChapterTest): chapter_name = "chapter_10_production_readiness" previous_chapter = "chapter_09_docker" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "other command") self.assertEqual(self.listings[3].type, "docker run tty") self.start_with_checkout() self.prep_virtualenv() self.prep_database() # skips self.skip_with_check(47, "should show dockerfile") self.skip_with_check(50, "should now be clean") self.skip_with_check(55, "Change the owner") self.skip_with_check(57, "Change the file to be group-writeable as well") self.skip_with_check(61, "note container id") # hack fast-forward, nu way self.skip_forward_if_skipto_set() while self.pos < len(self.listings): listing = self.listings[self.pos] print(self.pos, listing.type, repr(listing)) self.recognise_listing_and_process_it() self.check_final_diff( ignore=[ "Django==5.2", "gunicorn==2", "whitenoise==6.", ] ) self.assert_all_listings_checked(self.listings) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_11_server_prep.py ================================================ #!/usr/bin/env python3 import os import unittest from book_tester import ChapterTest class Chapter11Test(ChapterTest): chapter_name = "chapter_11_server_prep" previous_chapter = "chapter_10_production_readiness" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "server command") self.assertEqual(self.listings[1].type, "server command") self.assertEqual(self.listings[2].type, "output") self.start_with_checkout() self.prep_virtualenv() # self.sourcetree.run_command('mkdir -p static/stuff') # skips # self.skip_with_check(13, "we also need the Docker") # vm_restore = 'MANUAL_END' # hack fast-forward self.skip_forward_if_skipto_set() # if DO_SERVER_COMMANDS: # subprocess.check_call(['vagrant', 'snapshot', 'restore', vm_restore]) # # self.current_server_cd = '~/sites/$SITENAME' while self.pos < len(self.listings): listing = self.listings[self.pos] print(self.pos, listing.type, repr(listing)) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.sourcetree.run_command("git add . && git commit -m ch11") self.check_final_diff(ignore=["gunicorn==19"]) # if DO_SERVER_COMMANDS: # subprocess.run(['vagrant', 'snapshot', 'delete', 'MAKING_END'], check=False) # subprocess.run(['vagrant', 'snapshot', 'save', 'MAKING_END'], check=True) # if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_12_ansible.py ================================================ #!/usr/bin/env python3 import os import unittest from book_tester import ChapterTest class Chapter12Test(ChapterTest): chapter_name = "chapter_12_ansible" previous_chapter = "chapter_11_server_prep" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "against staging") self.start_with_checkout() self.prep_virtualenv() self.prep_database() # self.sourcetree.run_command('mkdir -p static/stuff') # skips self.skip_with_check(62, "git diff") self.skip_with_check(63, "should show our changes") # vm_restore = 'MANUAL_END' # hack fast-forward self.skip_forward_if_skipto_set() # if DO_SERVER_COMMANDS: # subprocess.check_call(['vagrant', 'snapshot', 'restore', vm_restore]) # # self.current_server_cd = '~/sites/$SITENAME' while self.pos < len(self.listings): listing = self.listings[self.pos] print(self.pos, listing.type, repr(listing)) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff(ignore=["gunicorn==19"]) # if DO_SERVER_COMMANDS: # subprocess.run(['vagrant', 'snapshot', 'delete', 'MAKING_END'], check=False) # subprocess.run(['vagrant', 'snapshot', 'save', 'MAKING_END'], check=True) # if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_13_organising_test_files.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter13Test(ChapterTest): chapter_name = "chapter_13_organising_test_files" previous_chapter = "chapter_12_ansible" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "code listing with git ref") self.assertEqual(self.listings[2].type, "test") # other prep self.start_with_checkout() self.prep_database() # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos, self.listings[self.pos].type) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff( ignore=[ # "django==1.11" ] ) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_14_database_layer_validation.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter13Test(ChapterTest): chapter_name = "chapter_14_database_layer_validation" previous_chapter = "chapter_13_organising_test_files" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "test") self.assertEqual(self.listings[1].type, "output") self.assertEqual(self.listings[2].type, "code listing with git ref") # other prep self.start_with_checkout() self.prep_database() # self.skip_with_check(5, "equivalent to running sqlite3") # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos, self.listings[self.pos].type) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff() if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_15_simple_form.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter15Test(ChapterTest): chapter_name = "chapter_15_simple_form" previous_chapter = "chapter_14_database_layer_validation" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "code listing with git ref") self.assertEqual(self.listings[2].type, "output") # skips # self.skip_with_check(31, "# review changes") # diff # prep self.start_with_checkout() self.prep_database() # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff(ignore=["moves"]) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_16_advanced_forms.py ================================================ #!/usr/bin/env python3 import os import unittest from book_tester import ChapterTest class Chapter16Test(ChapterTest): chapter_name = "chapter_16_advanced_forms" previous_chapter = "chapter_15_simple_form" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "test") self.assertEqual(self.listings[2].type, "output") # prep self.start_with_checkout() self.prep_database() # skips self.skip_with_check(29, "# should show changes") # diff # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos, self.listings[self.pos].type) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff(ignore=["Generated by Django 5."]) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_17_javascript.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter16Test(ChapterTest): chapter_name = "chapter_17_javascript" previous_chapter = "chapter_16_advanced_forms" def test_listings_and_commands_and_output(self): self.parse_listings() self.start_with_checkout() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "test") self.assertEqual(self.listings[2].type, "output") # skip some inline bash comments self.skip_with_check(15, "if you're on Windows") self.skip_with_check(17, "delete all the other stuff") self.skip_with_check(74, "all our js") self.skip_with_check(76, "changes to the base template") # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.sourcetree.run_command('git add . && git commit -m"final commit"') self.check_final_diff() if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_19_spiking_custom_auth.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter19Test(ChapterTest): chapter_name = "chapter_19_spiking_custom_auth" previous_chapter = "chapter_18_second_deploy" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks # self.assertEqual(self.listings[0].type, 'other command') self.assertEqual(self.listings[1].type, "code listing with git ref") self.assertEqual(self.listings[2].type, "other command") # self.assertTrue(self.listings[88].dofirst) # skips self.skip_with_check(34, "switch back to main") # comment self.skip_with_check(36, "remove any trace") # comment # prep self.start_with_checkout() self.prep_database() # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) # and do a final commit self.sourcetree.run_command('git add . && git commit -m"final commit"') self.check_final_diff(ignore=["Generated by Django 5."]) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_20_mocking_1.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter20Test(ChapterTest): chapter_name = "chapter_20_mocking_1" previous_chapter = "chapter_19_spiking_custom_auth" def test_listings_and_commands_and_output(self): self.parse_listings() self.start_with_checkout() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "code listing with git ref") self.assertEqual(self.listings[2].type, "test") # skips # self.skip_with_check(22, 'switch back to master') # comment self.prep_database() # self.sourcetree.run_command("rm src/accounts/tests.py") self.sourcetree.run_command("mkdir -p src/static") # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.sourcetree.run_command( 'git add . && git commit -m"final commit in chap 19"' ) self.check_final_diff(ignore=["moves"]) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_21_mocking_2.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter21Test(ChapterTest): chapter_name = "chapter_21_mocking_2" previous_chapter = "chapter_20_mocking_1" def test_listings_and_commands_and_output(self): self.parse_listings() self.start_with_checkout() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "code listing") self.assertEqual(self.listings[1].skip, True) self.assertEqual(self.listings[2].type, "code listing with git ref") # skips # self.skip_with_check(22, 'switch back to master') # comment self.prep_database() # self.sourcetree.run_command("rm src/accounts/tests.py") self.sourcetree.run_command("mkdir -p src/static") # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff(ignore=["moves"]) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_22_fixtures_and_wait_decorator.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter22Test(ChapterTest): chapter_name = "chapter_22_fixtures_and_wait_decorator" previous_chapter = "chapter_21_mocking_2" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "code listing with git ref") # skips # self.skip_with_check(22, 'switch back to master') # comment # prep self.start_with_checkout() self.prep_database() # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.sourcetree.tidy_up_after_patches() self.sourcetree.run_command('git add . && git commit -m"final commit ch17"') self.check_final_diff(ignore=["moves"]) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_23_debugging_prod.py ================================================ #!/usr/bin/env python3.7 import unittest from book_tester import ChapterTest class Chapter18Test(ChapterTest): chapter_name = "chapter_23_debugging_prod" previous_chapter = "chapter_22_fixtures_and_wait_decorator" def test_listings_and_commands_and_output(self): self.parse_listings() # sanity checks self.assertEqual(self.listings[0].type, "docker run tty") self.assertEqual(self.listings[1].type, "output") # skips self.skip_with_check(1, "naming to docker") # self.replace_command_with_check( # 13, # "EMAIL_PASSWORD=yoursekritpasswordhere", # "EMAIL_PASSWORD=" + os.environ["EMAIL_PASSWORD"], # ) # deploy_pos = 49 # assert "ansible-playbook" in self.listings[deploy_pos] # prep self.start_with_checkout() self.prep_database() self.sourcetree.run_command("touch container.db.sqlite3") self.sourcetree.run_command("sudo chown 1234 container.db.sqlite3") # for macos, see chap 10 self.sourcetree.run_command("sudo chmod g+rw container.db.sqlite3") # vm_restore = "FABRIC_END" # hack fast-forward self.skip_forward_if_skipto_set() # if DO_SERVER_COMMANDS: # subprocess.check_call(["vagrant", "snapshot", "restore", vm_restore]) while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.sourcetree.tidy_up_after_patches() self.sourcetree.run_command('git add . && git commit -m"final commit ch17"') self.check_final_diff(ignore=["moves", "YAHOO_PASSWORD"]) # if DO_SERVER_COMMANDS: # subprocess.check_call(["vagrant", "snapshot", "save", "SERVER_DEBUGGED"]) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_24_outside_in.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter24Test(ChapterTest): chapter_name = "chapter_24_outside_in" previous_chapter = "chapter_23_debugging_prod" def test_listings_and_commands_and_output(self): self.parse_listings() # self.prep_virtualenv() # sanity checks self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "code listing with git ref") # skips self.skip_with_check(42, 'views.py, templates') self.start_with_checkout() self.prep_database() # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos, self.listings[self.pos].type) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.check_final_diff(ignore=["moves", "Generated by Django 5"]) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_25_CI.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter25Test(ChapterTest): chapter_name = "chapter_25_CI" previous_chapter = "chapter_24_outside_in" def test_listings_and_commands_and_output(self): self.parse_listings() self.start_with_checkout() # self.prep_virtualenv() # sanity checks self.assertEqual(self.listings[0].skip, True) self.assertEqual(self.listings[1].skip, True) self.assertEqual(self.listings[10].type, "code listing with git ref") # skips # self.skip_with_check(22, 'switch back to master') # comment # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.sourcetree.run_command( "git add .gitlab-ci.yml", ) # TODO: test package.json # self.sourcetree.run_command( # "git add src/lists/static/package.json src/lists/static/tests" # ) self.sourcetree.run_command("git commit -m'final commit'") # self.check_final_diff(ignore=["moves"]) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_chapter_26_page_pattern.py ================================================ #!/usr/bin/env python3 import unittest from book_tester import ChapterTest class Chapter26Test(ChapterTest): chapter_name = "chapter_26_page_pattern" previous_chapter = "chapter_25_CI" def test_listings_and_commands_and_output(self): self.parse_listings() self.start_with_checkout() # self.prep_virtualenv() # sanity checks # self.assertEqual(self.listings[0].type, 'code listing') self.assertEqual(self.listings[0].type, "code listing with git ref") self.assertEqual(self.listings[1].type, "test") self.assertEqual(self.listings[2].type, "output") # skips # self.skip_with_check(22, 'switch back to master') # comment # hack fast-forward self.skip_forward_if_skipto_set() while self.pos < len(self.listings): print(self.pos) self.recognise_listing_and_process_it() self.assert_all_listings_checked(self.listings) self.sourcetree.tidy_up_after_patches() # final branch includes a suggested implementation... # so just check diff up to the last listing commit = self.sourcetree.get_commit_spec("ch26l013") diff = self.sourcetree.run_command(f"git diff -b {commit}") self.check_final_diff(ignore=["moves"], diff=diff) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_source_updater.py ================================================ #!/usr/bin/env python3 import unittest import tempfile from textwrap import dedent from source_updater import Source, SourceUpdateError class SourceTest(unittest.TestCase): def test_from_path_constructor_with_existing_file(self): tf = tempfile.NamedTemporaryFile(delete=False) self.addCleanup(tf.close) tf.write('stuff'.encode('utf8')) tf.flush() s = Source.from_path(tf.name) self.assertIsInstance(s, Source) self.assertEqual(s.path, tf.name) self.assertEqual(s.contents, 'stuff') def test_from_path_constructor_with_nonexistent_file(self): s = Source.from_path('no.such.path') self.assertIsInstance(s, Source) self.assertEqual(s.path, 'no.such.path') self.assertEqual(s.contents, '') def test_lines(self): s = Source() s.contents = 'abc\ndef' self.assertEqual(s.lines, ['abc', 'def']) def test_write_writes_new_content_to_path(self): s = Source() tf = tempfile.NamedTemporaryFile() s.get_updated_contents = lambda: 'abc\ndef' s.path = tf.name s.write() with open(tf.name) as f: self.assertEqual(f.read(), s.get_updated_contents()) class FunctionFinderTest(unittest.TestCase): def test_function_object(self): s = Source._from_contents(dedent( """ def a_function(stuff, args): pass """ )) f = s.functions['a_function'] self.assertEqual(f.name, 'a_function') self.assertEqual(f.full_line, 'def a_function(stuff, args):') def test_finds_functions(self): s = Source._from_contents(dedent( """ def firstfn(stuff, args): pass # stuff def second_fn(): pass """ )) assert list(s.functions) == ['firstfn', 'second_fn'] def test_finds_views(self): s = Source._from_contents(dedent( """ def firstfn(stuff, args): pass # stuff def a_view(request): pass def second_fn(): pass def another_view(request, stuff): pass """ )) assert list(s.functions) == ['firstfn', 'a_view', 'second_fn', 'another_view'] assert list(s.views) == ['a_view', 'another_view'] def test_finds_classes(self): s = Source._from_contents(dedent( """ import thingy class Jimbob(object): pass # stuff class Harlequin(thingy.Thing): pass """ )) assert list(s.classes) == ['Jimbob', 'Harlequin'] class ReplaceFunctionTest(unittest.TestCase): def test_finding_last_line_in_function(self): source = Source._from_contents(dedent(""" def myfn(): a += 1 return b """).strip() ) assert source.functions['myfn'].last_line == 2 def test_finding_last_line_in_function_with_brackets(self): source = Source._from_contents(dedent(""" def myfn(): a += 1 return ( '2' ) """).strip() ) assert source.functions['myfn'].last_line == 4 def test_finding_last_line_in_function_with_brackets_before_another(self): source = Source._from_contents(dedent(""" def myfn(): a += 1 return ( '2' ) # bla def anotherfn(): pass """).strip() ) assert source.functions['myfn'].last_line == 4 def test_changing_the_end_of_a_method(self): source = Source._from_contents(dedent(""" class A(object): def method1(self, stuff): # do step 1 # do step 2 # do step 3 # do step 4 return ( 'something' ) def method2(self): # do stuff pass """).lstrip() ) new = dedent(""" def method1(self, stuff): # do step 1 # do step 2 # do step A return ( 'something else' ) """ ).strip() expected = dedent(""" class A(object): def method1(self, stuff): # do step 1 # do step 2 # do step A return ( 'something else' ) def method2(self): # do stuff pass """ ).lstrip() to_write = source.replace_function(new.split('\n')) assert to_write == expected assert source.get_updated_contents() == expected class RemoveFunctionTest(unittest.TestCase): def test_removing_a_function(self): source = Source._from_contents(dedent( """ def fn1(args): # do stuff pass def fn2(arg2, arg3): # do things return 2 def fn3(): # do nothing # really pass """).lstrip() ) expected = dedent( """ def fn1(args): # do stuff pass def fn3(): # do nothing # really pass """ ).lstrip() assert source.remove_function('fn2') == expected assert source.get_updated_contents() == expected class AddToClassTest(unittest.TestCase): def test_finding_class_info(self): source = Source._from_contents(dedent( """ import topline class ClassA(object): def metha(self): pass def metha2(self): pass class ClassB(object): def methb(self): pass """).lstrip() ) assert source.classes['ClassA'].start_line == 2 assert source.classes['ClassA'].last_line == 7 assert source.classes['ClassA'].source == dedent( """ class ClassA(object): def metha(self): pass def metha2(self): pass """).strip() assert source.classes['ClassB'].last_line == 11 assert source.classes['ClassB'].source == dedent( """ class ClassB(object): def methb(self): pass """).strip() def test_addding_to_class(self): source = Source._from_contents(dedent(""" import topline class A(object): def metha(self): pass class B(object): def methb(self): pass """).lstrip() ) source.add_to_class('A', dedent( """ def metha2(self): pass """).strip().split('\n') ) expected = dedent(""" import topline class A(object): def metha(self): pass def metha2(self): pass class B(object): def methb(self): pass """ ).lstrip() assert source.contents == expected def test_addding_to_class_fixes_indents_and_superfluous_lines(self): source = Source._from_contents(dedent(""" import topline class A(object): def metha(self): pass """).lstrip() ) source.add_to_class('A', [ "", " def metha2(self):", " pass", ]) expected = dedent(""" import topline class A(object): def metha(self): pass def metha2(self): pass """ ).lstrip() assert source.contents == expected class ImportsTest(unittest.TestCase): def test_finding_different_types_of_import(self): source = Source._from_contents(dedent( """ import trees from django.core.widgets import things, more_things import cars from datetime import datetime from django.monkeys import banana_eating from lists.views import Thing not_an_import = 'import things' def foo(): # normal code pass """ )) assert set(source.imports) == { "import trees", "from django.core.widgets import things, more_things", "import cars", "from datetime import datetime", "from django.monkeys import banana_eating", "from lists.views import Thing", } assert set(source.django_imports) == { "from django.core.widgets import things, more_things", "from django.monkeys import banana_eating", } assert set(source.project_imports) == { "from lists.views import Thing", } assert set(source.general_imports) == { "import trees", "from datetime import datetime", "import cars", } def test_find_first_nonimport_line(self): source = Source._from_contents(dedent( """ import trees from django.core.widgets import things, more_things from django.monkeys import banana_eating from lists.views import Thing not_an_import = 'bla' # the end """).lstrip() ) assert source.find_first_nonimport_line() == 5 def test_find_first_nonimport_line_raises_if_imports_in_a_mess(self): source = Source._from_contents(dedent( """ import trees def foo(): return 'monkeys' import monkeys """).lstrip() ) with self.assertRaises(SourceUpdateError): source.find_first_nonimport_line() def test_fixed_imports(self): source = Source._from_contents(dedent( """ import btopline import atopline """).lstrip() ) assert source.fixed_imports == dedent( """ import atopline import btopline """).lstrip() source = Source._from_contents(dedent( """ import atopline from django.monkeys import monkeys from django.chickens import chickens """).lstrip() ) assert source.fixed_imports == dedent( """ import atopline from django.chickens import chickens from django.monkeys import monkeys """).lstrip() source = Source._from_contents(dedent( """ from lists.views import thing import atopline """).lstrip() ) assert source.fixed_imports == dedent( """ import atopline from lists.views import thing """).lstrip() source = Source._from_contents(dedent( """ from lists.views import thing from django.db import models import atopline from django.aardvarks import Wilbur """).lstrip() ) assert source.fixed_imports == dedent( """ import atopline from django.aardvarks import Wilbur from django.db import models from lists.views import thing """).lstrip() def test_add_import(self): source = Source._from_contents(dedent( """ import atopline from django.monkeys import monkeys from django.chickens import chickens from lists.views import thing # some stuff class C(): def foo(): return 1 """).lstrip() ) source.add_imports([ "import btopline" ]) assert source.fixed_imports == dedent( """ import atopline import btopline from django.chickens import chickens from django.monkeys import monkeys from lists.views import thing """ ).lstrip() source.add_imports([ "from django.dickens import ChuzzleWit" ]) assert source.fixed_imports == dedent( """ import atopline import btopline from django.chickens import chickens from django.dickens import ChuzzleWit from django.monkeys import monkeys from lists.views import thing """ ).lstrip() def test_add_import_chooses_longer_lines(self): source = Source._from_contents(dedent( """ import atopline from django.chickens import chickens from lists.views import thing # some stuff """).lstrip() ) source.add_imports([ "from django.chickens import chickens, eggs" ]) assert source.fixed_imports == dedent( """ import atopline from django.chickens import chickens, eggs from lists.views import thing """ ).lstrip() def test_add_import_ends_up_in_updated_contents_when_appending(self): source = Source._from_contents(dedent( """ import atopline # some stuff class C(): def foo(): return 1 """).lstrip() ) source.add_imports([ "from django.db import models" ]) assert source.contents == dedent( """ import atopline from django.db import models # some stuff class C(): def foo(): return 1 """ ).lstrip() def test_add_import_ends_up_in_updated_contents_when_prepending(self): source = Source._from_contents(dedent( """ import btopline # some stuff class C(): def foo(): return 1 """).lstrip() ) source.add_imports([ "import atopline" ]) assert source.contents == dedent( """ import atopline import btopline # some stuff class C(): def foo(): return 1 """ ).lstrip() class LineFindingTests(unittest.TestCase): def test_finding_start_line(self): source = Source._from_contents(dedent( """ stuff things bla bla bla indented more then end """).lstrip() ) assert source.find_start_line(['stuff', 'whatever']) == 0 assert source.find_start_line(['bla bla', 'whatever']) == 3 assert source.find_start_line(['indented', 'whatever']) == 4 assert source.find_start_line([' indented', 'whatever']) == 4 assert source.find_start_line(['no such line', 'whatever']) == None with self.assertRaises(SourceUpdateError): source.find_start_line(['']) with self.assertRaises(SourceUpdateError): source.find_start_line([]) def test_finding_end_line(self): source = Source._from_contents(dedent( """ stuff things bla bla bla indented more then end """).lstrip() ) assert source.find_end_line(['stuff', 'things']) == 1 assert source.find_end_line(['bla bla', 'whatever', 'more']) == 5 assert source.find_end_line(['bla bla', 'whatever']) == None assert source.find_end_line(['no such line', 'whatever']) == None with self.assertRaises(SourceUpdateError): source.find_end_line([]) with self.assertRaises(SourceUpdateError): source.find_end_line(['whatever','']) def test_finding_end_line_depends_on_start(self): source = Source._from_contents(dedent( """ stuff things bla more stuff things bla then end """).lstrip() ) assert source.find_end_line(['more stuff', 'things', 'bla']) == 6 class SourceUpdateTest(unittest.TestCase): def test_update_with_empty_contents(self): s = Source() s.update('new stuff\n') self.assertEqual(s.get_updated_contents(), 'new stuff\n') def test_adds_final_newline_if_necessary(self): s = Source() s.update('new stuff') self.assertEqual(s.get_updated_contents(), 'new stuff\n') if __name__ == '__main__': unittest.main() ================================================ FILE: tests/test_sourcetree.py ================================================ import os import subprocess import unittest from textwrap import dedent import pytest from book_parser import CodeListing from sourcetree import ( ApplyCommitException, Commit, SourceTree, check_chunks_against_future_contents, get_offset, strip_comments, ) class GetFileTest(unittest.TestCase): def test_get_contents(self): sourcetree = SourceTree() (sourcetree.tempdir / "foo.txt").write_text("bla bla") assert sourcetree.get_contents("foo.txt") == "bla bla" class StripCommentTest(unittest.TestCase): def test_strips_python_comments(self): assert strip_comments("foo #") == "foo" assert strip_comments("foo #") == "foo" def test_strips_js_comments(self): assert strip_comments("foo //") == "foo" assert strip_comments("foo //") == "foo" def test_doesnt_break_comment_lines(self): assert strip_comments("# normal comment") == "# normal comment" def test_doesnt_break_trailing_slashes(self): assert strip_comments("a_url/") == "a_url/" class StartWithCheckoutTest(unittest.TestCase): def test_get_local_repo_path(self): sourcetree = SourceTree() assert sourcetree.get_local_repo_path("chapter_name") == os.path.abspath( os.path.join(os.path.dirname(__file__), "../source/chapter_name/superlists") ) def test_checks_out_repo_chapter_as_main(self): sourcetree = SourceTree() sourcetree.get_local_repo_path = lambda c: os.path.abspath( os.path.join(os.path.dirname(__file__), "testrepo") ) sourcetree.start_with_checkout("chapter_17", "chapter_16") remotes = sourcetree.run_command("git remote").split() assert remotes == ["repo"] branch = sourcetree.run_command("git branch").strip() assert branch == "* main" diff = sourcetree.run_command("git diff repo/chapter_16").strip() assert diff == "" class ApplyFromGitRefTest(unittest.TestCase): def setUp(self): self.sourcetree = SourceTree() self.sourcetree.get_local_repo_path = lambda c: os.path.abspath( os.path.join(os.path.dirname(__file__), "testrepo") ) self.sourcetree.start_with_checkout("chapter_17", "chapter_16") self.sourcetree.run_command("git switch test-start") self.sourcetree.run_command("git reset") def test_from_real_git_stuff(self): listing = CodeListing( filename="file1.txt", contents=dedent( """ file 1 line 2 amended file 1 line 3 """ ).lstrip(), ) listing.commit_ref = "ch17l021" self.sourcetree.apply_listing_from_commit(listing) assert (self.sourcetree.tempdir / "file1.txt").read_text() == dedent( """ file 1 line 1 file 1 line 2 amended file 1 line 3 """ ).lstrip() assert listing.was_written def test_leaves_staging_empty(self): listing = CodeListing( filename="file1.txt", contents=dedent( """ file 1 line 2 amended file 1 line 3 """ ).lstrip(), ) listing.commit_ref = "ch17l021" self.sourcetree.apply_listing_from_commit(listing) staged = self.sourcetree.run_command("git diff --staged") assert staged == "" def test_raises_if_wrong_file(self): listing = CodeListing( filename="file2.txt", contents=dedent( """ file 1 line 1 file 1 line 2 amended file 1 line 3 """ ).lstrip(), ) listing.commit_ref = "ch17l021" with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) def _checkout_commit(self, commit): commit_spec = self.sourcetree.get_commit_spec(commit) self.sourcetree.run_command("git checkout " + commit_spec) self.sourcetree.run_command("git reset") def test_raises_if_too_many_files_in_commit(self): listing = CodeListing( filename="file1.txt", contents=dedent( """ file 1 line 1 file 1 line 2 """ ).lstrip(), ) listing.commit_ref = "ch17l023" self._checkout_commit("ch17l022") with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) def test_raises_if_listing_doesnt_show_all_new_lines_in_diff(self): listing = CodeListing( filename="file1.txt", contents=dedent( """ file 1 line 3 """ ).lstrip(), ) listing.commit_ref = "ch17l021" with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) def test_raises_if_listing_lines_in_wrong_order(self): listing = CodeListing( filename="file1.txt", contents=dedent( """ file 1 line 3 file 1 line 2 amended """ ).lstrip(), ) listing.commit_ref = "ch17l021" with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) def test_line_ordering_check_isnt_confused_by_dupe_lines(self): listing = CodeListing( filename="file2.txt", contents=dedent( """ another line changed some duplicate lines coming up... hello goodbye hello """ ).lstrip(), ) listing.commit_ref = "ch17l027" self._checkout_commit("ch17l026") self.sourcetree.apply_listing_from_commit(listing) def test_line_ordering_check_isnt_confused_by_new_lines_that_dupe_existing(self): listing = CodeListing( filename="file2.txt", contents=dedent( """ some duplicate lines coming up... hello one more line at end add a line with a dupe of existing hello goodbye """ ).lstrip(), ) listing.commit_ref = "ch17l031" self._checkout_commit("ch17l030") self.sourcetree.apply_listing_from_commit(listing) def DONTtest_non_dupes_are_still_order_checked(self): # TODO: get this working listing = CodeListing( filename="file2.txt", contents=dedent( """ some duplicate lines coming up... hello one more line at end add a line with a dupe of existing goodbye hello """ ).lstrip(), ) listing.commit_ref = "ch17l031" self._checkout_commit("ch17l030") with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) def test_raises_if_any_other_listing_lines_not_in_before_version(self): listing = CodeListing( filename="file1.txt", contents=dedent( """ what is this? file 1 line 2 amended file 1 line 3 """ ).lstrip(), ) listing.commit_ref = "ch17l021" with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) def test_happy_with_lines_in_before_and_after_version(self): listing = CodeListing( filename="file2.txt", contents=dedent( """ file 2 line 1 changed [...] hello hello one more line at end """ ).lstrip(), ) listing.commit_ref = "ch17l028" self._checkout_commit("ch17l027") self.sourcetree.apply_listing_from_commit(listing) def test_raises_if_listing_line_not_in_after_version(self): listing = CodeListing( filename="file2.txt", contents=dedent( """ hello goodbye hello one more line at end """ ).lstrip(), ) listing.commit_ref = "ch17l028" with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) def test_happy_with_lines_from_just_before_diff(self): listing = CodeListing( filename="file1.txt", contents=dedent( """ file 1 line 1 file 1 line 2 amended file 1 line 3 """ ).lstrip(), ) listing.commit_ref = "ch17l021" self.sourcetree.apply_listing_from_commit(listing) def test_listings_showing_a_move_mean_can_ignore_commit_lines_added_and_removed( self, ): listing = CodeListing( filename="pythonfile.py", contents=dedent( """ class NuKlass(object): def method1(self): [...] a = a + 3 [...] """ ).lstrip(), ) listing.commit_ref = "ch17l029" self._checkout_commit("ch17l028-1") self.sourcetree.apply_listing_from_commit(listing) def test_listings_showing_a_move_mean_can_ignore_commit_lines_added_and_removed_2( self, ): listing = CodeListing( filename="file2.txt", contents=dedent( """ hello one more line at end """ ).lstrip(), ) listing.commit_ref = "ch17l030" self._checkout_commit("ch17l029") self.sourcetree.apply_listing_from_commit(listing) def test_happy_with_elipsis(self): listing = CodeListing( filename="file1.txt", contents=dedent( """ [...] file 1 line 2 amended file 1 line 3 """ ).lstrip(), ) listing.commit_ref = "ch17l021" self.sourcetree.apply_listing_from_commit(listing) def DONTtest_listings_must_use_elipsis_to_indicate_skipped_lines(self): # TODO! lines = [ "file 1 line 1", "file 1 line 2 amended", "file 1 line 3", "file 1 line 4 inserted", "another line", ] listing = CodeListing(filename="file1.txt", contents="") listing.commit_ref = "ch17l022" self._checkout_commit("ch17l021") listing.contents = "\n".join(lines) self.sourcetree.apply_listing_from_commit(listing) # should not raise lines[1] = "[...]" listing.contents = "\n".join(lines) self.sourcetree.apply_listing_from_commit(listing) # should not raise lines.pop(1) listing.contents = "\n".join(lines) with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) def test_happy_with_python_callouts(self): listing = CodeListing( filename="file1.txt", contents=dedent( """ [...] file 1 line 2 amended # file 1 line 3 # """ ).lstrip(), ) listing.commit_ref = "ch17l021" self.sourcetree.apply_listing_from_commit(listing) def test_happy_with_js_callouts(self): listing = CodeListing( filename="file1.txt", contents=dedent( """ [...] file 1 line 2 amended // file 1 line 3 // """ ).lstrip(), ) listing.commit_ref = "ch17l021" self.sourcetree.apply_listing_from_commit(listing) def test_happy_with_blank_lines(self): listing = CodeListing( filename="file2.txt", contents=dedent( """ file 2 line 1 changed another line changed """ ).lstrip(), ) listing.commit_ref = "ch17l024" self._checkout_commit("ch17l023") self.sourcetree.apply_listing_from_commit(listing) def test_handles_indents(self): listing = CodeListing( filename="pythonfile.py", contents=dedent( """ def method1(self): # amend method 1 return 2 [...] """ ).lstrip(), ) listing.commit_ref = "ch17l026" self._checkout_commit("ch17l025") self.sourcetree.apply_listing_from_commit(listing) def test_over_indentation_differences_are_picked_up(self): listing = CodeListing( filename="pythonfile.py", contents=dedent( """ def method1(self): # amend method 1 return 2 [...] """ ).lstrip(), ) listing.commit_ref = "ch17l026" self._checkout_commit("ch17l025") with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) def test_under_indentation_differences_are_picked_up(self): listing = CodeListing( filename="pythonfile.py", contents=dedent( """ def method1(self): # amend method 1 return 2 [...] """ ).lstrip(), ) listing.commit_ref = "ch17l026" self._checkout_commit("ch17l025") with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) def test_with_diff_listing_passing_case(self): listing = CodeListing( filename="file2.txt", contents=dedent( """ diff --git a/file2.txt b/file2.txt index 93f054e..519d518 100644 --- a/file2.txt +++ b/file2.txt @@ -4,6 +4,5 @@ another line changed some duplicate lines coming up... hello -hello one more line at end """ ).lstrip(), ) listing.commit_ref = "ch17l030" self._checkout_commit("ch17l029") self.sourcetree.apply_listing_from_commit(listing) def test_with_diff_listing_failure_case(self): listing = CodeListing( filename="file2.txt", contents=dedent( """ diff --git a/file2.txt b/file2.txt index 93f054e..519d518 100644 --- a/file2.txt +++ b/file2.txt @@ -4,6 +4,5 @@ another line changed some duplicate lines coming up... hello -hello +something else one more line at end """ ).lstrip(), ) listing.commit_ref = "ch17l030" self._checkout_commit("ch17l029") with self.assertRaises(ApplyCommitException): self.sourcetree.apply_listing_from_commit(listing) class SourceTreeRunCommandTest(unittest.TestCase): def test_running_simple_command(self): sourcetree = SourceTree() sourcetree.run_command("touch foo", cwd=sourcetree.tempdir) assert os.path.exists(os.path.join(sourcetree.tempdir, "foo")) def test_default_directory_is_tempdir(self): sourcetree = SourceTree() sourcetree.run_command("touch foo") assert os.path.exists(os.path.join(sourcetree.tempdir, "foo")) def test_returns_output(self): sourcetree = SourceTree() output = sourcetree.run_command("echo hello", cwd=sourcetree.tempdir) assert output == "hello\n" def test_raises_on_errors(self): sourcetree = SourceTree() with self.assertRaises(Exception): sourcetree.run_command("synt!tax error", cwd=sourcetree.tempdir) sourcetree.run_command( "synt!tax error", cwd=sourcetree.tempdir, ignore_errors=True ) def test_environment_variables(self): sourcetree = SourceTree() os.environ["TEHFOO"] = "baz" output = sourcetree.run_command("echo $TEHFOO", cwd=sourcetree.tempdir) assert output.strip() == "baz" def test_doesnt_raise_for_some_things_where_a_return_code_is_ok(self): sourcetree = SourceTree() sourcetree.run_command("diff foo bar", cwd=sourcetree.tempdir) sourcetree.run_command("python test.py", cwd=sourcetree.tempdir) @pytest.mark.xfail( reason="disabled to allow passwordless sudo, see preexec fn in sourcetree.py" ) def test_cleanup_kills_backgrounded_processes_and_rmdirs(self): sourcetree = SourceTree() sourcetree.run_command( 'python -c"import time; time.sleep(5)" & #runserver', cwd=sourcetree.tempdir ) assert len(sourcetree.processes) == 1 sourcetree_pid = sourcetree.processes[0].pid pids = ( subprocess.check_output("pgrep -f time.sleep", shell=True) .decode("utf8") .split() ) print("sourcetree_pid", sourcetree_pid) print("pids", pids) sids = [] for pid in reversed(pids): print("checking", pid) cmd = "ps -o sid --no-header -p %s" % (pid,) print(cmd) try: sid = subprocess.check_output(cmd, shell=True) print("sid", sid) sids.append(sid) assert sourcetree_pid == int(sid) except subprocess.CalledProcessError: pass assert sids sourcetree.cleanup() assert "time.sleep" not in subprocess.check_output("ps aux", shell=True).decode( "utf8" ) # assert not os.path.exists(sourcetree.tempdir) def test_running_interactive_command(self): sourcetree = SourceTree() command = "python3 -c \"print('input please?'); a = input();print('OK' if a=='yes' else 'NO')\"" output = sourcetree.run_command(command, user_input="no") assert "NO" in output output = sourcetree.run_command(command, user_input="yes") assert "OK" in output class CommitTest(unittest.TestCase): def test_init_from_example(self): example = dedent( """ commit 9ecbb2c2222b9b31ab21e51e42ed8179ec79b273 Author: Harry Date: Thu Aug 22 20:26:09 2013 +0100 Some comment text. --ch09l021-- Conflicts: lists/tests/test_views.py diff --git a/lists/tests/test_views.py b/lists/tests/test_views.py index 8e18d77..03fc675 100644 --- a/lists/tests/test_views.py +++ b/lists/tests/test_views.py @@ -55,36 +55,6 @@ class NewListTest(TestCase): -class NewItemTest(TestCase): - - def test_can_save_a_POST_request_to_an_existing_list(self): - other_list = List.objects.create() - correct_list = List.objects.create() - self.assertEqual(new_item.list, correct_list) - - - def test_redirects_to_list_view(self): - other_list = List.objects.create() - correct_list = List.objects.create() - self.assertRedirects(response, '/lists/%d/' % (correct_list.id,)) - - - class ListViewTest(TestCase): def test_list_view_passes_list_to_list_template(self): @@ -112,3 +82,29 @@ class ListViewTest(TestCase): self.assertNotContains(response, 'other list item 1') self.assertNotContains(response, 'other list item 2') + + def test_can_save_a_POST_request_to_an_existing_list(self): + other_list = List.objects.create() + correct_list = List.objects.create() + self.assertEqual(new_item.list, correct_list) + + + def test_POST_redirects_to_list_view(self): + other_list = List.objects.create() + correct_list = List.objects.create() + self.assertRedirects(response, '/lists/%d/' % (correct_list.id,)) + """ ) commit = Commit.from_diff(example) assert commit.info == example assert commit.lines_to_add == [ " def test_can_save_a_POST_request_to_an_existing_list(self):", " other_list = List.objects.create()", " correct_list = List.objects.create()", " self.assertEqual(new_item.list, correct_list)", " def test_POST_redirects_to_list_view(self):", " other_list = List.objects.create()", " correct_list = List.objects.create()", " self.assertRedirects(response, '/lists/%d/' % (correct_list.id,))", ] assert commit.lines_to_remove == [ "class NewItemTest(TestCase):", " def test_can_save_a_POST_request_to_an_existing_list(self):", " other_list = List.objects.create()", " correct_list = List.objects.create()", " self.assertEqual(new_item.list, correct_list)", " def test_redirects_to_list_view(self):", " other_list = List.objects.create()", " correct_list = List.objects.create()", " self.assertRedirects(response, '/lists/%d/' % (correct_list.id,))", ] assert commit.moved_lines == [ " def test_can_save_a_POST_request_to_an_existing_list(self):", " other_list = List.objects.create()", " correct_list = List.objects.create()", " self.assertEqual(new_item.list, correct_list)", " other_list = List.objects.create()", " correct_list = List.objects.create()", " self.assertRedirects(response, '/lists/%d/' % (correct_list.id,))", ] assert commit.deleted_lines == [ "class NewItemTest(TestCase):", " def test_redirects_to_list_view(self):", ] assert commit.new_lines == [ " def test_POST_redirects_to_list_view(self):", ] class CheckChunksTest(unittest.TestCase): def test_get_offset_when_none(self): lines = [ "def method1(self):", " return 2", ] future_lines = [ "def method1(self):", " return 2", ] assert get_offset(lines, future_lines) == "" def test_get_offset_when_some(self): lines = [ "", "def method1(self):", " return 2", ] future_lines = [ " ", " def method1(self):", " return 2", ] assert get_offset(lines, future_lines) == " " def test_passing_case(self): code = dedent( """ def method1(self): # amend method 1 return 2 """ ).strip() future_contents = dedent( """ class Thing: def method1(self): # amend method 1 return 2 """ ).strip() check_chunks_against_future_contents(code, future_contents) # should not raise def test_over_indentation_differences_are_picked_up(self): code = dedent( """ def method1(self): # amend method 1 return 2 """ ).strip() future_contents = dedent( """ class Thing: def method1(self): # amend method 1 return 2 """ ).strip() with self.assertRaises(ApplyCommitException): check_chunks_against_future_contents(code, future_contents) def test_under_indentation_differences_are_picked_up(self): code = dedent( """ def method1(self): # amend method 1 return 2 """ ).strip() future_contents = dedent( """ class Thing: def method1(self): # amend method 1 return 2 """ ).strip() with self.assertRaises(ApplyCommitException): check_chunks_against_future_contents(code, future_contents) def test_leading_blank_lines_in_listing_are_ignored(self): code = dedent( """ def method1(self): # amend method 1 return 2 """ ) future_contents = dedent( """ class Thing: def method1(self): # amend method 1 return 2 """ ).strip() check_chunks_against_future_contents(code, future_contents) # should not raise def test_thing(self): code = dedent( """ [...] item.save() return render( request, "home.html", {"new_item_text": item.text}, ) """ ) future_contents = dedent( """ from django.shortcuts import render from lists.models import Item def home_page(request): item = Item() item.text = request.POST.get("item_text", "") item.save() return render( request, "home.html", {"new_item_text": item.text}, ) """ ).strip() check_chunks_against_future_contents(code, future_contents) # should not raise def test_trailing_blank_lines_in_listing_are_ignored(self): code = dedent( """ def method1(self): # amend method 1 return 2 """ ).lstrip() future_contents = dedent( """ class Thing: def method1(self): # amend method 1 return 2 """ ).strip() check_chunks_against_future_contents(code, future_contents) # should not raise def test_elipsis_lines_are_ignored(self): lines = dedent( """ def method1(self): # amend method 1 return 2 [...] """ ).strip() future_lines = dedent( """ def method1(self): # amend method 1 return 2 stuff """ ).rstrip() check_chunks_against_future_contents(lines, future_lines) # should not raise ================================================ FILE: tests/test_write_to_file.py ================================================ #!/usr/bin/env python3 import unittest import os import shutil from textwrap import dedent import tempfile from book_tester import CodeListing from write_to_file import ( _find_last_line_for_class, number_of_identical_chars, write_to_file, ) class ClassFinderTest(unittest.TestCase): def test_find_last_line_for_class(self): source = dedent( """ import topline class ClassA(object): def metha(self): pass def metha2(self): pass class ClassB(object): def methb(self): pass """ ) lineno = _find_last_line_for_class(source, 'ClassA') self.assertEqual(lineno, 9) # sanity-check self.assertEqual(source.split('\n')[lineno - 1].strip(), 'pass') lineno = _find_last_line_for_class(source, 'ClassB') self.assertEqual(lineno, 13) class LineFinderTest(unittest.TestCase): def test_number_of_identical_chars(self): self.assertEqual( number_of_identical_chars('1234', '5678'), 0 ) self.assertEqual( number_of_identical_chars('1234', '1235'), 3 ) self.assertEqual( number_of_identical_chars('1234', '1243'), 2 ) self.assertEqual( number_of_identical_chars('12345', '123WHATEVER45'), 5 ) class WriteToFileTest(unittest.TestCase): maxDiff = None def setUp(self): self.tempdir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.tempdir) def test_simple_case(self): #done listing = CodeListing(filename='foo.py', contents='abc\ndef') write_to_file(listing, self.tempdir) with open(os.path.join(self.tempdir, listing.filename)) as f: self.assertEqual(f.read(), listing.contents + '\n') self.assertTrue(listing.was_written) def test_multiple_files(self): listing = CodeListing(filename='foo.py, bar.py', contents='abc\ndef') write_to_file(listing, self.tempdir) with open(os.path.join(self.tempdir, 'foo.py')) as f: self.assertEqual(f.read(), listing.contents + '\n') with open(os.path.join(self.tempdir, 'bar.py')) as f: self.assertEqual(f.read(), listing.contents + '\n') self.assertTrue(listing.was_written) def assert_write_to_file_gives( self, old_contents, new_contents, expected_contents ): listing = CodeListing(filename='foo.py', contents=new_contents) with open(os.path.join(self.tempdir, 'foo.py'), 'w') as f: f.write(old_contents) write_to_file(listing, self.tempdir) with open(os.path.join(self.tempdir, listing.filename)) as f: actual = f.read() self.assertMultiLineEqual(actual, expected_contents) def test_strips_python_line_callouts_one_space(self): contents = 'hello\nbla #\nstuff' self.assert_write_to_file_gives('', contents, 'hello\nbla\nstuff\n') def test_strips_python_line_callouts_two_spaces(self): contents = 'hello\nbla #\nstuff' self.assert_write_to_file_gives('', contents, 'hello\nbla\nstuff\n') def test_strips_js_line_callouts(self): contents = 'hello\nbla //\nstuff' self.assert_write_to_file_gives('', contents, 'hello\nbla\nstuff\n') contents = 'hello\nbla //' self.assert_write_to_file_gives('', contents, 'hello\nbla\n') def test_doesnt_mess_with_multiple_newlines(self): contents = 'hello\n\n\nbla' self.assert_write_to_file_gives('', contents, 'hello\n\n\nbla\n') def test_existing_file_bears_no_relation_means_replaced(self): old = '#abc\n#def\n#ghi\n#jkl\n' new = '#mno\n#pqr\n#stu\n#vvv\n' expected = new self.assert_write_to_file_gives(old, new, expected) def test_existing_file_has_views_means_apppend(self): old = dedent( """ from django.stuff import things def a_view(request, param): pass """ ).lstrip() new = dedent( """ def another_view(request): pass """ ).strip() expected = dedent( """ from django.stuff import things def a_view(request, param): pass def another_view(request): pass """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_existing_file_has_single_class_means_replace(self): old = dedent( """ class Jimmy(object): pass """).lstrip() new = dedent( """ class Carruthers(object): pass """).strip() expected = dedent( """ class Carruthers(object): pass """).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_existing_file_has_multiple_classes_means_append(self): old = dedent( """ import apples import pears class Jimmy(object): pass class Bob(object): pass """ ).lstrip() new = dedent( """ class Carruthers(object): pass """ ).strip() expected = dedent( """ import apples import pears class Jimmy(object): pass class Bob(object): pass class Carruthers(object): pass """).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_leading_elipsis_is_ignored(self): old = dedent( """ class C(): def foo(): # bla 1 # bla 2 # bla 3 # bla 4 return 1 # the end """ ).lstrip() new = dedent( """ [...] # bla 2 # bla 3b # bla 4b return 1 """ ) expected = dedent( """ class C(): def foo(): # bla 1 # bla 2 # bla 3b # bla 4b return 1 # the end """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_adding_import_at_top_then_elipsis_then_modified_stuff(self): old = dedent( """ import topline # some stuff class C(): def foo(): return 1 """ ) new = dedent( """ import newtopline [...] def foo(): return 2 """ ) expected = dedent( """ import newtopline import topline # some stuff class C(): def foo(): return 2 """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def DONTtest_adding_import_at_top_without_elipsis_then_modified_stuff(self): old = dedent( """ import anoldthing import bthing import cthing class C(cthing.Bar): def foo(): return 1 # more stuff... """ ) new = dedent( """ import anewthing import bthing import cthing class C(anewthing.Baz): def foo(): [...] """ ) expected = dedent( """ import anewthing import bthing import cthing class C(anewthing.Baz): def foo(): return 1 # more stuff... """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_adding_import_at_top_then_elipsis_then_totally_new_stuff(self): old = dedent( """ import topline # some stuff class C(): pass """ ).lstrip() new = dedent( """ import newtopline [...] class Nu(): pass """ ) expected = dedent( """ import newtopline import topline # some stuff class C(): pass class Nu(): pass """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_elipsis_indicating_which_class_to_add_new_method_to(self): old = dedent( """ import topline class A(object): def metha(self): pass class B(object): def methb(self): pass """ ).lstrip() new = dedent( """ class A(object): [...] def metha2(self): pass """ ) expected = dedent( """ import topline class A(object): def metha(self): pass def metha2(self): pass class B(object): def methb(self): pass """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_adding_import_at_top_sorts_alphabetically_respecting_django_and_locals(self): old = dedent( """ import atopline from django.monkeys import monkeys from django.chickens import chickens from lists.views import thing # some stuff class C(): def foo(): return 1 """) new = dedent( """ import btopline [...] def foo(): return 2 """ ) expected = dedent( """ import atopline import btopline from django.chickens import chickens from django.monkeys import monkeys from lists.views import thing # some stuff class C(): def foo(): return 2 """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) new = dedent( """ from django.dickens import dickens [...] def foo(): return 2 """ ) expected = dedent( """ import atopline from django.chickens import chickens from django.dickens import dickens from django.monkeys import monkeys from lists.views import thing # some stuff class C(): def foo(): return 2 """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) new = dedent( """ from lists.zoos import thing [...] def foo(): return 2 """ ) expected = dedent( """ import atopline from django.chickens import chickens from django.monkeys import monkeys from lists.views import thing from lists.zoos import thing # some stuff class C(): def foo(): return 2 """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_with_new_contents_then_indented_elipsis_then_appendix(self): old = '#abc\n#def\n#ghi\n#jkl\n' new = ( '#abc\n' 'def foo(v):\n' ' return v + 1\n' ' #def\n' ' [... old stuff as before]\n' '# then add this' ) expected = ( '#abc\n' 'def foo(v):\n' ' return v + 1\n' ' #def\n' ' #ghi\n' ' #jkl\n' '\n' '# then add this\n' ) self.assert_write_to_file_gives(old, new, expected) def test_for_existing_file_replaces_matching_lines(self): old = dedent( """ class Foo(object): def method_1(self): return 1 def method_2(self): # two return 2 """ ).lstrip() new = dedent( """ def method_2(self): # two return 'two' """ ).strip() expected = dedent( """ class Foo(object): def method_1(self): return 1 def method_2(self): # two return 'two' """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_for_existing_file_doesnt_swallow_whitespace(self): old = dedent( """ one = ( 1, ) two = ( 2, ) three = ( 3, ) """).lstrip() new = dedent( """ two = ( 2, #two ) """ ).strip() expected = dedent( """ one = ( 1, ) two = ( 2, #two ) three = ( 3, ) """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_longer_new_file_starts_replacing_from_first_different_line(self): old = dedent( """ # line 1 # line 2 # line 3 """ ).lstrip() new = dedent( """ # line 1 # line 2 # line 3 # line 4 """ ).strip() expected = dedent( """ # line 1 # line 2 # line 3 # line 4 """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_changing_the_end_of_a_method(self): old = dedent( """ class A(object): def method1(self): # do step 1 # do step 2 # do step 3 # do step 4 # do step 5 pass def method2(self): # do stuff pass """ ).lstrip() new = dedent( """ def method1(self): # do step 1 # do step 2 # do step A # do step B """ ).strip() expected = dedent( """ class A(object): def method1(self): # do step 1 # do step 2 # do step A # do step B def method2(self): # do stuff pass """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_for_existing_file_inserting_new_lines_between_comments(self): old = dedent( """ # test 1 a = foo() assert a == 1 if a: # test 2 self.fail('finish me') # test 3 # the end # is here """).lstrip() new = dedent( """ # test 2 b = bar() assert b == 2 # test 3 assert True self.fail('finish me') # the end [...] """ ).lstrip() expected = dedent( """ # test 1 a = foo() assert a == 1 if a: # test 2 b = bar() assert b == 2 # test 3 assert True self.fail('finish me') # the end # is here """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_with_single_line_replacement(self): old = dedent( """ def wiggle(): abc def abcd fghi jkl mno """ ).lstrip() new = dedent( """ abcd abcd """ ).strip() expected = dedent( """ def wiggle(): abc def abcd abcd jkl mno """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_with_single_line_replacement_finds_most_probable_line(self): old = dedent( """ abc abc daf ghi abc dex xyz jkl mno """ ).lstrip() new = dedent( """ abc deFFF ghi """ ).strip() expected = dedent( """ abc abc deFFF ghi abc dex xyz jkl mno """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_with_single_line_assertion_replacement(self): old = dedent( """ class Wibble(unittest.TestCase): def test_number_1(self): self.assertEqual(1 + 1, 2) """ ).lstrip() new = dedent( """ self.assertEqual(1 + 1, 3) """ ).strip() expected = dedent( """ class Wibble(unittest.TestCase): def test_number_1(self): self.assertEqual(1 + 1, 3) """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_with_single_line_assertion_replacement_finds_right_one(self): old = dedent( """ class Wibble(unittest.TestCase): def test_number_1(self): self.assertEqual(1 + 1, 2) def test_number_2(self): self.assertEqual(1 + 2, 3) """ ).lstrip() new = dedent( """ self.assertEqual(1 + 2, 4) """ ).strip() expected = dedent( """ class Wibble(unittest.TestCase): def test_number_1(self): self.assertEqual(1 + 1, 2) def test_number_2(self): self.assertEqual(1 + 2, 4) """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_with_single_line_assertion_replacement_real_views_example(self): old = dedent( """ from lists.models import Item, List def home_page(request): return render(request, 'home.html') def view_list(request, list_id): list = List.objects.get(id=list_id) items = Item.objects.filter(list=list) return render(request, 'list.html', {'items': items}) def new_list(request): list = List.objects.create() Item.objects.create(text=request.POST['item_text'], list=list) return redirect('/lists/%d/' % (list.id,)) def add_item(request): pass """ ).lstrip() new = dedent( """ def add_item(request, list_id): pass """ ) expected = dedent( """ from lists.models import Item, List def home_page(request): return render(request, 'home.html') def view_list(request, list_id): list = List.objects.get(id=list_id) items = Item.objects.filter(list=list) return render(request, 'list.html', {'items': items}) def new_list(request): list = List.objects.create() Item.objects.create(text=request.POST['item_text'], list=list) return redirect('/lists/%d/' % (list.id,)) def add_item(request, list_id): pass """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_changing_function_signature_and_stripping_comment(self): old = dedent( """ # stuff def foo(): pass """ ).lstrip() new = dedent( """ def foo(bar): pass """ ).strip() expected = new + '\n' self.assert_write_to_file_gives(old, new, expected) def test_with_two_elipsis_dedented_change(self): old = dedent( """ class Wibble(object): def foo(self): return 2 def bar(self): return 3 """).lstrip() new = dedent( """ [...] def foo(self): return 4 def bar(self): [...] """ ).strip() expected = dedent( """ class Wibble(object): def foo(self): return 4 def bar(self): return 3 """ ).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_indents_in_new_dont_confuse_things(self): old = dedent( """ class Wibble(): def foo(self): # comment 1 do something # comment 2 do something else and keep going """).lstrip() new = ( " # comment 2\n" " time.sleep(2)\n" " do something else\n" ) expected = dedent( """ class Wibble(): def foo(self): # comment 1 do something # comment 2 time.sleep(2) do something else and keep going """).lstrip() self.assert_write_to_file_gives(old, new, expected) def test_double_indents_in_new_dont_confuse_things(self): old = dedent( """ class Wibble(): def foo(self): if something: do something # end of class """).lstrip() new = dedent( """ if something: do something else # end of class """) expected = dedent( """ class Wibble(): def foo(self): if something: do something else # end of class """).lstrip() self.assert_write_to_file_gives(old, new, expected) if __name__ == '__main__': unittest.main() ================================================ FILE: tests/update_source_repo.py ================================================ #!/usr/bin/env python import getpass import os import subprocess from pathlib import Path from chapters import CHAPTERS REMOTE = "local" if "harry" in getpass.getuser() else "origin" BASE_FOLDER = Path(__file__).parent.parent def fetch_if_possible(target_dir: Path): fetch = subprocess.Popen( ["git", "fetch", REMOTE], cwd=target_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout, stderr = fetch.communicate() print(stdout.decode(), stderr.decode()) if fetch.returncode: if ( "Name or service not known" in stderr.decode() or "Could not resolve" in stderr.decode() or "github.com port 22: Undefined error" in stderr.decode() ): # no internet print("No Internet") return False raise Exception("Error running git fetch") return True def update_sources_for_chapter(chapter, previous_chapter=None): source_dir = BASE_FOLDER / "source" / chapter / "superlists" if not source_dir.exists(): print("No folder at", source_dir, "skipping...") return print("updating", source_dir) fetch_if_possible(source_dir) subprocess.check_output(["git", "submodule", "update", str(source_dir)]) commit_specified_by_submodule = ( subprocess.check_output(["git", "log", "-n 1", "--format=%H"], cwd=source_dir) .decode() .strip() ) if previous_chapter is not None: # make sure branch for previous chapter is available to start tests prev_chap_source_dir = BASE_FOLDER / "source" / previous_chapter / "superlists" subprocess.check_output( ["git", "checkout", str(previous_chapter)], cwd=source_dir ) # we use the submodule commit, # as specfified in the previous chapter source/x dir prev_chap_commit_specified_by_submodule = ( subprocess.check_output( ["git", "log", "-n 1", "--format=%H"], cwd=prev_chap_source_dir ) .decode() .strip() ) print( f"resetting {previous_chapter} branch to {prev_chap_commit_specified_by_submodule}" ) subprocess.check_output(["git", "checkout", previous_chapter], cwd=source_dir) subprocess.check_output( ["git", "reset", "--hard", prev_chap_commit_specified_by_submodule], cwd=source_dir, ) # check out current branch, local version, for final diff subprocess.check_output(["git", "checkout", chapter], cwd=source_dir) if os.environ.get("CI"): # if in CI, we use the submodule commit, to check that the submodule # config is up to date print(f"resetting {chapter} branch to {commit_specified_by_submodule}") subprocess.check_output( ["git", "reset", "--hard", commit_specified_by_submodule], cwd=source_dir ) def checkout_testrepo_branches(): testrepo_dir = BASE_FOLDER / "tests/testrepo" for branchname in ["chapter_16", "master", "chapter_20", "chapter_17"]: subprocess.check_output(["git", "checkout", str(branchname)], cwd=testrepo_dir) def main(): """ update submodule folders for all chapters, making sure previous and current branches are locally checked out """ if "SKIP_CHAPTER_SUBMODULES" not in os.environ: for chapter, previous_chapter in zip(CHAPTERS, [None, *CHAPTERS]): update_sources_for_chapter(chapter, previous_chapter=previous_chapter) checkout_testrepo_branches() if __name__ == "__main__": main() ================================================ FILE: tests/write_to_file.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import ast import os import re from textwrap import dedent from source_updater import ( VIEW_FINDER, get_indent, Source, ) def _replace_lines_from_to(old_lines, new_lines, start_pos, end_pos): print('replace lines from line', start_pos, 'to line', end_pos) old_indent = get_indent(old_lines[start_pos]) new_indent = get_indent(new_lines[0]) if new_indent: missing_indent = old_indent[:-len(new_indent)] else: missing_indent = old_indent indented_new_lines = [missing_indent + l for l in new_lines] return '\n'.join( old_lines[:start_pos] + indented_new_lines + old_lines[end_pos + 1:] ) def _get_function(source, function_name): functions = [ n for n in ast.walk(ast.parse(source)) if isinstance(n, ast.FunctionDef) ] try: return next(c for c in functions if c.name == function_name) except StopIteration: raise Exception('Could not find function named %s' % (function_name,)) def _replace_lines_from(old_lines, new_lines, start_pos): print('replace lines from line', start_pos) start_line_in_old = old_lines[start_pos] indent = get_indent(start_line_in_old) for ix, new_line in enumerate(new_lines): if len(old_lines) > start_pos + ix: old_lines[start_pos + ix] = indent + new_line else: old_lines.append(indent + new_line) return '\n'.join(old_lines) def _number_of_identical_chars_at_beginning(string1, string2): n = 0 for char1, char2 in zip(string1, string2): if char1 != char2: return n n += 1 return n def number_of_identical_chars(string1, string2): string1, string2 = string1.strip(), string2.strip() start_num = _number_of_identical_chars_at_beginning(string1, string2) end_num = _number_of_identical_chars_at_beginning( reversed(string1), reversed(string2) ) return min(len(string1), start_num + end_num) def _replace_single_line(old_lines, new_lines): print('replace single line') new_line = new_lines[0] line_finder = lambda l: number_of_identical_chars(l, new_line) likely_line = sorted(old_lines, key=line_finder)[-1] new_line = get_indent(likely_line) + new_line new_content = '\n'.join(old_lines).replace(likely_line, new_line) return new_content def _replace_lines_in(old_lines, new_lines): source = Source._from_contents('\n'.join(old_lines)) if new_lines[0].strip() == '': new_lines.pop(0) new_lines = dedent('\n'.join(new_lines)).split('\n') if len(new_lines) == 1: return _replace_single_line(old_lines, new_lines) start_pos = source.find_start_line(new_lines) if start_pos is None: print('no start line found') if 'import' in new_lines[0] and 'import' in old_lines[0]: new_contents = new_lines[0] + '\n' return new_contents + _replace_lines_in(old_lines[1:], new_lines[1:]) if VIEW_FINDER.match(new_lines[0]): if source.views: view_name = VIEW_FINDER.search(new_lines[0]).group(1) if view_name in source.views: return source.replace_function(new_lines) return '\n'.join(old_lines) + '\n\n' + '\n'.join(new_lines) class_finder = re.compile(r'^class \w+\(.+\):$', re.MULTILINE) if class_finder.match(new_lines[0]): print('found class in input') if len(source.classes) > 1: print('found classes') return '\n'.join(old_lines) + '\n\n\n' + '\n'.join(new_lines) return '\n'.join(new_lines) end_pos = source.find_end_line(new_lines) if end_pos is None: if new_lines[0].strip().startswith('def '): return source.replace_function(new_lines) else: #TODO: can we get rid of this? return _replace_lines_from(old_lines, new_lines, start_pos) else: return _replace_lines_from_to(old_lines, new_lines, start_pos, end_pos) def add_import_and_new_lines(new_lines, old_lines): source = Source._from_contents('\n'.join(old_lines)) print('add import and new lines') source.add_imports(new_lines[:1]) lines_with_import = source.get_updated_contents().split('\n') new_lines_remaining = '\n'.join(new_lines[2:]).strip('\n').split('\n') start_pos = source.find_start_line(new_lines_remaining) if start_pos is None: return '\n'.join(lines_with_import) + '\n\n\n' + '\n'.join(new_lines_remaining) else: return _replace_lines_in(lines_with_import, new_lines_remaining) def _find_last_line_for_class(source, classname): all_nodes = list(ast.walk(ast.parse(source))) classes = [n for n in all_nodes if isinstance(n, ast.ClassDef)] our_class = next(c for c in classes if c.name == classname) last_line_in_our_class = max( getattr(thing, 'lineno', 0) for thing in ast.walk(our_class) ) return last_line_in_our_class def add_to_class(new_lines, old_lines): print('adding to class') source = Source._from_contents('\n'.join(old_lines)) classname = re.search(r'class (\w+)\(\w+\):', new_lines[0]).group(1) source.add_to_class(classname, new_lines[2:]) return source.get_updated_contents() def write_to_file(codelisting, cwd): if ',' in codelisting.filename: files = codelisting.filename.split(', ') else: files = [codelisting.filename] new_contents = codelisting.contents for filename in files: path = os.path.join(cwd, filename) _write_to_file(path, new_contents) #with open(os.path.join(path)) as f: # print(f.read()) codelisting.was_written = True def _write_to_file(path, new_contents): source = Source.from_path(path) # strip callouts new_contents = re.sub(r' +#$', '', new_contents, flags=re.MULTILINE) new_contents = re.sub(r' +//$', '', new_contents, flags=re.MULTILINE) if not os.path.exists(path): dir = os.path.dirname(path) if not os.path.exists(dir): os.makedirs(dir) else: old_lines = source.lines new_lines = new_contents.strip('\n').split('\n') if "[..." not in new_contents: new_contents = _replace_lines_in(old_lines, new_lines) else: if new_contents.count("[...") == 1: split_line = [l for l in new_lines if "[..." in l][0] split_line_pos = new_lines.index(split_line) if split_line_pos == 0: new_contents = _replace_lines_in(old_lines, new_lines[1:]) elif split_line == new_lines[-1]: new_contents = _replace_lines_in(old_lines, new_lines[:-1]) elif split_line_pos == 1: if 'import' in new_lines[0]: new_contents = add_import_and_new_lines(new_lines, old_lines) elif 'class' in new_lines[0]: new_contents = add_to_class(new_lines, old_lines) else: lines_before = new_lines[:split_line_pos] last_line_before = lines_before[-1] lines_after = new_lines[split_line_pos + 1:] last_old_line = [ l for l in old_lines if l.strip() == last_line_before.strip() ][0] last_old_line_pos = old_lines.index(last_old_line) old_lines_after = old_lines[last_old_line_pos + 1:] # special-case: stray browser.quit in chap. 2 if 'rest of comments' in split_line: assert 'browser.quit()' in [l.strip() for l in old_lines_after] assert old_lines_after[-2] == 'browser.quit()' old_lines_after = old_lines_after[:-2] new_contents = '\n'.join( lines_before + [get_indent(split_line) + l for l in old_lines_after] + lines_after ) elif new_contents.strip().startswith("[...]") and new_contents.endswith("[...]"): new_contents = _replace_lines_in(old_lines, new_lines[1:-1]) else: raise Exception("I don't know how to deal with this") # strip trailing whitespace new_contents = re.sub(r'^ +$', '', new_contents, flags=re.MULTILINE) source.update(new_contents) source.write() ================================================ FILE: theme/epub/epub.css ================================================ /* Styling for custom captions on code blocks */ .sourcecode p { text-align: right; display: block; margin-bottom: -3pt; font-style: italic; hyphens: none; } div[data-type="example"].sourcecode pre { margin: 25px 0 25px 25px; } div[data-type="example"].sourcecode { margin-bottom: 0; } ================================================ FILE: theme/epub/epub.xsl ================================================

SCRATCHPAD:
 ================================================ FILE: theme/epub/layout.html ================================================ {{ doctype }} {{ title }} {{ content }} ================================================ FILE: theme/html/html.xsl ================================================ ================================================ FILE: theme/mobi/layout.html ================================================ {{ doctype }} {{ title }} {{ content }} ================================================ FILE: theme/mobi/mobi.css ================================================ /* Styling for custom captions on code blocks */ .sourcecode p { text-align: right; display: block; margin-bottom: -3pt; font-style: italic; hyphens: none; } div[data-type="example"].sourcecode pre { margin: 25px 0 25px 25px; } div[data-type="example"].sourcecode { margin-bottom: 0; } ================================================ FILE: theme/mobi/mobi.xsl ================================================

SCRATCHPAD:
 ================================================ FILE: theme/pdf/pdf.css ================================================ @charset "UTF-8"; /*----Rendering for special role="caption" lines below code blocks, per AU request; see RT #151714----*/ /*----Amended by author in may 2013 to use role="sourcecode" and be the official title for the codeblock...----*/ /*--Adjusting padding in TOC to avoid bad break--*/ @page toc:first { /* first page */ padding-bottom: 0.5in; } @page toc: { padding-bottom: 0.5in; } /* Custom widths */ .width-10 img { width: 10% !important; } .width-20 img { width: 20% !important; } .width-30 img { width: 30% !important; } .width-40 img { width: 40% !important; } .width-50 img { width: 50% !important; } .width-60 img { width: 60% !important; } .width-70 img { width: 70% !important; } .width-80 img { width: 80% !important; } .width-90 img { width: 90% !important; } .width-95 img { width: 95% !important; } /* less space for sidebar pagebreaks*/ .less_space {margin-top: 0 !important;} /* tighten tracking for paragraphs */ .fix_tracking { letter-spacing: -0.1pt; } /* Adding font fallback for characters not represented in standard text font */ body[data-type="book"] { font-family: MinionPro, Symbola, ArialUnicodeMS } /* Globally preventing code blocks from breaking across pages */ div[data-type="example"], pre { page-break-inside: avoid; } /* Removing labels from formal code blocks */ section[data-type="chapter"] div[data-type="example"] h5:before { content: none; } section[data-type="appendix"] div[data-type="example"] h5:before { content: none; } section[data-type="preface"] div[data-type="example"] h5:before { content: none; } div[data-type="part"] div[data-type="example"] h5:before { content: none; } div[data-type="part"] section[data-type="chapter"] div[data-type="example"] h5:before { content: none; } div[data-type="part"] section[data-type="appendix"] div[data-type="example"] h5:before { content: none; } /* Styling the file name captions on code blocks */ div.sourcecode h5 { text-align: right; display: block; margin-bottom: 1pt; font-style: italic; hyphens: none; font-size: 9.3pt; } /*Splitting a list into two columns*/ ul.two-col { columns: 2; } /* Styling formal code blocks with file name captions like informal code blocks */ div.sourcecode pre { margin-left: 17pt; } /* Add some space below sourcecode code blocks (STYL-991) */ div.sourcecode { margin-bottom: 1.5em; } /* Not sure what this custom formatting is for, was blindly ported from first edition */ div.small-code pre, pre.small-code { font-size: 75%; } aside[data-type="sidebar"] pre.small-code code { font-size: 95%; } blockquote.blockquote{ font-style:italic; } .blockquote p{ text-align:right; } /*---custom formatting for scratchpad items---*/ aside[data-type="sidebar"].scratchpad { overflow: auto; margin-top: 0.625in; } aside[data-type="sidebar"].scratchpad:before { content: " "; background-image: url('../../images/papertop.png'); position: relative; background-repeat: no-repeat; background-size: 4in auto; display: block; height: 0.386in; margin-bottom: 0; padding-bottom: 0; width: 4in; margin-left: -25px; top: -0.386in; } aside[data-type="sidebar"].scratchpad { background-image: url('../../images/papermiddle.png'); background-position: left top; background-repeat: repeat-y; background-size: 4in auto; padding-top: 0; padding-bottom: 0; padding-left: 25px; width: 4in; page-break-inside: avoid; border: none; } aside[data-type="sidebar"].scratchpad:after { content: " "; background-image: url('../../images/paperbottom.png'); background-repeat: no-repeat; background-size: 4in auto; display: block; height: 0.377in; margin-top: 0; width: 4in; margin-left: -25px; margin-top: 3pt; } aside[data-type="sidebar"].scratchpad ul { margin-top: -28pt; margin-left: 10pt; list-style-type: none; padding-right: 1.2in; } aside[data-type="sidebar"].scratchpad ul li { line-height: 0.165in; margin-top: 0; margin-bottom: 1pt; } aside[data-type="sidebar"].scratchpad ul li p { font-family: "ORAHand-Medium", ArialUnicodeMS; margin-top: 0; margin-bottom: 0; } /* Mock H3 */ span.fake-h3 { font-family: "MyriadPro-SemiboldCond"; font-weight: 600; font-size: 11.56pt; } /*----Uncomment to change the TOC start page (set the number to one page _after_ the one you want; so 6 to start on v, 8 to start on vii, etc.) ----*/ /* handling for elements to keep them from breaking across pages */ .nobreakinside {page-break-inside: avoid; page-break-before: auto;} /*----Uncomment to change the TOC start page (set the number to one page _after_ the one you want; so 6 to start on v, 8 to start on vii, etc.) ----*/ /* handling for elements to keep them from breaking across pages */ .nobreakinside {page-break-inside: avoid; page-break-before: auto;} /*--------Put Your Custom CSS Rules Below--------*/ /*--- This oneoff overrides the code in https://github.com/oreillymedia//blob/master/pdf/pdf.css---*/ /*----Uncomment to temporarily turn on code-eyballer highlighting (make sure to recomment after you build) pre { background-color: yellow; }---*/ /*----Uncomment to turn on automatic code wrapping pre { white-space: pre-wrap; word-wrap: break-word; } ----*/ /*----Uncomment to change the TOC start page (set the number to one page _after_ the one you want; so 6 to start on v, 8 to start on vii, etc.) @page toc:first { counter-reset: page 6; } ----*/ /*----Uncomment to fix a bad break in the title (increase padding value to push down, decrease value to pull up) section[data-type="titlepage"] h1 { padding-left: 1.5in; } ----*/ /*----Uncomment to fix a bad break in the subtitle (increase padding value to push down, decrease value to pull up) section[data-type="titlepage"] h2 { padding-left: 1in; } ----*/ /*----Uncomment to fix a bad break in the author names (increase padding value to push down, decrease value to pull up) section[data-type="titlepage"] p.author { padding-left: 3in; } ----*/ ================================================ FILE: theme/pdf/pdf.xsl ================================================ ================================================ FILE: titlepage.html ================================================

Test-Driven Development with Python

Third Edition

Obey the Testing Goat: Using Django, Selenium, and JavaScript

Harry J.W. Percival

================================================ FILE: toc.html ================================================