` after being safely escaped.
Usage:
Load the tag library, then invoke the tag with a name and optional
HTML attributes.
Template:
{% load visualization %}
{% concordia_visualization "daily-activity"
style="float:left;" class="chart" data-role="viz" %}
Output:
Notes:
- The `class` attribute you pass is appended to
`visualization-container`.
- All attribute names and values are escaped.
- This tag does not include any `",
}
data["description"] += "
Arm "
form = SanitizedDescriptionAdminForm(data)
self.assertTrue(form.is_valid())
self.assertEqual(form.clean_short_description(), short_description)
self.assertEqual(form.clean_description(), "
Arm ")
class ClearCacheFormTests(TestCase):
@override_settings(
CACHES={
"default": {
"BACKEND": "django.core.cache.backends.dummy.DummyCache",
},
"view_cache": {
"BACKEND": "django.core.cache.backends.dummy.DummyCache",
},
}
)
def test_cache_name_choices(self):
choices = get_cache_name_choices()
choice_names = [name for name, description in choices]
self.assertNotIn("default", choice_names)
self.assertIn("view_cache", choice_names)
================================================
FILE: concordia/tests/test_admin_views.py
================================================
import copy
import json
from functools import wraps
from http import HTTPStatus
from io import BytesIO
from unittest import mock
from django.contrib.messages import get_messages
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import RequestFactory, TestCase
from django.urls import reverse
from django.utils import timezone
from django.utils.text import slugify
from django.utils.timezone import now
from concordia.admin.views import SerializedObjectView
from concordia.models import Campaign, Project, TranscriptionStatus
from concordia.tests.utils import (
CreateTestUsers,
StreamingTestMixin,
create_asset,
create_campaign,
create_card,
create_item,
create_project,
create_site_report,
create_transcription,
)
from concordia.utils import get_anonymous_user
from importer.tests.utils import create_import_asset
class TestProjectLevelExportView(CreateTestUsers, TestCase):
def setUp(self):
self.login_user(is_staff=True, is_superuser=True)
self.url = reverse("admin:project-level-export")
self.asset = create_asset(download_url="http://example.com/1234.jpg")
self.asset2 = create_asset(
slug="asset-2",
item=self.asset.item,
download_url="http://example.com/5678.jpg",
)
self.asset3 = create_asset(
slug="asset-3",
item=self.asset.item,
download_url="http://example.com/9012.jpg",
)
def test_get(self):
response = self.client.get(self.url)
self.assertEqual(response.status_code, 200)
self.assertContains(
response, f"
{self.asset.item.project.campaign.title} ", html=True
)
def test_get_campaign(self):
response = self.client.get(
self.url, {"id": self.asset.item.project.campaign.id}
)
self.assertContains(
response, f"
{self.asset.item.project.title} ", html=True
)
def test_post(self):
with mock.patch("exporter.views.boto3.resource", autospec=True) as bucket_mock:
# The parameter is 'project_name', but it actually expects the project id.
response = self.client.post(
f"{self.url}?slug={self.asset.item.project.campaign.slug}",
{"project_name": f"{self.asset.item.project.id}"},
)
self.assertEqual(response.status_code, 200)
self.assertEqual(response["Content-Type"], "application/zip")
self.assertFalse(bucket_mock.called)
class TestFunctionBasedViews(CreateTestUsers, TestCase, StreamingTestMixin):
def test_admin_bulk_import_review(self):
self.login_user(is_staff=True, is_superuser=True)
self.assertTrue(self.user.is_active)
self.assertTrue(self.user.is_staff)
self.assertTrue(self.user.is_superuser)
path = reverse("admin:bulk-review")
response = self.client.get(path)
self.assertEqual(response.status_code, 200)
data = {}
response = self.client.post(path, data=data)
self.assertEqual(response.status_code, 200)
def test_admin_site_report_view(self):
self.login_user(is_staff=True, is_superuser=True)
mocked_datetime = timezone.now()
mocked_datetime_formatted = mocked_datetime.isoformat()
with mock.patch("django.utils.timezone.now") as now_mocked:
now_mocked.return_value = mocked_datetime
create_site_report()
response = self.client.get(reverse("admin:site-report"))
self.assertEqual(response.status_code, 200)
content = self.get_streaming_content(response).split(b"\r\n")
self.assertEqual(len(content), 3) # Includes empty line at the end of the file
test_data = [
b"Date,report name,Campaign,topic__title,assets total,assets published,"
b"assets not started,assets in progress,assets waiting review,"
b"assets completed,assets unpublished,assets started,items published,"
b"items unpublished,projects published,projects unpublished,"
b"anonymous transcriptions,transcriptions saved,daily review actions,"
b"distinct tags,tag uses,campaigns published,campaigns unpublished,"
b"users registered,users activated,registered contributors,"
b"daily active users",
b"%s,,,,,,,,,,,,,,,,,,,,,,,,,," % str.encode(mocked_datetime_formatted),
b"",
]
self.assertEqual(content, test_data)
def test_admin_retired_site_report_view(self):
self.login_user(is_staff=True, is_superuser=True)
response = self.client.get(reverse("admin:retired-site-report"))
self.assertEqual(response.status_code, 200)
content = self.get_streaming_content(response).split(b"\r\n")
self.assertEqual(len(content), 3) # Includes empty line at the end of the file
test_data = [
b"Date,report name,Campaign,topic__title,assets total,assets published,"
b"assets not started,assets in progress,assets waiting review,"
b"assets completed,assets unpublished,assets started,items published,"
b"items unpublished,projects published,projects unpublished,"
b"anonymous transcriptions,transcriptions saved,daily review actions,"
b"distinct tags,tag uses,campaigns published,campaigns unpublished,"
b"users registered,users activated,registered contributors,"
b"daily active users",
b",RETIRED TOTAL,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0",
b"",
]
self.assertEqual(content, test_data)
class TestAdminBulkImportView(CreateTestUsers, TestCase):
def setUp(self):
self.login_user(is_staff=True, is_superuser=True)
self.path = reverse("admin:bulk-import")
self.campaign_title = "Test Campaign"
self.campaign_short_description = "Short description"
self.campaign_long_description = "Long description"
self.campaign_slug = "test-campaign"
self.project_slug = "test-project"
self.project_title = "Test Project"
self.project_description = "Project description"
self.url = "http://example.com"
self.spreadsheet_data = {
"Campaign": self.campaign_title,
"Campaign Short Description": self.campaign_short_description,
"Campaign Long Description": self.campaign_long_description,
"Campaign Slug": self.campaign_slug,
"Project Slug": self.project_slug,
"Project": self.project_title,
"Project Description": self.project_description,
"Import URLs": self.url,
}
self.post_data = {"spreadsheet_file": BytesIO()}
def test_get(self):
response = self.client.get(self.path)
self.assertEqual(response.status_code, 200)
self.assertIn("form", response.context)
def test_invalid_form(self):
response = self.client.post(self.path)
self.assertEqual(response.status_code, 200)
self.assertIn("form", response.context)
def test_fully_valid_form(self):
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.import_items_into_project_from_url",
autospec=True,
),
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [self.spreadsheet_data]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0], f"Created new campaign {self.campaign_title}")
self.assertEqual(messages[1], f"Created new project {self.project_title}")
self.assertEqual(
messages[2],
f"Queued {self.campaign_title} {self.project_title} "
f"import for {self.url}",
)
campaign = Campaign.objects.get()
self.assertEqual(campaign.title, self.campaign_title)
self.assertEqual(campaign.slug, self.campaign_slug)
self.assertEqual(campaign.description, self.campaign_long_description)
self.assertEqual(
campaign.short_description, self.campaign_short_description
)
project = Project.objects.get()
self.assertEqual(project.title, self.project_title)
self.assertEqual(project.slug, self.project_slug)
self.assertEqual(project.description, self.project_description)
# Submit it again to test that it doesn't re-create the campaign or project
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(
messages[0],
f"Reusing campaign {self.campaign_title} without modification",
)
self.assertEqual(
messages[1],
f"Reusing project {self.project_title} without modification",
)
self.assertEqual(
messages[2],
f"Queued {self.campaign_title} {self.project_title} "
f"import for {self.url}",
)
self.assertEqual(1, Campaign.objects.count())
self.assertEqual(1, Project.objects.count())
def test_missing_field(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
del spreadsheet_data["Campaign"]
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.import_items_into_project_from_url",
autospec=True,
),
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 1)
self.assertEqual(
str(messages[0]), "Skipping row 0: missing fields ['Campaign']"
)
def test_empty_field(self):
# Only three fields require values: Campaign, Projet and Import URLs.
# Other fields must be present but can be empty.
# This tests that blank value check
spreadsheet_data = copy.copy(self.spreadsheet_data)
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.import_items_into_project_from_url",
autospec=True,
),
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
# Test one empty field
spreadsheet_data["Campaign"] = ""
slurp_mock.return_value = [spreadsheet_data]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 1)
self.assertEqual(
messages[0],
"Skipping row 0: at least one required field "
"(Campaign, Project, Import URLs) is empty",
)
def test_all_empty_fields(self):
# If all values in a spreadsheet row are empty, the row is skipped silently
spreadsheet_data = {key: "" for key in self.spreadsheet_data.keys()}
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.import_items_into_project_from_url",
autospec=True,
),
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 0)
def test_empty_campaign_slug(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Campaign Slug"] = ""
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.import_items_into_project_from_url",
autospec=True,
),
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0], f"Created new campaign {self.campaign_title}")
self.assertEqual(messages[1], f"Created new project {self.project_title}")
self.assertEqual(
messages[2],
f"Queued {self.campaign_title} {self.project_title} import for {self.url}",
)
# Since the provided campaign slug was blank, it should slugify the Campaign
# field instead
campaign = Campaign.objects.get()
self.assertEqual(
campaign.slug, slugify(self.campaign_title, allow_unicode=True)
)
def test_bad_campaign_slug(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Campaign Slug"] = "bad#slug@"
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.import_items_into_project_from_url",
autospec=True,
),
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 2)
self.assertEqual(messages[0], "Campaign slug doesn't match pattern.")
self.assertEqual(
messages[1],
"Unable to create campaign Test Campaign: {'slug': "
"['Enter a valid “slug” consisting of Unicode letters, "
"numbers, underscores, or hyphens.']}",
)
def test_empty_project_slug(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Project Slug"] = ""
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.import_items_into_project_from_url",
autospec=True,
),
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0], f"Created new campaign {self.campaign_title}")
self.assertEqual(messages[1], f"Created new project {self.project_title}")
self.assertEqual(
messages[2],
f"Queued {self.campaign_title} {self.project_title} import for {self.url}",
)
# Since the provided project slug was blank, it should slugify the project
# field instead
project = Project.objects.get()
self.assertEqual(project.slug, slugify(self.project_title, allow_unicode=True))
def test_bad_project_slug(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Project Slug"] = "bad#slug@"
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.import_items_into_project_from_url",
autospec=True,
),
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0], f"Created new campaign {self.campaign_title}")
self.assertEqual(messages[1], "Project slug doesn't match pattern.")
self.assertEqual(
messages[2],
"Unable to create project Test Project: {'slug': "
"['Enter a valid “slug” consisting of Unicode letters, "
"numbers, underscores, or hyphens.']}",
)
def test_bad_url(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Import URLs"] = bad_url = "ftp://example.com"
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.import_items_into_project_from_url",
autospec=True,
),
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0], f"Created new campaign {self.campaign_title}")
self.assertEqual(messages[1], f"Created new project {self.project_title}")
self.assertEqual(messages[2], f"Skipping unrecognized URL value: {bad_url}")
def test_import_task_exception(self):
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.import_items_into_project_from_url",
autospec=True,
) as import_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [self.spreadsheet_data]
import_mock.side_effect = Exception("Test Exception")
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0], f"Created new campaign {self.campaign_title}")
self.assertEqual(messages[1], f"Created new project {self.project_title}")
self.assertEqual(
messages[2],
f"Unhandled error attempting to import {self.url}: Test Exception",
)
class TestAdminBulkChangeAssetStatus(CreateTestUsers, TestCase):
def setUp(self):
item = create_item()
self.assets = [
create_asset(item=item),
create_asset(item=item, slug="test-asset-2"),
create_asset(item=item, slug="test-asset-3"),
create_asset(item=item, slug="test-asset-4"),
]
# Seed with existing transcriptions
self.accepted_transcription = create_transcription(
asset=self.assets[0], accepted=now()
)
self.rejected_transcription = create_transcription(
asset=self.assets[1], rejected=now()
)
self.submitted_transcription = create_transcription(asset=self.assets[2])
self.accepted_transcription2 = create_transcription(
asset=self.assets[3], accepted=now()
)
anon = get_anonymous_user()
self.spreadsheet_data = [
{
"asset__slug": self.assets[0].slug,
"New Status": "submitted",
"user": anon.id,
},
{
"asset__slug": self.assets[1].slug,
"New Status": "completed",
"user": anon.id,
},
{
"asset__slug": self.assets[2].slug,
"New Status": "completed",
"user": anon.id,
},
{
"asset__slug": self.assets[3].slug,
"New Status": "in_progress",
"user": anon.id,
},
]
def test_admin_bulk_change_asset_status(self):
self.login_user(is_staff=True, is_superuser=True)
fake_file = SimpleUploadedFile(
"test.xlsx", b"x", content_type="application/vnd.ms-excel"
)
post_data = {"spreadsheet_file": fake_file}
with mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock:
slurp_mock.return_value = self.spreadsheet_data
path = reverse("admin:bulk-change")
response = self.client.post(path, data=post_data)
for asset in self.assets:
asset.refresh_from_db()
self.assertEqual(response.status_code, 200)
slurp_mock.assert_called()
self.assertEqual(
self.assets[0].transcription_status,
TranscriptionStatus.SUBMITTED,
)
self.assertEqual(
self.assets[1].transcription_status,
TranscriptionStatus.COMPLETED,
)
self.assertEqual(
self.assets[2].transcription_status,
TranscriptionStatus.COMPLETED,
)
self.assertEqual(
self.assets[3].transcription_status,
TranscriptionStatus.IN_PROGRESS,
)
class TestAdminBulkImportReview(CreateTestUsers, TestCase):
def setUp(self):
self.login_user(is_staff=True, is_superuser=True)
self.path = reverse("admin:bulk-review")
self.campaign_title = "Test Campaign"
self.campaign_short_description = "Short description"
self.campaign_long_description = "Long description"
self.campaign_slug = "test-campaign"
self.project_slug = "test-project"
self.project_title = "Test Project"
self.project_description = "Project description"
self.url = "http://example.com"
self.spreadsheet_data = {
"Campaign": self.campaign_title,
"Campaign Short Description": self.campaign_short_description,
"Campaign Long Description": self.campaign_long_description,
"Campaign Slug": self.campaign_slug,
"Project Slug": self.project_slug,
"Project": self.project_title,
"Project Description": self.project_description,
"Import URLs": self.url,
}
self.post_data = {"spreadsheet_file": BytesIO()}
def test_get(self):
response = self.client.get(self.path)
self.assertEqual(response.status_code, 200)
self.assertIn("form", response.context)
def test_invalid_form(self):
response = self.client.post(self.path)
self.assertEqual(response.status_code, 200)
self.assertIn("form", response.context)
def test_fully_valid_form(self):
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.fetch_all_urls",
autospec=True,
) as fetch_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [self.spreadsheet_data]
fetch_mock.return_value = [["Fetch test message"], 1]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0], "Fetch test message")
self.assertEqual(messages[1], "Total Asset Count:1")
self.assertEqual(messages[2], "All Processes Completed")
def test_missing_field(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
del spreadsheet_data["Campaign"]
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.fetch_all_urls",
autospec=True,
) as fetch_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
fetch_mock.return_value = [["Fetch test message"], 1]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 4)
self.assertEqual(messages[0], "Skipping row 0: missing fields ['Campaign']")
self.assertEqual(messages[1], "Fetch test message")
self.assertEqual(messages[2], "Total Asset Count:1")
self.assertEqual(messages[3], "All Processes Completed")
def test_empty_field(self):
# Only three fields require values: Campaign, Projet and Import URLs.
# Other fields must be present but can be empty.
# This tests that blank value check
spreadsheet_data = copy.copy(self.spreadsheet_data)
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.fetch_all_urls",
autospec=True,
) as fetch_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
# Test one empty field
spreadsheet_data["Campaign"] = ""
slurp_mock.return_value = [spreadsheet_data]
fetch_mock.return_value = [["Fetch test message"], 1]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 4)
self.assertEqual(
messages[0],
"Skipping row 0: at least one required field "
"(Campaign, Project, Import URLs) is empty",
)
self.assertEqual(messages[1], "Fetch test message")
self.assertEqual(messages[2], "Total Asset Count:1")
self.assertEqual(messages[3], "All Processes Completed")
def test_all_empty_fields(self):
# If all values in a spreadsheet row are empty, the row is skipped silently
spreadsheet_data = {key: "" for key in self.spreadsheet_data.keys()}
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.fetch_all_urls",
autospec=True,
) as fetch_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
fetch_mock.return_value = [["Fetch test message"], 1]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0], "Fetch test message")
self.assertEqual(messages[1], "Total Asset Count:1")
self.assertEqual(messages[2], "All Processes Completed")
def test_empty_campaign_slug(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Campaign Slug"] = ""
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.fetch_all_urls",
autospec=True,
) as fetch_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
fetch_mock.return_value = [["Fetch test message"], 1]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0], "Fetch test message")
self.assertEqual(messages[1], "Total Asset Count:1")
self.assertEqual(messages[2], "All Processes Completed")
def test_bad_campaign_slug(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Campaign Slug"] = "bad#slug@"
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.fetch_all_urls",
autospec=True,
) as fetch_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
fetch_mock.return_value = [["Fetch test message"], 1]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 4)
self.assertEqual(messages[0], "Campaign slug doesn't match pattern.")
self.assertEqual(messages[1], "Fetch test message")
self.assertEqual(messages[2], "Total Asset Count:1")
self.assertEqual(messages[3], "All Processes Completed")
def test_empty_project_slug(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Project Slug"] = ""
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.fetch_all_urls",
autospec=True,
) as fetch_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
fetch_mock.return_value = [["Fetch test message"], 1]
fetch_mock.return_value = [["Fetch test message"], 1]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 3)
self.assertEqual(messages[0], "Fetch test message")
self.assertEqual(messages[1], "Total Asset Count:1")
self.assertEqual(messages[2], "All Processes Completed")
def test_bad_project_slug(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Project Slug"] = "bad#slug@"
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.fetch_all_urls",
autospec=True,
) as fetch_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
fetch_mock.return_value = [["Fetch test message"], 1]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 4)
self.assertEqual(messages[0], "Project slug doesn't match pattern.")
self.assertEqual(messages[1], "Fetch test message")
self.assertEqual(messages[2], "Total Asset Count:1")
self.assertEqual(messages[3], "All Processes Completed")
def test_bad_url(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Import URLs"] = bad_url = "ftp://example.com"
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.fetch_all_urls",
autospec=True,
) as fetch_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
fetch_mock.return_value = [["Fetch test message"], 1]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 4)
self.assertEqual(messages[0], f"Skipping unrecognized URL value: {bad_url}")
self.assertEqual(messages[1], "Fetch test message")
self.assertEqual(messages[2], "Total Asset Count:1")
self.assertEqual(messages[3], "All Processes Completed")
def test_large_number_urls(self):
spreadsheet_data = copy.copy(self.spreadsheet_data)
spreadsheet_data["Import URLs"] = " ".join([self.url for i in range(51)])
with (
mock.patch(
"concordia.admin.views.AdminProjectBulkImportForm", autospec=True
) as form_mock,
mock.patch(
"concordia.admin.views.slurp_excel", autospec=True
) as slurp_mock,
mock.patch(
"concordia.admin.views.fetch_all_urls",
autospec=True,
) as fetch_mock,
):
form_mock.return_value.is_valid.return_value = True
form_mock.return_value.cleaned_data = {}
slurp_mock.return_value = [spreadsheet_data]
fetch_mock.return_value = [["Fetch test message"], 1]
response = self.client.post(self.path, data=self.post_data)
self.assertEqual(response.status_code, 200)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(len(messages), 4)
self.assertEqual(messages[0], "Fetch test message")
self.assertEqual(messages[1], "Fetch test message")
# This count is weird because we mock the fetch_all_urls function
self.assertEqual(messages[2], "Total Asset Count:2")
self.assertEqual(messages[3], "All Processes Completed")
class TestCeleryTaskReview(CreateTestUsers, TestCase):
def setUp(self):
# We don't set up our data here because we want to test
# both with and without data
self.login_user(is_staff=True, is_superuser=True)
self.path = reverse("admin:celery-review")
def add_campaigns(self):
self.add_active_campaigns()
self.add_completed_campaigns()
self.add_retired_campaigns()
def add_active_campaigns(self):
self.campaign1 = create_campaign(
slug="test-active-campaign-1", title="Test Active Campaign 1"
)
self.campaign2 = create_campaign(
slug="test-active-campaign-2", title="Test Active Campaign 2"
)
def add_completed_campaigns(self):
self.completed_campaign1 = create_campaign(
slug="test-completed-campaign-1",
title="Test Completed Campaign 1",
status=Campaign.Status.COMPLETED,
)
self.completed_campaign2 = create_campaign(
slug="test-completed-campaign-2",
title="Test Completed Campaign 1",
status=Campaign.Status.COMPLETED,
)
def add_retired_campaigns(self):
self.retired_campaign1 = create_campaign(
slug="test-retired-campaign-1",
title="Test Retired Campaign 1",
status=Campaign.Status.RETIRED,
)
self.retired_campaign2 = create_campaign(
slug="test-retired-campaign-2",
title="Test Retired Campaign 1",
status=Campaign.Status.RETIRED,
)
def add_projects(self):
# Active campaign 1, three projects
create_project(
campaign=self.campaign1,
slug="campaign1-project-1",
title="Campaign 1 Project 1",
)
create_project(
campaign=self.campaign1,
slug="campaign1-project-2",
title="Campaign 1 Project 2",
)
create_project(
campaign=self.campaign1,
slug="campaign1-project-3",
title="Campaign 1 Project 3",
)
# Active campaign 2, two projects
create_project(
campaign=self.campaign2,
slug="campaign1-project-1",
title="Campaign 2 Project 1",
)
create_project(
campaign=self.campaign2,
slug="campaign1-project-2",
title="Campaign 2 Project 2",
)
# Completed campaign 1, two projects
create_project(
campaign=self.completed_campaign1,
slug="completed-campaign1-project-1",
title="Completed Campaign 1 Project 1",
)
create_project(
campaign=self.completed_campaign1,
slug="completed-campaign1-project-2",
title="Completed Campaign 1 Project 2",
)
# Completed campaign 2, one project
create_project(
campaign=self.completed_campaign2,
slug="completed-campaign2-project-1",
title="Completed Campaign 1 Project 1",
)
# We don't create any for retired campaigns since the campaigns
# are only created to make sure the view ignores them
def add_tasks(self, campaign):
data = []
for project in campaign.project_set.all():
import_asset = create_import_asset(1, project=project)
item = import_asset.import_item
import_job = item.job
create_import_asset(
2,
import_item=item,
import_job=import_job,
project=project,
last_started=timezone.now(),
)
create_import_asset(
3,
import_item=item,
import_job=import_job,
project=project,
failed=timezone.now(),
last_started=timezone.now(),
)
create_import_asset(
4,
import_item=item,
import_job=import_job,
project=project,
completed=timezone.now(),
last_started=timezone.now(),
)
data.append(
{
"title": project.title,
"id": project.id,
"campaign_id": str(campaign.id),
"successful": 1,
"incomplete": 1,
"unstarted": 1,
"failure": 1,
}
)
return data
def test_empty_dashboard(self):
response = self.client.get(self.path)
context = response.context
self.assertEqual(response.status_code, 200)
self.assertIn("campaigns", context)
campaigns = list(context["campaigns"])
self.assertEqual(campaigns, [])
def test_dashboard(self):
self.add_active_campaigns()
response = self.client.get(self.path)
context = response.context
self.assertEqual(response.status_code, 200)
self.assertIn("campaigns", context)
self.assertIn(self.campaign1, context["campaigns"])
self.assertIn(self.campaign2, context["campaigns"])
self.add_completed_campaigns()
response = self.client.get(self.path)
context = response.context
self.assertEqual(response.status_code, 200)
self.assertIn("campaigns", context)
campaigns = list(context["campaigns"])
self.assertIn(self.campaign1, campaigns)
self.assertIn(self.campaign2, campaigns)
self.assertIn(self.completed_campaign1, campaigns)
self.assertIn(self.completed_campaign2, campaigns)
self.add_retired_campaigns()
response = self.client.get(self.path)
context = response.context
self.assertEqual(response.status_code, 200)
self.assertIn("campaigns", context)
campaigns = list(context["campaigns"])
self.assertIn(self.campaign1, campaigns)
self.assertIn(self.campaign2, campaigns)
self.assertIn(self.completed_campaign1, campaigns)
self.assertIn(self.completed_campaign2, campaigns)
self.assertNotIn(self.retired_campaign1, campaigns)
self.assertNotIn(self.retired_campaign2, campaigns)
def test_campaign_dashboard(self):
self.add_campaigns()
self.add_projects()
data = self.add_tasks(self.campaign1)
response = self.client.get(self.path, {"id": self.campaign1.id})
context = response.context
self.assertIn("campaigns", context)
self.assertEqual(context["campaigns"], [])
self.assertIn("totalassets", context)
self.assertEqual(context["totalassets"], 12)
self.assertIn("projects", context)
self.assertEqual(context["projects"], data)
data = self.add_tasks(self.campaign2)
response = self.client.get(self.path, {"id": self.campaign2.id})
context = response.context
self.assertIn("campaigns", context)
self.assertEqual(context["campaigns"], [])
self.assertIn("totalassets", context)
self.assertEqual(context["totalassets"], 8)
self.assertIn("projects", context)
self.assertEqual(context["projects"], data)
data = self.add_tasks(self.completed_campaign1)
response = self.client.get(self.path, {"id": self.completed_campaign1.id})
context = response.context
self.assertIn("campaigns", context)
self.assertEqual(context["campaigns"], [])
self.assertIn("totalassets", context)
self.assertEqual(context["totalassets"], 8)
self.assertIn("projects", context)
self.assertEqual(context["projects"], data)
data = self.add_tasks(self.completed_campaign2)
response = self.client.get(self.path, {"id": self.completed_campaign2.id})
context = response.context
self.assertIn("campaigns", context)
self.assertEqual(context["campaigns"], [])
self.assertIn("totalassets", context)
self.assertEqual(context["totalassets"], 4)
self.assertIn("projects", context)
self.assertEqual(context["projects"], data)
class TestSerializedObjectView(TestCase):
def setUp(self):
self.card = create_card()
# Every test needs access to the request factory.
self.factory = RequestFactory()
def test_exists(self):
request = self.factory.get(
"/admin/card/",
{"model_name": "Card", "object_id": self.card.id, "field_name": "title"},
)
response = SerializedObjectView.as_view()(request)
self.assertEqual(response.status_code, 200)
self.assertEqual(json.loads(response.content)["title"], self.card.title)
def test_dne(self):
request = self.factory.get(
"/admin/card/",
{"model_name": "Card", "object_id": 3, "field_name": "title"},
)
response = SerializedObjectView.as_view()(request)
self.assertEqual(response.status_code, HTTPStatus.NOT_FOUND)
self.assertJSONEqual(response.content, {"status": "false"})
def mock_cache(object_to_patch):
def decorator(cls):
# Decorator to mock `django.core.cache.caches`.
# Passes the mock cache and caches_mock to each
# test method as additional arguments.
# We have to write this as a custom decorator
# in order to not have to create these mocks in
# each invidivual test method, since we need to override
# __getitem__ on the caches mock
# We need to create a helper function so each method
# gets a unique wrapper and mocks
def create_wrapper(attr):
@wraps(attr)
def wrapper(self, *args, **kwargs):
with mock.patch(object_to_patch) as caches_mock:
cache_mock = mock.MagicMock()
caches_mock.__getitem__.return_value = cache_mock
return attr(self, caches_mock, cache_mock, *args, **kwargs)
return wrapper
# Wrap each test method to include the mocks as arguments
for attr_name in dir(cls):
attr = getattr(cls, attr_name)
if callable(attr) and attr_name.startswith("test_"):
setattr(cls, attr_name, create_wrapper(attr))
return cls
return decorator
@mock_cache("concordia.admin.views.caches")
class TestClearCacheView(CreateTestUsers, TestCase):
def setUp(self):
self.login_user(is_staff=True, is_superuser=True)
self.path = reverse("admin:clear-cache")
def test_get(self, caches_mock, cache_mock):
response = self.client.get(self.path)
self.assertEqual(response.status_code, 200)
self.assertIn("form", response.context)
self.assertFalse(caches_mock.__getitem__.called)
self.assertFalse(cache_mock.clear.called)
def test_invalid_form(self, caches_mock, cache_mock):
response = self.client.post(self.path)
self.assertEqual(response.status_code, 200)
self.assertIn("form", response.context)
self.assertFalse(caches_mock.__getitem__.called)
self.assertFalse(cache_mock.clear.called)
def test_valid_form(self, caches_mock, cache_mock):
response = self.client.post(self.path, {"cache_name": "view_cache"})
self.assertEqual(response.status_code, 302)
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(messages[0], "Successfully cleared 'view_cache' cache")
self.assertTrue(caches_mock.__getitem__.called)
self.assertTrue(cache_mock.clear.called)
def test_form_with_invalid_data(self, caches_mock, cache_mock):
response = self.client.post(self.path, {"cache_name": "default"})
self.assertEqual(response.status_code, 200)
self.assertIn("form", response.context)
self.assertFalse(caches_mock.__getitem__.called)
self.assertFalse(cache_mock.clear.called)
def test_exception(self, caches_mock, cache_mock):
caches_mock.__getitem__.side_effect = Exception("Test Exception")
response = self.client.post(self.path, {"cache_name": "view_cache"})
messages = [str(message) for message in get_messages(response.wsgi_request)]
self.assertEqual(
messages[0],
"Couldn't clear cache 'view_cache', something went wrong. "
"Received error: Test Exception",
)
================================================
FILE: concordia/tests/test_api_views.py
================================================
from datetime import date
from unittest import mock
from urllib.parse import urlparse
from django.test import TestCase, override_settings
from django.urls import reverse
from django.utils.timezone import now
from concordia import api_views
from concordia.models import (
Asset,
Campaign,
Item,
Project,
Topic,
Transcription,
User,
)
from concordia.utils import get_anonymous_user
from .utils import (
JSONAssertMixin,
create_asset,
create_item,
create_project,
create_topic,
)
class URLAwareEncoderTest(TestCase):
def test_default(self):
encoder = api_views.URLAwareEncoder()
self.assertEqual(encoder.default(None), None)
obj = mock.Mock(spec=["url"])
self.assertEqual(encoder.default(obj), obj.url)
obj = mock.Mock(spec=["get_absolute_url"])
self.assertEqual(encoder.default(obj), obj.get_absolute_url())
# Test non-model object
obj = date.today()
self.assertEqual(encoder.default(obj), date.today().isoformat())
class APIViewMixinTest(TestCase):
def setUp(self):
self.mixin = api_views.APIViewMixin()
def test_serialize_conctext(self):
context = {"test-key": "test-value"}
self.assertEqual(self.mixin.serialize_context(context), context)
@mock.patch("concordia.api_views.model_to_dict")
def test_serialize_object(self, mtd_mock):
return_data = {"test-key": "test-value"}
mtd_mock.return_value = return_data
obj = mock.Mock(spec=["get_absolute_url"])
data = self.mixin.serialize_object(obj)
self.assertEqual(data, return_data | {"url": obj.get_absolute_url()})
obj = mock.Mock(spec=[])
data = self.mixin.serialize_object(obj)
self.assertEqual(data, return_data)
@mock.patch("concordia.api_views.time")
class APIListViewTest(TestCase):
def test_serialize_context(self, time_mock):
time_mock.return_value = "test-time"
view = api_views.APIListView()
context = {"object_list": []}
data = view.serialize_context(context)
self.assertEqual(data, {"objects": [], "sent": "test-time"})
@override_settings(RATELIMIT_ENABLE=False)
class ConcordiaViewTests(JSONAssertMixin, TestCase):
@classmethod
def setUpTestData(cls):
cls.anon_user = get_anonymous_user()
cls.reviewer = User.objects.create_user(
username="reviewer", email="tester@example.com"
)
# clear data from other tests
Project.objects.all().delete()
Topic.objects.all().delete()
cls.test_project = create_project()
cls.test_topic = create_topic(project=cls.test_project)
cls.items = [
create_item(
item_id=f"item_{i}",
title=f"Item {i}",
project=cls.test_project,
do_save=False,
)
for i in range(0, 3)
]
Item.objects.bulk_create(cls.items)
cls.assets = []
for item in cls.items:
cls.assets.append(
create_asset(
title=f"Thumbnail URL test for {item.id}",
item=item,
download_url="http://tile.loc.gov/image-services/iiif/"
"service:music:mussuffrage:mussuffrage-100183:mussuffrage-100183.0001/"
"full/pct:100/0/default.jpg",
do_save=False,
)
)
for i in range(0, 15):
cls.assets.append(
create_asset(title=f"{item.id} — {i}", item=item, do_save=False)
)
Asset.objects.bulk_create(cls.assets)
cls.transcriptions = []
for asset in cls.assets:
last_t = None
for n in range(0, 3):
cls.transcriptions.append(
Transcription(
asset=asset,
supersedes=last_t,
text=f"{asset} — {n}",
user=cls.anon_user,
)
)
Transcription.objects.bulk_create(cls.transcriptions)
submitted_t = cls.transcriptions[-1]
submitted_t.submitted = now()
submitted_t.full_clean()
submitted_t.save()
def get_api_response(self, url, **request_args):
"""
This issues a call to one of our API views and confirms that the
response follows our basic conventions of returning a valid JSON
response
"""
qs = {"format": "json"}
if request_args is not None:
qs.update(request_args)
resp = self.client.get(url, qs)
data = self.assertValidJSON(resp)
return resp, data
def get_api_list_response(self, url, page_size=10, **request_args):
"""
This issues a call to one of our API views and confirms that the
response follows our basic conventions of returning a top level object
with members“objects” (list) and “pagination” (object).
"""
qs = {"per_page": page_size}
if request_args is not None:
qs.update(request_args)
resp, data = self.get_api_response(url, **qs)
self.assertIn("objects", data)
self.assertIn("pagination", data)
object_count = len(data["objects"])
self.assertLessEqual(object_count, page_size)
self.assertAbsoluteURLs(data["objects"])
self.assertAbsoluteURLs(data["pagination"])
return resp, data
def assertAbsoluteUrl(self, url, allow_none=True):
"""Require a URL to either be None or an absolute URL"""
if url is None and allow_none:
return
parsed = urlparse(url)
self.assertIn(
parsed.scheme, ["http", "https"], msg=f"Expected {url} to have HTTP scheme"
)
self.assertTrue(parsed.netloc)
def assertAbsoluteURLs(self, data):
if isinstance(data, dict):
for k, v in data.items():
if k.endswith("url"):
self.assertAbsoluteUrl(v)
elif isinstance(v, (dict, list)):
self.assertAbsoluteURLs(v)
elif isinstance(data, list):
for i in data:
self.assertAbsoluteURLs(i)
else:
raise TypeError(
"assertAbsoluteURLs must be called with a dictionary or list"
)
def assertAssetStatuses(self, asset_list, expected_statuses):
asset_pks = [i["id"] for i in asset_list]
self.assertQuerySetEqual(
Asset.objects.filter(pk__in=asset_pks).exclude(
transcription_status__in=expected_statuses
),
[],
)
def assertAssetsHaveLatestTranscriptions(self, asset_list):
asset_pks = {i["id"]: i for i in asset_list}
for asset in Asset.objects.filter(pk__in=asset_pks.keys()):
latest_trans = asset.transcription_set.latest("pk")
if latest_trans is None:
self.assertIsNone(asset_pks[asset.id]["latest_transcription"])
else:
self.assertDictEqual(
asset_pks[asset.id]["latest_transcription"],
{
"id": latest_trans.pk,
"text": latest_trans.text,
"submitted_by": latest_trans.user_id,
},
)
def test_topic_detail(self):
resp, data = self.get_api_response(
reverse("topic-detail", kwargs={"slug": self.test_topic.slug})
)
self.assertIn("object", data)
self.assertNotIn("objects", data)
serialized_project = data["object"]
self.assertIn("id", serialized_project)
self.assertIn("url", serialized_project)
topic = self.test_topic
self.assertEqual(
serialized_project,
serialized_project
| {
"id": topic.id,
"title": topic.title,
"description": topic.description,
"slug": topic.slug,
"thumbnail_image": topic.thumbnail_image,
},
)
self.assertURLEqual(
serialized_project["url"], f"http://testserver{topic.get_absolute_url()}"
)
def test_campaign_list(self):
resp, data = self.get_api_list_response(reverse("transcriptions:campaign-list"))
self.assertGreater(len(data["objects"]), 0)
test_campaigns = {
i["id"]: i
for i in Campaign.objects.published().values(
"id", "title", "description", "short_description", "slug"
)
}
for obj in data["objects"]:
self.assertIn("id", obj)
self.assertIn("url", obj)
self.assertEqual(obj, obj | test_campaigns[obj["id"]])
def test_campaign_detail(self):
resp, data = self.get_api_response(
reverse(
"transcriptions:campaign-detail",
kwargs={"slug": self.test_project.campaign.slug},
)
)
self.assertIn("object", data)
self.assertNotIn("objects", data)
serialized_project = data["object"]
self.assertIn("id", serialized_project)
self.assertIn("url", serialized_project)
campaign = self.test_project.campaign
self.assertEqual(
serialized_project,
serialized_project
| {
"id": campaign.id,
"title": campaign.title,
"description": campaign.description,
"slug": campaign.slug,
"metadata": campaign.metadata,
"thumbnail_image": campaign.thumbnail_image,
},
)
self.assertURLEqual(
serialized_project["url"], f"http://testserver{campaign.get_absolute_url()}"
)
def test_project_detail(self):
project = self.test_project
resp, data = self.get_api_list_response(project.get_absolute_url())
# Until we clean up the project view code, projects have two key
# elements: objects lists the children (i.e. items) and the project
# itself is in a second top-level “project” object:
self.assertIn("objects", data)
self.assertIn("project", data)
self.assertNotIn("object", data)
serialized_project = data["project"]
self.assertIn("id", serialized_project)
self.assertIn("url", serialized_project)
self.assertURLEqual(
serialized_project["url"], f"http://testserver{project.get_absolute_url()}"
)
self.assertEqual(
serialized_project,
serialized_project
| {
"description": project.description,
"id": project.id,
"metadata": project.metadata,
"slug": project.slug,
"thumbnail_image": project.thumbnail_image,
"title": project.title,
},
)
for obj in data["objects"]:
self.assertIn("description", obj)
self.assertIn("item_id", obj)
self.assertIn("item_url", obj)
self.assertIn("metadata", obj)
self.assertIn("thumbnail_url", obj)
self.assertIn("title", obj)
self.assertIn("url", obj)
def test_item_detail(self):
item = self.test_project.item_set.first()
resp, data = self.get_api_list_response(item.get_absolute_url())
# Until we clean up the project view code, projects have two key
# elements: objects lists the children (i.e. items) and the project
# itself is in a second top-level “project” object:
self.assertIn("objects", data)
self.assertIn("item", data)
self.assertNotIn("object", data)
serialized_item = data["item"]
self.assertIn("id", serialized_item)
self.assertIn("url", serialized_item)
self.assertIn("thumbnail_url", serialized_item)
self.assertURLEqual(
serialized_item["url"], f"http://testserver{item.get_absolute_url()}"
)
self.assertEqual(
serialized_item,
serialized_item
| {
"description": item.description,
"id": item.id,
"item_id": item.item_id,
"metadata": item.metadata,
"title": item.title,
},
)
for obj in data["objects"]:
self.assertIn("description", obj)
self.assertIn("difficulty", obj)
self.assertIn("metadata", obj)
self.assertIn("image_url", obj)
self.assertIn("thumbnail_url", obj)
self.assertIn("resource_url", obj)
self.assertIn("title", obj)
self.assertIn("slug", obj)
self.assertIn("url", obj)
self.assertIn("year", obj)
if "Thumbnail test" in obj["title"]:
self.assertIn("https", obj["thumbnail_url"])
================================================
FILE: concordia/tests/test_authentication.py
================================================
from django.test import RequestFactory, TestCase
from concordia.authentication_backends import EmailOrUsernameModelBackend
from .utils import CreateTestUsers
class AuthenticationBackendTests(TestCase, CreateTestUsers):
def test_EmailOrUsernameModelBackend(self):
backend = EmailOrUsernameModelBackend()
request_factory = RequestFactory()
test_user = self.create_user("tester")
request = request_factory.get("/")
# Fail to authenticate with no information
user = backend.authenticate(request)
self.assertEqual(user, None)
# Fail to authenticate with no password, using username
user = backend.authenticate(request, test_user.username)
self.assertEqual(user, None)
# Authenticate with correct password, using username
user = backend.authenticate(request, test_user.username, test_user._password)
self.assertEqual(user, test_user)
# Fail to authenticate with no password, using email
user = backend.authenticate(request, test_user.email)
self.assertEqual(user, None)
# Authenticate with correct password, using email
user = backend.authenticate(request, test_user.email, test_user._password)
self.assertEqual(user, test_user)
# Fail to authenticate with incorrect password, using username
user = backend.authenticate(request, test_user.username, "bad-password")
self.assertEqual(user, None)
# Fail to authenticate with incorrect password, using email
user = backend.authenticate(request, test_user.email, "bad-password")
self.assertEqual(user, None)
# Same tests, with user with a username
# the same as the first user's email address
test_user2 = self.create_user(test_user.email)
# Fail to authenticate with no password, using username
user = backend.authenticate(request, test_user2.username)
self.assertEqual(user, None)
# Authenticate with correct password, using username
user = backend.authenticate(request, test_user2.username, test_user2._password)
self.assertEqual(user, test_user2)
# Fail to authenticate with no password, using email
user = backend.authenticate(request, test_user2.email)
self.assertEqual(user, None)
# Authenticate with correct password, using email
user = backend.authenticate(request, test_user2.email, test_user2._password)
self.assertEqual(user, test_user2)
# Fail to authenticate with incorrect password, using username
user = backend.authenticate(request, test_user2.username, "bad-password")
self.assertEqual(user, None)
# Fail to authenticate with incorrect password, using email
user = backend.authenticate(request, test_user2.email, "bad-password")
self.assertEqual(user, None)
================================================
FILE: concordia/tests/test_celery.py
================================================
import tempfile
from types import SimpleNamespace
from unittest import mock
from django.test import TestCase
import concordia.celery as celery_mod
from concordia.celery import import_all_submodules
class ConcordiaCeleryTests(TestCase):
def test_returns_early_for_non_package(self):
mock_pkg = SimpleNamespace(__name__="not_a_pkg") # no __path__
with (
mock.patch.object(
celery_mod.importlib, "import_module", return_value=mock_pkg
) as mock_import,
mock.patch.object(celery_mod.pkgutil, "walk_packages") as mock_walk,
):
import_all_submodules("not_a_pkg")
mock_import.assert_called_once_with("not_a_pkg")
mock_walk.assert_not_called()
def test_imports_all_submodules_for_package(self):
sub1 = SimpleNamespace(name="dummy_pkg.sub1")
sub2 = SimpleNamespace(name="dummy_pkg.sub2")
with tempfile.TemporaryDirectory() as td:
mock_pkg = SimpleNamespace(__name__="dummy_pkg", __path__=[td])
with (
mock.patch.object(celery_mod.importlib, "import_module") as mock_import,
mock.patch.object(
celery_mod.pkgutil, "walk_packages", return_value=[sub1, sub2]
) as mock_walk,
):
def side_effect(name):
if name == "dummy_pkg":
return mock_pkg
return SimpleNamespace(__name__=name)
mock_import.side_effect = side_effect
import_all_submodules("dummy_pkg")
mock_walk.assert_called_once()
args, _kwargs = mock_walk.call_args
self.assertEqual(args[0], mock_pkg.__path__)
self.assertEqual(args[1], mock_pkg.__name__ + ".")
self.assertIn(mock.call("dummy_pkg"), mock_import.mock_calls)
self.assertIn(mock.call("dummy_pkg.sub1"), mock_import.mock_calls)
self.assertIn(mock.call("dummy_pkg.sub2"), mock_import.mock_calls)
def test_package_with_no_submodules(self):
with tempfile.TemporaryDirectory() as td:
mock_pkg = SimpleNamespace(__name__="empty_pkg", __path__=[td])
with (
mock.patch.object(celery_mod.importlib, "import_module") as mock_import,
mock.patch.object(
celery_mod.pkgutil, "walk_packages", return_value=[]
) as mock_walk,
):
mock_import.side_effect = lambda name: (
mock_pkg if name == "empty_pkg" else SimpleNamespace(__name__=name)
)
import_all_submodules("empty_pkg")
mock_walk.assert_called_once()
mock_import.assert_called_once_with("empty_pkg")
def test__load_all_task_modules_invokes_imports(self):
with mock.patch.object(celery_mod, "import_all_submodules") as mock_import_all:
celery_mod._load_all_task_modules(sender=celery_mod.app)
mock_import_all.assert_has_calls(
[
mock.call("concordia.tasks"),
mock.call("importer.tasks"),
],
any_order=False,
)
def test_on_after_finalize_signal_triggers_handler(self):
with mock.patch.object(celery_mod, "import_all_submodules") as mock_import_all:
celery_mod.app.on_after_finalize.send(sender=celery_mod.app)
mock_import_all.assert_has_calls(
[mock.call("concordia.tasks"), mock.call("importer.tasks")],
any_order=False,
)
self.assertEqual(mock_import_all.call_count, 2)
================================================
FILE: concordia/tests/test_consumers.py
================================================
from asgiref.sync import sync_to_async
from channels.testing import WebsocketCommunicator
from django.test import RequestFactory, TransactionTestCase
from django.urls import reverse
from concordia.consumers import AssetConsumer
from concordia.utils import get_or_create_reservation_token
from concordia.views.ajax import obtain_reservation
from .utils import CreateTestUsers, create_asset, create_item, create_transcription
class TestAssetConsumer(CreateTestUsers, TransactionTestCase):
"""
Normally defining communicator would be in setUp
and communicator.disconnect would be called in tearDown
Asynchronous code doesn't seem to work well with those methods
so those lines are in each test.
"""
async def test_asset_update(self):
communicator = WebsocketCommunicator(
AssetConsumer.as_asgi(), "ws/asset/asset_updates/"
)
connected, subprotocol = await communicator.connect()
self.assertTrue(connected)
asset = await sync_to_async(create_asset)()
response = await communicator.receive_json_from()
message = response["message"]
self.assertEqual(message["type"], "asset_update")
self.assertEqual(message["asset_pk"], asset.pk)
self.assertEqual(message["status"], "not_started")
self.assertEqual(message["latest_transcription"], None)
await sync_to_async(create_item)(item_id="item-2", project=asset.item.project)
response = await communicator.receive_nothing()
self.assertTrue(response)
user = await sync_to_async(self.create_test_user)()
transcription = await sync_to_async(create_transcription)(
asset=asset, user=user
)
response = await communicator.receive_json_from()
message = response["message"]
self.assertEqual(message["type"], "asset_update")
self.assertEqual(message["asset_pk"], asset.pk)
self.assertEqual(message["status"], "in_progress")
self.assertEqual(message["latest_transcription"]["id"], transcription.pk)
await communicator.disconnect()
async def test_asset_reservation_obtained(self):
asset = await sync_to_async(create_asset)()
communicator = WebsocketCommunicator(
AssetConsumer.as_asgi(), "ws/asset/asset_updates/"
)
connected, subprotocol = await communicator.connect()
self.assertTrue(connected)
request_factory = RequestFactory()
request = request_factory.get("/")
request.session = {}
token = get_or_create_reservation_token(request)
await sync_to_async(obtain_reservation)(asset.pk, token)
response = await communicator.receive_json_from()
message = response["message"]
self.assertEqual(message["type"], "asset_reservation_obtained")
self.assertEqual(message["asset_pk"], asset.pk)
await communicator.disconnect()
async def test_asset_reservation_released(self):
asset = await sync_to_async(create_asset)()
await self.async_client.get(
reverse("reserve-asset", kwargs={"asset_pk": asset.pk})
)
communicator = WebsocketCommunicator(
AssetConsumer.as_asgi(), "ws/asset/asset_updates/"
)
connected, subprotocol = await communicator.connect()
self.assertTrue(connected)
await self.async_client.post(
reverse("reserve-asset", kwargs={"asset_pk": asset.pk}),
{"release": "release"},
)
response = await communicator.receive_json_from()
message = response["message"]
self.assertEqual(message["type"], "asset_reservation_released")
self.assertEqual(message["asset_pk"], asset.pk)
await communicator.disconnect()
================================================
FILE: concordia/tests/test_contextmanagers.py
================================================
from unittest import TestCase
from unittest.mock import patch
from concordia.contextmanagers import DEFAULT_LOCK_DURATION, cache_lock
class CacheLockTests(TestCase):
def setUp(self):
self.lock_id = "test-lock"
self.oid = "worker-1"
self.cache_patch = patch("concordia.contextmanagers.cache")
self.mock_cache = self.cache_patch.start()
self.addCleanup(self.cache_patch.stop)
self.time_patch = patch("concordia.contextmanagers.time.monotonic")
self.mock_monotonic = self.time_patch.start()
self.addCleanup(self.time_patch.stop)
self.start_time = 100.0
self.mock_monotonic.return_value = self.start_time
def test_acquires_and_releases_lock(self):
self.mock_cache.add.return_value = True
with cache_lock(self.lock_id, self.oid) as acquired:
self.assertTrue(acquired)
self.mock_cache.add.assert_called_once_with(
self.lock_id, self.oid, DEFAULT_LOCK_DURATION
)
self.mock_cache.delete.assert_called_once_with(self.lock_id)
def test_does_not_release_if_lock_not_acquired(self):
self.mock_cache.add.return_value = False
with cache_lock(self.lock_id, self.oid) as acquired:
self.assertFalse(acquired)
self.mock_cache.delete.assert_not_called()
def test_does_not_release_if_expired(self):
self.mock_cache.add.return_value = True
# Simulate expiration: time has passed beyond timeout
def advance_time():
return self.start_time + DEFAULT_LOCK_DURATION + 1
self.mock_monotonic.side_effect = [self.start_time, advance_time()]
with cache_lock(self.lock_id, self.oid) as acquired:
self.assertTrue(acquired)
self.mock_cache.delete.assert_not_called()
================================================
FILE: concordia/tests/test_decorators.py
================================================
from unittest import TestCase
from unittest.mock import MagicMock, patch
from celery import Task
from concordia.decorators import locked_task
class LockedTaskDecoratorTests(TestCase):
def setUp(self):
self.hostname = "test-worker"
self.logger_patch = patch("concordia.decorators.logger")
self.logger = self.logger_patch.start()
self.addCleanup(self.logger_patch.stop)
self.cache_lock_patch = patch("concordia.decorators.cache_lock")
self.mock_cache_lock = self.cache_lock_patch.start()
self.addCleanup(self.cache_lock_patch.stop)
def make_task_instance(self, name="test-task"):
task = MagicMock(spec=Task)
task.name = name
task.request.hostname = self.hostname
return task
def test_lock_by_args_allows_only_one_execution(self):
task = self.make_task_instance()
calls = []
@locked_task
def dummy(self, arg):
calls.append(arg)
return f"Ran with {arg}"
dummy_task = dummy.__get__(task)
self.mock_cache_lock.return_value.__enter__.return_value = True
result = dummy_task("foo")
self.assertEqual(result, "Ran with foo")
self.assertEqual(calls, ["foo"])
self.mock_cache_lock.return_value.__enter__.return_value = False
result = dummy_task("foo")
self.assertIsNone(result)
self.logger.info.assert_called_once()
def test_lock_by_task_name(self):
task = self.make_task_instance()
calls = []
@locked_task(lock_by_args=False)
def dummy(self, arg):
calls.append(arg)
return f"Ran with {arg}"
dummy_task = dummy.__get__(task)
self.mock_cache_lock.return_value.__enter__.return_value = True
result = dummy_task("foo")
self.assertEqual(result, "Ran with foo")
self.assertEqual(calls, ["foo"])
def test_force_runs_even_if_lock_not_acquired(self):
task = self.make_task_instance()
calls = []
@locked_task
def dummy(self, arg):
calls.append(arg)
return f"Forced {arg}"
dummy_task = dummy.__get__(task)
self.mock_cache_lock.return_value.__enter__.return_value = False
result = dummy_task("bar", force=True)
self.assertEqual(result, "Forced bar")
self.logger.warning.assert_called_once()
def test_error_in_key_generation_logs_and_raises(self):
task = self.make_task_instance()
@locked_task
def dummy(self, arg):
return "This shouldn't run"
dummy_task = dummy.__get__(task)
# Use a non-repr-able object to simulate key generation failure
class Unreprable:
def __repr__(self):
raise ValueError("Can't repr")
with self.assertRaises(ValueError):
dummy_task(Unreprable())
self.logger.exception.assert_called_once_with(
"Unable to create cache key from arguments for %s.", task.name
)
================================================
FILE: concordia/tests/test_fields.py
================================================
from unittest import mock
from urllib.error import HTTPError
from django.forms import ValidationError
from django.test import TestCase, override_settings
from concordia.turnstile.fields import TurnstileField
class TestFields(TestCase):
@override_settings(
TURNSTILE_PROXIES={},
TURNSTILE_SECRET="test-secret", # nosec B106: test-only dummy secret
TURNSTILE_VERIFY_URL="http://example.com",
TURNSTILE_TIMEOUT=0,
)
def test_TurnstileField(self):
with (
override_settings(
TURNSTILE_DEFAULT_CONFIG={"appearance": "interaction-only"}
),
mock.patch("concordia.turnstile.fields.Request"),
mock.patch("concordia.turnstile.fields.build_opener") as opener_mock,
):
open_mock = opener_mock.return_value.open
read_mock = open_mock.return_value.read
field = TurnstileField(required=False)
self.assertEqual(
field.widget_attrs(field.widget),
{"data-appearance": "interaction-only"},
)
# Successful validation from Turnstile
read_mock.return_value = '{"success" : true}'.encode()
self.assertEqual(field.validate("test-value"), None)
# Unsuccessful validation from Turnstile
read_mock.return_value = '{"test-key" : "test-value"}'.encode()
self.assertRaises(ValidationError, field.validate, "test-value")
# Error trying to contact Turnstile
open_mock.side_effect = HTTPError(
"http://example.com", 404, "Test message%", "", mock.MagicMock()
)
self.assertRaises(ValidationError, field.validate, "test-value")
# Testing special parameters on the widget
field = TurnstileField(
onload="testOnload()",
render="test-render",
hl="test-hl",
test_parameter="test-data",
)
self.assertEqual(
field.widget_attrs(field.widget),
{
"data-appearance": "interaction-only",
"data-test_parameter": "test-data",
},
)
self.assertEqual(
field.widget.extra_url,
{
"onload": "testOnload()",
"render": "test-render",
"hl": "test-hl",
},
)
================================================
FILE: concordia/tests/test_logging.py
================================================
import warnings
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from django.test import TestCase
from concordia.logging import ConcordiaLogger
class ConcordiaLoggerTests(TestCase):
def setUp(self):
self.mock_structlog_logger = MagicMock()
self.logger = ConcordiaLogger(self.mock_structlog_logger)
def test_debug_logs_with_event(self):
self.logger.debug("debug msg", event_code="debug_event", key1="value1")
self.mock_structlog_logger.debug.assert_called_once()
args, kwargs = self.mock_structlog_logger.debug.call_args
self.assertEqual(args[0], "debug msg")
self.assertEqual(kwargs["event_code"], "debug_event")
self.assertEqual(kwargs["key1"], "value1")
def test_info_logs_with_event(self):
self.logger.info("info msg", event_code="info_event", key2="value2")
self.mock_structlog_logger.info.assert_called_once()
args, kwargs = self.mock_structlog_logger.info.call_args
self.assertEqual(args[0], "info msg")
self.assertEqual(kwargs["event_code"], "info_event")
self.assertEqual(kwargs["key2"], "value2")
def test_warning_requires_reason_and_reason_code(self):
with self.assertRaises(TypeError):
self.logger.warning(
"warning msg", event_code="warn_event", reason="only_reason"
)
self.logger.warning(
"warning msg",
event_code="warn_event",
reason="test reason",
reason_code="warn_code",
key3="value3",
)
self.mock_structlog_logger.warning.assert_called_once()
args, kwargs = self.mock_structlog_logger.warning.call_args
self.assertEqual(kwargs["event_code"], "warn_event")
self.assertEqual(kwargs["reason"], "test reason")
self.assertEqual(kwargs["reason_code"], "warn_code")
self.assertEqual(kwargs["key3"], "value3")
def test_error_requires_reason_and_reason_code(self):
with self.assertRaises(TypeError):
self.logger.error(
"error msg", event_code="error_event", reason_code="only_code"
)
self.logger.error(
"error msg",
event_code="error_event",
reason="error reason",
reason_code="error_code",
)
self.mock_structlog_logger.error.assert_called_once()
args, kwargs = self.mock_structlog_logger.error.call_args
self.assertEqual(kwargs["event_code"], "error_event")
self.assertEqual(kwargs["reason"], "error reason")
self.assertEqual(kwargs["reason_code"], "error_code")
def test_missing_event_raises(self):
with self.assertRaises(ValueError):
self.logger.info("msg", event_code=None)
def test_log_merges_context_correctly(self):
mock_obj = SimpleNamespace(id=42)
self.logger.register_extractor("thing", lambda o: {"thing_id": o.id})
self.logger.info("msg", event_code="test_event", thing=mock_obj)
args, kwargs = self.mock_structlog_logger.info.call_args
self.assertEqual(kwargs["thing_id"], 42)
def test_log_explicit_key_overrides_extracted(self):
mock_obj = SimpleNamespace(id=42)
self.logger.register_extractor("thing", lambda o: {"thing_id": 123})
self.logger.info("msg", event_code="test_event", thing=mock_obj, thing_id=999)
args, kwargs = self.mock_structlog_logger.info.call_args
self.assertEqual(kwargs["thing_id"], 999)
def test_bind_merges_context(self):
bound = self.logger.bind(foo="bar")
self.assertIsInstance(bound, ConcordiaLogger)
self.assertIn("foo", bound._context)
self.assertEqual(bound._context["foo"], "bar")
def test_bind_merges_context_into_logging(self):
bound = self.logger.bind(user="uval")
bound.register_extractor("user", lambda o: {"user_id": o})
bound.info("msg", event_code="bound_event")
args, kwargs = self.mock_structlog_logger.info.call_args
self.assertEqual(kwargs["user_id"], "uval")
def test_unregister_extractor_removes_extractor(self):
self.logger.register_extractor("foo", lambda o: {"foo_id": 1})
self.logger.unregister_extractor("foo")
self.assertNotIn("foo", self.logger._extractors)
def test_register_extractor_warns_on_chained_override(self):
def fake_asset_extractor(x):
return {"asset_id": 1, "item_id": 2}
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
self.logger.register_extractor("asset", fake_asset_extractor)
self.assertTrue(
any(
"default extractors may still reference" in str(warn.message)
for warn in w
)
)
def test_log_raises_when_message_is_none(self):
with self.assertRaises(ValueError):
self.logger.info(None, event_code="event")
def test_log_raises_when_message_is_none_direct(self):
with self.assertRaises(ValueError):
self.logger.log("info", None, event_code="event")
def test_log_raises_when_message_is_empty(self):
with self.assertRaises(ValueError):
self.logger.info("", event_code="event")
def test_log_raises_when_message_is_empty_direct(self):
with self.assertRaises(ValueError):
self.logger.log("info", "", event_code="event")
def test_log_skips_none_values_from_extractor(self):
class Dummy:
def __init__(self):
self.id = None
self.logger.register_extractor("thing", lambda o: {"thing_id": o.id})
self.logger.info("msg", event_code="event", thing=Dummy())
args, kwargs = self.mock_structlog_logger.info.call_args
self.assertNotIn("thing_id", kwargs)
def test_log_includes_nonextractor_bound_context(self):
bound = self.logger.bind(extra1="foo", extra2="bar")
bound.info("msg", event_code="event")
args, kwargs = self.mock_structlog_logger.info.call_args
self.assertEqual(kwargs["extra1"], "foo")
self.assertEqual(kwargs["extra2"], "bar")
def test_log_skips_none_values_in_context(self):
self.logger.info("msg", event_code="event", explicit=None)
args, kwargs = self.mock_structlog_logger.info.call_args
self.assertNotIn("explicit", kwargs)
def test_log_overrides_bound_and_extracted_context(self):
obj = SimpleNamespace(id=123)
base = self.logger.bind(thing=obj, value="a")
base.register_extractor("thing", lambda o: {"thing_id": o.id, "value": "b"})
base.info("msg", event_code="event", value="c")
args, kwargs = self.mock_structlog_logger.info.call_args
# Extracted value ("b") overridden by explicit context ("c")
self.assertEqual(kwargs["value"], "c")
self.assertEqual(kwargs["thing_id"], 123)
def test_extractor_returns_none_value_skipped(self):
obj = SimpleNamespace()
self.logger.register_extractor("thing", lambda o: {"thing_id": None})
self.logger.info("msg", event_code="event", thing=obj)
args, kwargs = self.mock_structlog_logger.info.call_args
self.assertNotIn("thing_id", kwargs)
def test_get_logger_uses_structlog(self):
with patch("concordia.logging.structlog.get_logger") as mock_get_logger:
mock_logger_instance = MagicMock()
mock_get_logger.return_value = mock_logger_instance
logger = ConcordiaLogger.get_logger("concordia.tests")
mock_get_logger.assert_called_once_with("structlog.concordia.tests")
self.assertIsInstance(logger, ConcordiaLogger)
self.assertEqual(logger._logger, mock_logger_instance)
def test_log_raises_valueerror_for_empty_reason_and_code(self):
with self.assertRaises(ValueError):
self.logger.log(
"warning", "bad", event_code="something", reason="", reason_code="fail"
)
with self.assertRaises(ValueError):
self.logger.log(
"error", "bad", event_code="something", reason="fail", reason_code=None
)
def test_exception_logs_with_exc_info(self):
try:
raise ValueError("Something went wrong")
except ValueError:
self.logger.exception(
"Exception occurred",
event_code="test_exception",
reason="An error was raised",
reason_code="value_error",
extra="context",
)
self.mock_structlog_logger.error.assert_called_once()
args, kwargs = self.mock_structlog_logger.error.call_args
self.assertEqual(args[0], "Exception occurred")
self.assertEqual(kwargs["event_code"], "test_exception")
self.assertEqual(kwargs["reason"], "An error was raised")
self.assertEqual(kwargs["reason_code"], "value_error")
self.assertEqual(kwargs["extra"], "context")
self.assertTrue(kwargs.get("exc_info"))
================================================
FILE: concordia/tests/test_maintenance.py
================================================
from django.core.cache import cache
from django.test import RequestFactory, TestCase
from maintenance_mode.core import set_maintenance_mode
from concordia.maintenance import need_maintenance_response
from .utils import CreateTestUsers
class TestMaintenance(TestCase, CreateTestUsers):
def setUp(self):
self.request_factory = RequestFactory()
cache.clear()
def tearDown(self):
cache.clear()
def test_need_maintenance_response_maintenance_default(self):
request = self.request_factory.get("/")
self.assertFalse(need_maintenance_response(request))
def test_need_maintenance_response_maintenance_off(self):
set_maintenance_mode(False)
request = self.request_factory.get("/")
self.assertFalse(need_maintenance_response(request))
def test_need_maintenance_response_maintenance_on(self):
set_maintenance_mode(True)
request = self.request_factory.get("/")
self.assertTrue(need_maintenance_response(request))
request.user = self.create_test_user()
request.user.is_staff = True
# User is set and is staff, but frontend is off
# (the default) so they should still get a maintenance
# mode response
self.assertTrue(need_maintenance_response(request))
def test_need_maintenance_response_maintenance_frontend(self):
set_maintenance_mode(True)
request = self.request_factory.get("/")
request.user = self.create_test_user()
cache.set("maintenance_mode_frontend_available", True)
# User is set but isn't super user, so they should get
# a maintenance mode response
self.assertTrue(need_maintenance_response(request))
request.user.is_staff = True
# User is staff, so they shouldn't get a maintenance
# mode response
self.assertFalse(need_maintenance_response(request))
================================================
FILE: concordia/tests/test_management_commands.py
================================================
from io import StringIO
from unittest import mock
from django.core.management import call_command
from django.test import TestCase
from concordia.tests.utils import create_asset, create_campaign
class EnsureInitialSiteConfigurationTests(TestCase):
def test_command_output(self, *args, **kwargs):
out = StringIO()
call_command(
"ensure_initial_site_configuration", admin_email="admin@loc.gov", stdout=out
)
call_command(
"ensure_initial_site_configuration", site_domain="crowd.loc.gov", stdout=out
)
with mock.patch(
"django.contrib.sites.models.Site.objects.update"
) as update_mock:
update_mock.return_value = 0
call_command(
"ensure_initial_site_configuration",
site_domain="crowd.loc.gov",
stdout=out,
)
class ImportSiteReportsTests(TestCase):
def test_command_output(self, *args, **kwargs):
out = StringIO()
create_campaign(id=1)
call_command(
"import_site_reports",
csv_file="concordia/tests/data/site_reports.csv",
stdout=out,
)
class PrintFrontendTestUrlsTests(TestCase):
def test_command_output(self, *args, **kwargs):
out = StringIO()
call_command("print_frontend_test_urls", stdout=out)
self.assertIn("", out.getvalue())
create_asset()
call_command("print_frontend_test_urls", stdout=out)
self.assertIn("", out.getvalue())
================================================
FILE: concordia/tests/test_models.py
================================================
import json
from datetime import date, datetime, timedelta
from decimal import Decimal
from secrets import token_hex
from unittest import mock
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.db.models import signals
from django.test import TestCase
from django.utils import timezone
from concordia.models import (
Asset,
AssetTranscriptionReservation,
Campaign,
CardFamily,
ConcordiaUser,
HelpfulLink,
KeyMetricsReport,
MediaType,
NextReviewableCampaignAsset,
NextReviewableTopicAsset,
NextTranscribableCampaignAsset,
NextTranscribableTopicAsset,
SiteReport,
Topic,
Transcription,
TranscriptionStatus,
UserProfile,
UserProfileActivity,
_update_useractivity_cache,
resource_file_upload_path,
update_userprofileactivity_table,
validated_get_or_create,
)
from concordia.signals.handlers import create_user_profile, on_transcription_save
from concordia.utils import get_anonymous_user
from .utils import (
CreateTestUsers,
create_asset,
create_banner,
create_campaign,
create_campaign_retirement_progress,
create_card,
create_card_family,
create_carousel_slide,
create_concordia_file,
create_guide,
create_helpful_link,
create_item,
create_simple_page,
create_tag,
create_tag_collection,
create_topic,
create_transcription,
create_user_profile_activity,
)
class AssetTestCase(CreateTestUsers, TestCase):
def setUp(self):
self.asset = create_asset()
self.anon = get_anonymous_user()
create_transcription(asset=self.asset, user=self.anon)
create_transcription(
asset=self.asset,
user=self.create_test_user(username="tester"),
reviewed_by=self.anon,
)
def test_get_ocr_transcript(self):
self.asset.storage_image = "tests/test-european.jpg"
self.asset.save()
phrase = "marrón rápido salta sobre el perro"
self.assertFalse(phrase in self.asset.get_ocr_transcript())
self.assertFalse(
phrase in self.asset.get_ocr_transcript(language="bad-language-code")
)
self.assertTrue(phrase in self.asset.get_ocr_transcript(language="spa"))
def test_get_contributor_count(self):
self.assertEqual(self.asset.get_contributor_count(), 2)
def test_turn_off_ocr(self):
self.assertFalse(self.asset.turn_off_ocr())
self.asset.disable_ocr = True
self.asset.save()
self.assertTrue(self.asset.turn_off_ocr())
self.assertFalse(self.asset.item.turn_off_ocr())
self.asset.item.disable_ocr = True
self.asset.item.save()
self.assertTrue(self.asset.item.turn_off_ocr())
self.assertFalse(self.asset.item.project.turn_off_ocr())
self.asset.item.project.disable_ocr = True
self.asset.item.project.save()
self.assertTrue(self.asset.item.project.turn_off_ocr())
def test_get_storage_path(self):
self.assertEqual(
self.asset.get_storage_path(filename=self.asset.storage_image.name),
"test-campaign/test-project/testitem.0123456789/1.jpg",
)
def test_saving_without_campaign(self):
try:
Asset.objects.create(
item=self.asset.item,
title="No campaign",
slug="no-campaign",
media_type=MediaType.IMAGE,
storage_image="unittest1.jpg",
)
except (ValidationError, ObjectDoesNotExist):
self.fail("Creating an Asset without a campaign failed validation.")
def test_rollforward_with_only_rollforward_transcriptions(self):
asset = create_asset(slug="rollforward-test", item=self.asset.item)
create_transcription(asset=asset, user=self.anon, rolled_forward=True)
with self.assertRaisesMessage(
ValueError,
"Can not rollforward transcription on an asset with "
"no non-rollforward transcriptions",
):
asset.rollforward_transcription(self.anon)
def test_rollforward_with_too_many_rollforward_transcriptions(self):
asset = create_asset(slug="rollforward-test", item=self.asset.item)
transcription1 = create_transcription(asset=asset, user=self.anon)
create_transcription(
asset=asset, user=self.anon, supersedes=transcription1, rolled_forward=True
)
create_transcription(
asset=asset, user=self.anon, supersedes=transcription1, rolled_forward=True
)
with self.assertRaisesMessage(
ValueError,
"More rollforward transcription exist than non-roll-forward "
"transcriptions, which shouldn't be possible. Possibly "
"incorrectly modified transcriptions for this asset.",
):
asset.rollforward_transcription(self.anon)
def test_rollforward_with_no_superseded_transcription(self):
# This isn't a state that would happen normally, but could be created
# accidentally when manually editing transcription history
asset = create_asset(slug="rollforward-test", item=self.asset.item)
transcription1 = create_transcription(asset=asset, user=self.anon)
create_transcription(asset=asset, user=self.anon, supersedes=transcription1)
create_transcription(
asset=asset, user=self.anon, rolled_back=True, source=transcription1
)
with self.assertRaisesMessage(
ValueError,
"Can not rollforward transcription on an asset if the latest "
"rollback transcription did not supersede a previous transcription",
):
asset.rollforward_transcription(self.anon)
def test_get_storage_path_handles_jpeg(self):
# Ensure ".jpeg" is normalized to ".jpg"
expected = self.asset.get_asset_image_filename("jpg")
self.assertEqual(self.asset.get_storage_path("anything.jpeg"), expected)
class ItemModelTests(TestCase):
def test_thumbnail_link_prefers_image_url_when_present(self):
item = create_item()
class Img:
url = "http://example.test/media/thumb.jpg"
item.thumbnail_image = Img()
self.assertEqual(item.thumbnail_link, Img.url)
def test_thumbnail_link_falls_back_when_image_url_raises(self):
# If .url access raises ValueError, fall back to thumbnail_url
item = create_item()
class BadImg:
@property
def url(self):
raise ValueError("missing from storage")
item.thumbnail_image = BadImg()
item.thumbnail_url = "http://example.test/media/fallback.jpg"
self.assertEqual(item.thumbnail_link, item.thumbnail_url)
def test_thumbnail_link_returns_thumbnail_url_when_no_image(self):
item = create_item()
item.thumbnail_image = None
item.thumbnail_url = "http://example.test/media/fallback.jpg"
self.assertEqual(item.thumbnail_link, item.thumbnail_url)
def test_thumbnail_link_returns_none_when_no_image_or_url(self):
item = create_item()
item.thumbnail_image = None
item.thumbnail_url = None
self.assertIsNone(item.thumbnail_link)
class TranscriptionManagerTestCase(CreateTestUsers, TestCase):
def setUp(self):
self.transcription1 = create_transcription(
user=self.create_user(username="tester1"),
rejected=timezone.now() - timedelta(days=2),
)
self.transcription2 = create_transcription(
asset=self.transcription1.asset, user=get_anonymous_user()
)
def test_recent_review_actions(self):
transcriptions = Transcription.objects
self.assertEqual(transcriptions.recent_review_actions().count(), 0)
self.transcription1.accepted = timezone.now()
self.transcription1.save()
self.assertEqual(transcriptions.recent_review_actions().count(), 1)
self.transcription2.rejected = timezone.now()
self.transcription2.save()
self.assertEqual(transcriptions.recent_review_actions().count(), 2)
def test_review_actions(self):
start = timezone.now() - timedelta(days=5)
end = timezone.now() - timedelta(days=1)
self.assertEqual(Transcription.objects.review_actions(start, end).count(), 1)
def test_review_incidents(self):
self.transcription1.accepted = timezone.now()
self.transcription1.reviewed_by = self.create_user(username="tester2")
self.transcription1.save()
self.transcription2.accepted = self.transcription1.accepted + timedelta(
seconds=29
)
self.transcription2.reviewed_by = self.transcription1.reviewed_by
self.transcription2.save()
create_transcription(
asset=self.transcription1.asset,
user=self.transcription1.user,
reviewed_by=self.transcription1.reviewed_by,
rejected=self.transcription2.accepted + timedelta(seconds=29),
)
create_transcription(
asset=self.transcription1.asset,
user=self.transcription1.user,
reviewed_by=self.transcription1.reviewed_by,
rejected=self.transcription2.accepted + timedelta(seconds=58),
)
users = Transcription.objects.review_incidents()
self.assertNotIn(self.transcription1.user.id, users)
transcription3 = create_transcription(
asset=self.transcription1.asset,
user=self.transcription1.user,
reviewed_by=self.transcription1.reviewed_by,
accepted=self.transcription1.accepted + timedelta(seconds=58),
)
transcription4 = create_transcription(
asset=self.transcription1.asset,
user=self.transcription1.user,
reviewed_by=self.transcription1.reviewed_by,
accepted=transcription3.accepted + timedelta(minutes=1, seconds=1),
)
users = Transcription.objects.review_incidents()
self.assertEqual(len(users), 1)
self.assertEqual(
users[0],
(
self.transcription1.reviewed_by.id,
self.transcription1.reviewed_by.username,
2,
4,
),
)
create_transcription(
asset=self.transcription1.asset,
user=self.transcription1.user,
reviewed_by=self.transcription1.reviewed_by,
accepted=transcription4.accepted + timedelta(seconds=29),
)
create_transcription(
asset=self.transcription1.asset,
user=self.transcription1.user,
reviewed_by=self.transcription1.reviewed_by,
accepted=transcription4.accepted + timedelta(seconds=58),
)
users = Transcription.objects.review_incidents()
self.assertEqual(len(users), 1)
self.assertEqual(
users[0],
(
self.transcription1.reviewed_by.id,
self.transcription1.reviewed_by.username,
4,
6,
),
)
def test_transcribe_incidents(self):
self.transcription1.submitted = timezone.now()
self.transcription1.save()
self.transcription2.submitted = self.transcription1.submitted + timedelta(
seconds=29
)
self.transcription2.user = self.transcription1.user
self.transcription2.save()
users = Transcription.objects.transcribe_incidents()
self.assertEqual(len(users), 0)
self.assertNotIn(self.transcription1.user.id, users)
transcription3 = create_transcription(
asset=create_asset(slug="asset-two", item=self.transcription1.asset.item),
user=self.transcription1.user,
submitted=self.transcription1.submitted + timedelta(seconds=58),
)
transcription4 = create_transcription(
asset=create_asset(slug="asset-three", item=self.transcription1.asset.item),
user=self.transcription1.user,
submitted=transcription3.submitted + timedelta(minutes=1, seconds=1),
)
create_transcription(
asset=transcription4.asset,
user=self.transcription1.user,
submitted=transcription4.submitted + timedelta(seconds=59),
)
users = Transcription.objects.transcribe_incidents()
self.assertEqual(len(users), 1)
self.assertEqual(
users[0],
(self.transcription1.user.id, self.transcription1.user.username, 2, 5),
)
create_transcription(
asset=create_asset(slug="asset-five", item=self.transcription1.asset.item),
user=self.transcription1.user,
submitted=self.transcription1.submitted + timedelta(minutes=1, seconds=59),
)
users = Transcription.objects.transcribe_incidents()
self.assertEqual(len(users), 1)
self.assertEqual(
users[0],
(self.transcription1.user.id, self.transcription1.user.username, 3, 6),
)
def test_review_incidents_returns_empty_when_counts_zero(self):
reviewer = self.create_user(username="rev-zero")
asset = self.transcription1.asset
t1 = create_transcription(
asset=asset,
user=self.create_user(username="u-a"),
reviewed_by=reviewer,
accepted=timezone.now() - timedelta(minutes=5),
)
create_transcription(
asset=asset,
user=self.create_user(username="u-b"),
reviewed_by=reviewer,
accepted=t1.accepted + timedelta(seconds=61),
)
out = Transcription.objects.review_incidents()
self.assertEqual(out, [])
def test_user_review_incidents_no_threshold_hit(self):
asset = self.transcription1.asset
reviewer = self.create_user("reviewer-1")
reviewer_proxy = ConcordiaUser.objects.get(pk=reviewer.pk)
base = timezone.now()
create_transcription(
asset=asset,
user=self.create_user("ri_u1"),
reviewed_by=reviewer_proxy,
accepted=base,
)
create_transcription(
asset=asset,
user=self.create_user("ri_u2"),
reviewed_by=reviewer_proxy,
accepted=base + timedelta(seconds=61),
)
recent = Transcription.objects.filter(accepted__isnull=False)
incidents = reviewer_proxy.review_incidents(recent)
self.assertEqual(incidents, 0)
def test_review_incidents_no_threshold_match_inner_loop_break(self):
# Two accepts for same reviewer but >60s apart:
a1 = create_asset(slug="rev-gap-a1", item=self.transcription1.asset.item)
a2 = create_asset(slug="rev-gap-a2", item=a1.item)
reviewer = self.create_user("reviewer-1")
t0 = timezone.now()
create_transcription(
asset=a1, user=self.create_user("u1"), reviewed_by=reviewer, accepted=t0
)
create_transcription(
asset=a2,
user=self.create_user("u2"),
reviewed_by=reviewer,
accepted=t0 + timedelta(seconds=61),
)
recent = Transcription.objects.filter(accepted__isnull=False)
reviewer_proxy = ConcordiaUser.objects.get(pk=reviewer.pk)
incidents = reviewer_proxy.review_incidents(recent)
self.assertEqual(incidents, 0)
def test_review_incidents_loops_until_threshold(self):
reviewer = self.create_user(username="test-reviewer-1")
# Three accepts within 60s so threshold=3 will require two inner
# iterations (count goes 1->2, not equal to threshold, then 2->3)
base = timezone.now()
create_transcription(
asset=self.transcription1.asset,
user=self.transcription1.user,
reviewed_by=reviewer,
accepted=base,
)
create_transcription(
asset=self.transcription1.asset,
user=self.transcription1.user,
reviewed_by=reviewer,
accepted=base + timedelta(seconds=20),
)
create_transcription(
asset=self.transcription1.asset,
user=self.transcription1.user,
reviewed_by=reviewer,
accepted=base + timedelta(seconds=40),
)
recent_accepts = Transcription.objects.filter(
accepted__gte=base - timedelta(seconds=1)
)
reviewer_proxy = ConcordiaUser.objects.get(id=reviewer.id)
incidents = reviewer_proxy.review_incidents(
recent_accepts=recent_accepts, threshold=3
)
self.assertEqual(incidents, 1)
class TranscriptionTestCase(CreateTestUsers, TestCase):
def setUp(self):
self.user = self.create_user("test-user-1")
self.user2 = self.create_user("test-user-2")
self.asset = create_asset()
self.transcription1 = create_transcription(
user=self.user,
asset=self.asset,
rejected=timezone.now() - timedelta(days=2),
)
self.transcription2 = create_transcription(asset=self.asset, user=self.user2)
def test_campaign_slug(self):
self.assertEqual(
self.asset.item.project.campaign.slug, self.transcription1.campaign_slug()
)
def test_clean(self):
bad_transcription = Transcription(asset=self.asset, user=self.user)
bad_transcription.clean()
bad_transcription2 = Transcription(
asset=self.asset,
user=self.user,
reviewed_by=self.user,
accepted=timezone.now(),
)
with self.assertRaises(ValidationError):
bad_transcription2.clean()
bad_transcription3 = Transcription(
asset=self.asset,
user=self.user,
reviewed_by=self.user2,
accepted=timezone.now(),
rejected=timezone.now(),
)
with self.assertRaises(ValidationError):
bad_transcription3.clean()
@mock.patch("concordia.tests.test_models.on_transcription_save")
def test_save(self, mock_handler):
signals.post_save.connect(on_transcription_save, sender=Transcription)
transcription = create_transcription(asset=self.asset)
self.assertTrue(mock_handler.called)
self.assertEqual(mock_handler.call_count, 1)
transcription.save()
self.assertEqual(mock_handler.call_count, 2)
signals.post_save.disconnect(on_transcription_save, sender=Transcription)
def test_status(self):
transcription = create_transcription(user=self.user, asset=self.asset)
self.assertEqual(
transcription.status,
TranscriptionStatus.CHOICE_MAP[TranscriptionStatus.IN_PROGRESS],
)
transcription2 = create_transcription(
asset=transcription.asset, user=self.user, submitted=timezone.now()
)
self.assertEqual(
transcription2.status,
TranscriptionStatus.CHOICE_MAP[TranscriptionStatus.SUBMITTED],
)
transcription3 = create_transcription(
asset=transcription.asset,
user=self.user,
reviewed_by=self.user2,
accepted=timezone.now(),
)
self.assertEqual(
transcription3.status,
TranscriptionStatus.CHOICE_MAP[TranscriptionStatus.COMPLETED],
)
class SignalHandlersTest(CreateTestUsers, TestCase):
@mock.patch("django.core.cache.cache.get")
@mock.patch("django.core.cache.cache.set")
def test_update_useractivity_cache(self, mock_set, mock_get):
campaign = create_campaign()
user = self.create_test_user()
mock_get.return_value = {}
_update_useractivity_cache(user.id, campaign.id, "transcribe")
self.assertEqual(mock_set.call_count, 1)
expected_key = f"userprofileactivity_{campaign.pk}"
expected_value = {user.id: (1, 0)}
mock_set.assert_called_with(expected_key, expected_value, timeout=None)
reviewed_by = self.create_test_user(username="testuser2")
mock_get.return_value = {}
_update_useractivity_cache(reviewed_by.id, campaign.id, "review")
self.assertEqual(mock_set.call_count, 2)
expected_value = {reviewed_by.id: (0, 1)}
mock_set.assert_called_with(expected_key, expected_value, timeout=None)
class AssetTranscriptionReservationTest(CreateTestUsers, TestCase):
def setUp(self):
self.asset = create_asset()
self.user = self.create_user("test-user")
self.uid = str(self.user.id).zfill(6)
self.token = token_hex(22)
self.reservation_token = self.token + self.uid
self.reservation = AssetTranscriptionReservation.objects.create(
asset=self.asset, reservation_token=self.reservation_token
)
def test_get_token(self):
self.assertEqual(self.reservation.get_token(), self.token)
def test_get_user(self):
self.assertEqual(self.reservation.get_user(), self.uid)
class UserProfileActivityTestCase(TestCase):
def setUp(self):
self.user_profile_activity = UserProfileActivity(
campaign=Campaign(), transcribe_count=135, review_count=204
)
def test_get_status(self):
self.user_profile_activity.campaign.status = Campaign.Status.ACTIVE
self.assertEqual(self.user_profile_activity.get_status(), "Active")
self.user_profile_activity.campaign.status = Campaign.Status.COMPLETED
self.assertEqual(self.user_profile_activity.get_status(), "Completed")
self.user_profile_activity.campaign.status = Campaign.Status.RETIRED
self.assertEqual(self.user_profile_activity.get_status(), "Retired")
def test_total_actions(self):
self.assertEqual(self.user_profile_activity.total_actions(), 339)
def test_str(self):
activity = create_user_profile_activity()
self.assertEqual(f"{activity.user} - {activity.campaign}", str(activity))
class UserProfileTestCase(CreateTestUsers, TestCase):
def test_update_userprofileactivity_table(self):
signals.post_save.disconnect(
create_user_profile, sender=settings.AUTH_USER_MODEL
)
user = self.create_test_user()
self.assertFalse(hasattr(user, "profile"))
transcription = create_transcription(user=user)
update_userprofileactivity_table(
user, transcription.asset.item.project.campaign.id, "transcribe_count"
)
self.assertTrue(hasattr(user, "profile"))
self.assertEqual(user.profile.transcribe_count, 1)
signals.post_save.connect(create_user_profile, sender=settings.AUTH_USER_MODEL)
def test_update_userprofileactivity_table_updates_existing_and_profile(self):
# Avoid auto-profile creation so we control both branches
signals.post_save.disconnect(
create_user_profile, sender=settings.AUTH_USER_MODEL
)
user = self.create_test_user()
UserProfile.objects.create(user=user)
transcription = create_transcription(user=user)
campaign = transcription.asset.item.project.campaign
upa, _ = UserProfileActivity.objects.get_or_create(
user=user, campaign=campaign, defaults={"transcribe_count": 1}
)
update_userprofileactivity_table(user, campaign.id, "transcribe_count")
# F() increments apply on save; refresh to observe DB values
upa.refresh_from_db()
user.refresh_from_db()
user.profile.refresh_from_db()
self.assertEqual(upa.transcribe_count, 2)
self.assertEqual(user.profile.transcribe_count, 1)
signals.post_save.connect(create_user_profile, sender=settings.AUTH_USER_MODEL)
class CampaignTestCase(TestCase):
def test_queryset(self):
campaign = create_campaign(unlisted=True)
self.assertIn(campaign, Campaign.objects.unlisted())
campaign.status = Campaign.Status.COMPLETED
campaign.save()
self.assertIn(campaign, Campaign.objects.completed())
campaign.status = Campaign.Status.RETIRED
campaign.save()
self.assertIn(campaign, Campaign.objects.retired())
class CardTestCase(TestCase):
def test_str(self):
card = create_card()
self.assertEqual(card.title, str(card))
class CardFamilyTestCase(TestCase):
def setUp(self):
self.family1 = create_card_family(default=True)
def test_str(self):
self.assertEqual(self.family1.slug, str(self.family1))
def test_on_cardfamily_save(self):
with mock.patch("concordia.models.on_cardfamily_save") as mocked_handler:
signals.post_save.connect(mocked_handler, sender=CardFamily)
self.family1.save()
self.assertTrue(mocked_handler.called)
self.assertEqual(mocked_handler.call_count, 1)
class HelpfulLinkTestCase(TestCase):
def setUp(self):
self.helpful_link = create_helpful_link()
def test_str(self):
self.assertEqual(self.helpful_link.title, str(self.helpful_link))
def test_queryset(self):
self.assertEqual(HelpfulLink.objects.related_links().count(), 1)
create_helpful_link(
link_type=HelpfulLink.HelpfulLinkType.COMPLETED_TRANSCRIPTION_LINK
)
self.assertEqual(HelpfulLink.objects.completed_transcription_links().count(), 1)
class ConcordiaFileTestCase(TestCase):
def setUp(self):
self.concordia_file = create_concordia_file()
def test_str(self):
self.assertEqual(self.concordia_file.name, str(self.concordia_file))
def test_delete(self):
with (
mock.patch.object(
self.concordia_file.uploaded_file, "delete"
) as delete_mock,
mock.patch.object(
self.concordia_file.uploaded_file, "storage", autospec=True
) as storage_mock,
):
storage_mock.exists.return_value = True
self.concordia_file.delete()
self.assertTrue(delete_mock.called)
concordia_file2 = create_concordia_file()
with (
mock.patch.object(concordia_file2.uploaded_file, "delete") as delete_mock,
mock.patch.object(
concordia_file2.uploaded_file, "storage", autospec=True
) as storage_mock,
):
storage_mock.exists.return_value = False
concordia_file2.delete()
self.assertFalse(delete_mock.called)
def test_concordia_file_upload_path(self):
current_year = date.today().year
path = resource_file_upload_path(self.concordia_file, "SHOULDNTBEUSED.PDF")
self.assertEqual(path, "file.pdf")
self.concordia_file.path = None
path = resource_file_upload_path(self.concordia_file, "TEST.PDF")
self.assertEqual(path, f"cm-uploads/resources/{current_year}/test.pdf")
path = resource_file_upload_path(self.concordia_file, "TEST%%s.PDF")
self.assertEqual(path, f"cm-uploads/resources/{current_year}/test%s.pdf")
path = resource_file_upload_path(self.concordia_file, "%%YTEST.PDF")
self.assertEqual(path, f"cm-uploads/resources/{current_year}/%ytest.pdf")
class TagTestCase(TestCase):
def test_str(self):
tag = create_tag()
self.assertEqual(tag.value, str(tag))
class UserAssetTagCollectionTestCase(TestCase):
def test_str(self):
tag_collection = create_tag_collection()
self.assertEqual(
"{} - {}".format(tag_collection.asset, tag_collection.user),
str(tag_collection),
)
class BannerTestCase(TestCase):
def setUp(self):
self.banner = create_banner()
def test_str(self):
self.assertEqual(f"Banner: {self.banner.slug}", str(self.banner))
def test_alert_class(self):
self.assertEqual(
self.banner.alert_class(), "alert-" + self.banner.alert_status.lower()
)
def test_btn_class(self):
self.assertEqual(
self.banner.btn_class(), "btn-" + self.banner.alert_status.lower()
)
class CarouselSlideTestCase(TestCase):
def test_str(self):
slide = create_carousel_slide()
self.assertEqual(f"CarouselSlide: {slide.headline}", str(slide))
class CampaignRetirementProgressTestCase(TestCase):
def test_str(self):
progress = create_campaign_retirement_progress()
self.assertEqual(f"Removal progress for {progress.campaign}", str(progress))
class GuideTestCase(TestCase):
def test_str(self):
guide = create_guide()
self.assertEqual(guide.title, str(guide))
class SimplePageTestCase(TestCase):
def test_str(self):
simple_page = create_simple_page()
self.assertEqual(f"SimplePage: {simple_page.path}", str(simple_page))
class ValidatedGetOrCreateTestCase(TestCase):
def test_validated_get_or_create(self):
kwargs = {
"title": "Test Campaign",
"slug": "test-campaign",
}
campaign, created = validated_get_or_create(Campaign, **kwargs)
self.assertTrue(created)
campaign, created = validated_get_or_create(Campaign, **kwargs)
self.assertFalse(created)
self.assertEqual(campaign.title, kwargs["title"])
class NextAssetModelTests(TestCase):
def setUp(self):
self.asset = create_asset()
self.topic = create_topic(project=self.asset.item.project)
self.campaign = self.asset.campaign
self.project = self.asset.item.project
def test_create_next_transcribable_campaign_asset(self):
obj = NextTranscribableCampaignAsset.objects.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.project,
project_slug=self.project.slug,
sequence=self.asset.sequence,
campaign=self.campaign,
)
self.assertEqual(str(obj), self.asset.title)
self.assertEqual(obj.transcription_status, "not_started")
def test_create_next_reviewable_campaign_asset(self):
obj = NextReviewableCampaignAsset.objects.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.project,
project_slug=self.project.slug,
sequence=self.asset.sequence,
campaign=self.campaign,
)
self.assertEqual(str(obj), self.asset.title)
self.assertEqual(obj.transcriber_ids, [])
def test_create_next_transcribable_topic_asset(self):
obj = NextTranscribableTopicAsset.objects.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.project,
project_slug=self.project.slug,
sequence=self.asset.sequence,
topic=self.topic,
)
self.assertEqual(obj.transcription_status, "not_started")
def test_create_next_reviewable_topic_asset(self):
obj = NextReviewableTopicAsset.objects.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.project,
project_slug=self.project.slug,
sequence=self.asset.sequence,
topic=self.topic,
)
self.assertEqual(obj.transcriber_ids, [])
def test_needed_for_campaign_respects_target_count(self):
manager = NextTranscribableCampaignAsset.objects
current_needed = manager.needed_for_campaign(self.campaign.id)
self.assertEqual(current_needed, settings.NEXT_TRANSCRIBABE_ASSET_COUNT)
# Add one and check count again
manager.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.project,
project_slug=self.project.slug,
sequence=self.asset.sequence,
campaign=self.campaign,
)
new_needed = manager.needed_for_campaign(self.campaign.id)
self.assertEqual(new_needed, settings.NEXT_TRANSCRIBABE_ASSET_COUNT - 1)
def test_needed_for_topic_respects_target_count(self):
manager = NextReviewableTopicAsset.objects
current_needed = manager.needed_for_topic(self.topic.id)
self.assertEqual(current_needed, settings.NEXT_REVIEWABLE_ASSET_COUNT)
manager.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.project,
project_slug=self.project.slug,
sequence=self.asset.sequence,
topic=self.topic,
)
new_needed = manager.needed_for_topic(self.topic.id)
self.assertEqual(new_needed, settings.NEXT_REVIEWABLE_ASSET_COUNT - 1)
def test_needed_for_campaign_raises_without_target(self):
from django.db import models
from concordia.models import NextCampaignAssetManager
class DummyManager(NextCampaignAssetManager):
target_count = None
class DummyModel(models.Model):
campaign = models.ForeignKey("concordia.Campaign", on_delete=models.CASCADE)
objects = DummyManager()
class Meta:
app_label = "concordia"
with self.assertRaises(NotImplementedError):
DummyModel.objects.needed_for_campaign(self.campaign.id)
def test_needed_for_topic_raises_without_target(self):
from django.db import models
from concordia.models import NextTopicAssetManager
class DummyManager(NextTopicAssetManager):
target_count = None
class DummyModel(models.Model):
topic = models.ForeignKey("concordia.Topic", on_delete=models.CASCADE)
objects = DummyManager()
class Meta:
app_label = "concordia"
with self.assertRaises(NotImplementedError):
DummyModel.objects.needed_for_topic(self.topic.id)
def test_needed_for_campaign_with_explicit_target_count(self):
manager = NextTranscribableCampaignAsset.objects
# Should return full count when no assets exist yet
needed = manager.needed_for_campaign(self.campaign.id, target_count=10)
self.assertEqual(needed, 10)
# Add one asset
manager.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.project,
project_slug=self.project.slug,
sequence=self.asset.sequence,
campaign=self.campaign,
)
needed = manager.needed_for_campaign(self.campaign.id, target_count=10)
self.assertEqual(needed, 9)
def test_needed_for_topic_with_explicit_target_count(self):
manager = NextReviewableTopicAsset.objects
needed = manager.needed_for_topic(self.topic.id, target_count=5)
self.assertEqual(needed, 5)
manager.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.project,
project_slug=self.project.slug,
sequence=self.asset.sequence,
topic=self.topic,
)
needed = manager.needed_for_topic(self.topic.id, target_count=5)
self.assertEqual(needed, 4)
class SiteReportAndManagerTestCase(TestCase):
def _aware(self, y, m, d, hh=12, mm=0, ss=0):
tz = timezone.get_current_timezone()
return timezone.make_aware(datetime(y, m, d, hh, mm, ss), tz)
def _mk_sr(
self,
*,
dt,
report_name=None,
campaign=None,
topic=None,
**kwargs,
):
sr = SiteReport.objects.create(
report_name=report_name or "",
campaign=campaign,
topic=topic,
**kwargs,
)
# Set created_on deterministically for ordering logic
SiteReport.objects.filter(pk=sr.pk).update(created_on=dt)
return SiteReport.objects.get(pk=sr.pk)
def test_calculate_assets_started(self):
# Uses (assets_total - assets_not_started) deltas; floor at 0.
v = SiteReport.calculate_assets_started(
previous_assets_total=100,
previous_assets_not_started=100,
current_assets_total=107,
current_assets_not_started=92,
)
self.assertEqual(v, 15)
# None treated as 0.
v2 = SiteReport.calculate_assets_started(
previous_assets_total=None,
previous_assets_not_started=None,
current_assets_total=200,
current_assets_not_started=190,
)
self.assertEqual(v2, 10)
# Negative deltas are floored at 0.
v3 = SiteReport.calculate_assets_started(
previous_assets_total=107,
previous_assets_not_started=92,
current_assets_total=100,
current_assets_not_started=90,
)
self.assertEqual(v3, 0)
def test_series_navigation_and_sums(self):
# Site-wide TOTAL series snapshots across three days
d1 = self._aware(2024, 1, 10)
d2 = self._aware(2024, 1, 20)
d3 = self._aware(2024, 1, 31)
r1 = self._mk_sr(
dt=d1,
report_name=SiteReport.ReportName.TOTAL,
assets_started=3,
)
r2 = self._mk_sr(
dt=d2,
report_name=SiteReport.ReportName.TOTAL,
assets_started=7,
)
r3 = self._mk_sr(
dt=d3,
report_name=SiteReport.ReportName.TOTAL,
assets_started=10,
)
prev = SiteReport.objects.previous_in_series(
report_name=SiteReport.ReportName.TOTAL,
before=self._aware(2024, 1, 25),
)
self.assertEqual(prev.pk, r2.pk)
# last_on_or_before_date_for_series
last = SiteReport.objects.last_on_or_before_date_for_series(
report_name=SiteReport.ReportName.TOTAL,
on_or_before_date=date(2024, 1, 31),
)
self.assertEqual(last.pk, r3.pk)
first = SiteReport.objects.first_on_or_after_date_for_series(
report_name=SiteReport.ReportName.TOTAL,
on_or_after_date=date(2024, 1, 15),
on_or_before_date=date(2024, 1, 29),
)
self.assertEqual(first.pk, r2.pk)
self.assertEqual(r2.previous_in_series().pk, r1.pk)
self.assertEqual(r2.next_in_series().pk, r3.pk)
summed = SiteReport.objects.sum_assets_started_for_series_between_dates(
report_name=SiteReport.ReportName.TOTAL,
start_date=date(2024, 1, 1),
end_date=date(2024, 1, 31),
)
self.assertEqual(summed, 3 + 7 + 10)
def test_per_campaign_and_topic_series_filters(self):
camp = Campaign.objects.create(title="C1", slug="c1")
# Per-campaign series
d1 = self._aware(2023, 12, 1)
d2 = self._aware(2023, 12, 2)
s1 = self._mk_sr(dt=d1, campaign=camp, assets_total=1)
s2 = self._mk_sr(dt=d2, campaign=camp, assets_total=2)
prev = SiteReport.objects.previous_for_instance(s2)
nxt = SiteReport.objects.next_for_instance(s1)
self.assertEqual(prev.pk, s1.pk)
self.assertEqual(nxt.pk, s2.pk)
# Unspecified series (fallback), ensure no crash and no result
none_prev = SiteReport.objects.previous_in_series()
self.assertIsNone(none_prev)
def test__series_filter_campaign_branch(self):
camp = Campaign.objects.create(title="C", slug="c")
# Two rows in same per-campaign series
s1 = SiteReport.objects.create(campaign=camp, assets_total=1)
s2 = SiteReport.objects.create(campaign=camp, assets_total=2)
# Force a deterministic order
SiteReport.objects.filter(pk=s1.pk).update(created_on=self._aware(2024, 1, 1))
SiteReport.objects.filter(pk=s2.pk).update(created_on=self._aware(2024, 1, 2))
prev = SiteReport.objects.previous_in_series(
campaign=camp, before=self._aware(2024, 1, 3)
)
self.assertEqual(prev.pk, s2.pk)
# And last_on_or_before path also using campaign filter
last = SiteReport.objects.last_on_or_before_date_for_series(
campaign=camp, on_or_before_date=date(2024, 1, 2)
)
self.assertEqual(last.pk, s2.pk)
def test__series_filter_topic_branch(self):
topic = Topic.objects.create(title="T", slug="t")
s1 = SiteReport.objects.create(topic=topic, assets_total=1)
s2 = SiteReport.objects.create(topic=topic, assets_total=2)
SiteReport.objects.filter(pk=s1.pk).update(created_on=self._aware(2024, 2, 1))
SiteReport.objects.filter(pk=s2.pk).update(created_on=self._aware(2024, 2, 2))
first = SiteReport.objects.first_on_or_after_date_for_series(
topic=topic,
on_or_after_date=date(2024, 2, 1),
on_or_before_date=date(2024, 2, 5),
)
self.assertEqual(first.pk, s1.pk)
def test_series_filter_for_instance_topic_branch(self):
topic = Topic.objects.create(title="T2", slug="t2")
a = SiteReport.objects.create(topic=topic, assets_total=10)
b = SiteReport.objects.create(topic=topic, assets_total=20)
SiteReport.objects.filter(pk=a.pk).update(created_on=self._aware(2024, 3, 1))
SiteReport.objects.filter(pk=b.pk).update(created_on=self._aware(2024, 3, 2))
# IMPORTANT: refresh to pick up the updated created_on values
a.refresh_from_db()
b.refresh_from_db()
self.assertEqual(b.previous_in_series().pk, a.pk)
self.assertEqual(a.next_in_series().pk, b.pk)
def test_series_filter_for_instance_retired_and_fallback(self):
r = SiteReport.objects.create(
report_name=SiteReport.ReportName.RETIRED_TOTAL, assets_total=1
)
# With only a single row, previous/next resolve via the RETIRED series Q()
self.assertIsNone(r.previous_in_series())
self.assertIsNone(r.next_in_series())
blank = SiteReport.objects.create(assets_total=3) # report_name=""
self.assertIsNone(blank.previous_in_series())
self.assertIsNone(blank.next_in_series())
def test_to_debug_dict_includes_related_fields_and_counters(self):
camp = Campaign.objects.create(title="CTitle", slug="cslug")
topic = Topic.objects.create(title="TTitle", slug="tslug")
sr_campaign = SiteReport.objects.create(
campaign=camp, assets_total=9, assets_published=3
)
sr_topic = SiteReport.objects.create(
topic=topic, items_published=4, items_unpublished=1
)
d1 = sr_campaign.to_debug_dict()
self.assertIn("campaign", d1)
self.assertEqual(d1["campaign"]["id"], camp.id)
self.assertEqual(d1["campaign"]["title"], "CTitle")
self.assertEqual(d1["campaign"]["slug"], "cslug")
self.assertIn("counters", d1)
self.assertEqual(d1["counters"]["assets_total"], 9)
self.assertEqual(d1["counters"]["assets_published"], 3)
d2 = sr_topic.to_debug_dict()
self.assertIn("topic", d2)
self.assertEqual(d2["topic"]["id"], topic.id)
self.assertEqual(d2["topic"]["title"], "TTitle")
self.assertEqual(d2["topic"]["slug"], "tslug")
self.assertEqual(d2["counters"]["items_published"], 4)
self.assertEqual(d2["counters"]["items_unpublished"], 1)
def test_first_on_or_after_with_upper_bound_campaign(self):
camp = Campaign.objects.create(title="C-Bound", slug="c-bound")
a = SiteReport.objects.create(campaign=camp)
b = SiteReport.objects.create(campaign=camp)
SiteReport.objects.filter(pk=a.pk).update(created_on=self._aware(2024, 5, 1))
SiteReport.objects.filter(pk=b.pk).update(created_on=self._aware(2024, 5, 2))
out = SiteReport.objects.first_on_or_after_date_for_series(
campaign=camp,
on_or_after_date=date(2024, 5, 1),
on_or_before_date=date(2024, 5, 1),
)
self.assertEqual(out.pk, a.pk)
def test_previous_in_series_defaults_to_now(self):
r1 = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
r2 = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=r1.pk).update(created_on=self._aware(2024, 1, 10))
SiteReport.objects.filter(pk=r2.pk).update(created_on=self._aware(2024, 1, 20))
out = SiteReport.objects.previous_in_series(
report_name=SiteReport.ReportName.TOTAL
)
self.assertEqual(out.pk, r2.pk)
def test_sum_assets_started_treats_null_as_zero(self):
r1 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL, assets_started=None
)
r2 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL, assets_started=None
)
SiteReport.objects.filter(pk=r1.pk).update(created_on=self._aware(2024, 2, 1))
SiteReport.objects.filter(pk=r2.pk).update(created_on=self._aware(2024, 2, 2))
total = SiteReport.objects.sum_assets_started_for_series_between_dates(
report_name=SiteReport.ReportName.TOTAL,
start_date=date(2024, 2, 1),
end_date=date(2024, 2, 28),
)
self.assertEqual(total, 0)
def test_to_debug_dict_campaign_status_and_topic_loop(self):
camp = Campaign.objects.create(
title="Camp", slug="camp"
) # status has a default
topic = Topic.objects.create(title="Top", slug="top")
sr_campaign = SiteReport.objects.create(campaign=camp, assets_total=1)
sr_topic = SiteReport.objects.create(topic=topic, assets_total=2)
d1 = sr_campaign.to_debug_dict()
self.assertIn("campaign", d1)
# ensure the loop includes all three fields, including status
self.assertEqual(d1["campaign"]["title"], "Camp")
self.assertEqual(d1["campaign"]["slug"], "camp")
self.assertIn("status", d1["campaign"])
d2 = sr_topic.to_debug_dict()
self.assertIn("topic", d2)
# ensure the loop includes both fields for topic
self.assertEqual(d2["topic"]["title"], "Top")
self.assertEqual(d2["topic"]["slug"], "top")
def test_to_debug_json_serializes_and_includes_counters(self):
camp = Campaign.objects.create(title="CJ", slug="cj")
sr = SiteReport.objects.create(
campaign=camp, assets_total=4, assets_published=2
)
out = sr.to_debug_json()
parsed = json.loads(out)
# basic shape checks
self.assertIn("created_on", parsed) # ISO string
self.assertEqual(parsed["report_name"], "")
self.assertEqual(parsed["campaign"]["id"], camp.id)
# counters included and numeric values preserved
self.assertEqual(parsed["counters"]["assets_total"], 4)
self.assertEqual(parsed["counters"]["assets_published"], 2)
def test_first_on_or_after_without_upper_bound_topic(self):
# Create two topic reports; query without an upper bound s
# hould still return the earliest on/after.
topic = Topic.objects.create(title="UBT", slug="ubt")
s1 = SiteReport.objects.create(topic=topic)
s2 = SiteReport.objects.create(topic=topic)
SiteReport.objects.filter(pk=s1.pk).update(created_on=self._aware(2024, 6, 1))
SiteReport.objects.filter(pk=s2.pk).update(created_on=self._aware(2024, 6, 2))
out = SiteReport.objects.first_on_or_after_date_for_series(
topic=topic,
on_or_after_date=date(2024, 6, 2),
# no on_or_before_date here on purpose
)
self.assertEqual(out.pk, s2.pk)
def test_to_debug_dict_skips_none_campaign_attrs(self):
# Force the related-object cache to a stub that lacks some attrs
from types import SimpleNamespace
camp = Campaign.objects.create(title="C", slug="c")
sr = SiteReport.objects.create(campaign=camp, assets_total=1)
# Populate fields_cache so descriptor returns this stub instead of hitting DB
sr._state.fields_cache["campaign"] = SimpleNamespace(title="OnlyTitle")
d = sr.to_debug_dict()
self.assertIn("campaign", d)
self.assertEqual(d["campaign"]["id"], camp.id)
# title present, slug/status omitted because getattr(...) returned None
self.assertEqual(d["campaign"]["title"], "OnlyTitle")
self.assertNotIn("slug", d["campaign"])
self.assertNotIn("status", d["campaign"])
def test_to_debug_dict_skips_none_topic_attrs(self):
# Force the related-object cache to a stub that lacks one of the looped attrs
from types import SimpleNamespace
t = Topic.objects.create(title="TT", slug="tt")
sr = SiteReport.objects.create(topic=t, assets_total=2)
sr._state.fields_cache["topic"] = SimpleNamespace(slug="only-slug")
d = sr.to_debug_dict()
self.assertIn("topic", d)
self.assertEqual(d["topic"]["id"], t.id)
# slug present, title omitted because getattr(...) returned None
self.assertEqual(d["topic"]["slug"], "only-slug")
self.assertNotIn("title", d["topic"])
class KeyMetricsReportTestCase(TestCase):
def _aware(self, y, m, d, hh=12, mm=0, ss=0):
tz = timezone.get_current_timezone()
return timezone.make_aware(datetime(y, m, d, hh, mm, ss), tz)
def _mk_sr(self, dt, report_name, **counters):
sr = SiteReport.objects.create(
report_name=report_name,
**counters,
)
SiteReport.objects.filter(pk=sr.pk).update(created_on=dt)
return SiteReport.objects.get(pk=sr.pk)
def test_helpers(self):
# FY math
self.assertEqual(
KeyMetricsReport.get_fiscal_year_for_date(date(2023, 10, 1)),
2024,
)
self.assertEqual(
KeyMetricsReport.get_fiscal_year_for_date(date(2024, 9, 30)),
2024,
)
self.assertEqual(
KeyMetricsReport.get_fiscal_quarter_for_date(date(2024, 2, 1)),
2,
)
self.assertEqual(
KeyMetricsReport.get_fiscal_quarter_for_date(date(2024, 10, 1)),
1,
)
# Month bounds (leap year Feb)
first, last = KeyMetricsReport.month_bounds(date(2024, 2, 10))
self.assertEqual(first, date(2024, 2, 1))
self.assertEqual(last, date(2024, 2, 29))
def test_upsert_month_from_sitereports(self):
# Baselines at 2023-12-31; EOM at 2024-01-31
base_dt = self._aware(2023, 12, 31, 9, 0, 0)
eom_dt = self._aware(2024, 1, 31, 23, 0, 0)
# TOTAL baseline + EOM
self._mk_sr(
base_dt,
SiteReport.ReportName.TOTAL,
assets_published=100,
assets_completed=50,
users_activated=10,
anonymous_transcriptions=5,
transcriptions_saved=20,
tag_uses=40,
)
self._mk_sr(
eom_dt,
SiteReport.ReportName.TOTAL,
assets_published=130,
assets_completed=70,
users_activated=16,
anonymous_transcriptions=8,
transcriptions_saved=26,
tag_uses=50,
)
# RETIRED_TOTAL baseline + EOM
self._mk_sr(
base_dt,
SiteReport.ReportName.RETIRED_TOTAL,
assets_published=10,
assets_completed=5,
users_activated=1,
anonymous_transcriptions=2,
transcriptions_saved=3,
tag_uses=4,
)
self._mk_sr(
eom_dt,
SiteReport.ReportName.RETIRED_TOTAL,
assets_published=15,
assets_completed=8,
users_activated=2,
anonymous_transcriptions=3,
transcriptions_saved=5,
tag_uses=6,
)
# Daily assets_started within the month (sums to 15 + 5 = 20)
self._mk_sr(
self._aware(2024, 1, 10),
SiteReport.ReportName.TOTAL,
assets_started=10,
)
self._mk_sr(
self._aware(2024, 1, 20),
SiteReport.ReportName.TOTAL,
assets_started=5,
)
self._mk_sr(
self._aware(2024, 1, 11),
SiteReport.ReportName.RETIRED_TOTAL,
assets_started=3,
)
self._mk_sr(
self._aware(2024, 1, 21),
SiteReport.ReportName.RETIRED_TOTAL,
assets_started=2,
)
# Upsert month
m = KeyMetricsReport.upsert_month(year=2024, month=1)
self.assertIsNotNone(m)
self.assertEqual(m.fiscal_year, 2024)
self.assertEqual(m.fiscal_quarter, 2)
self.assertEqual(m.month, 1)
# Deltas: see analysis; expect 35, 23, 7, 4, 8, 12 and started=20
self.assertEqual(m.assets_published, 35)
self.assertEqual(m.assets_completed, 23)
self.assertEqual(m.users_activated, 7)
self.assertEqual(m.anonymous_transcriptions, 4)
self.assertEqual(m.transcriptions_saved, 8)
self.assertEqual(m.tag_uses, 12)
self.assertEqual(m.assets_started, 20)
# __str__ + filenames
self.assertIn("FY2024M01", str(m))
self.assertTrue(m.csv_filename().startswith("key_metrics_monthly_fy2024"))
def test_upsert_quarter_and_fiscal_year_rollups(self):
# Create monthly rows for FY2024 Q2 (Jan & Feb present)
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 1, 31),
fiscal_year=2024,
fiscal_quarter=2,
month=1,
assets_published=10,
assets_started=2,
assets_completed=3,
users_activated=5,
anonymous_transcriptions=7,
transcriptions_saved=11,
tag_uses=13,
crowd_visits=None,
avg_visit_seconds=Decimal("10.50"),
)
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 2, 1),
period_end=date(2024, 2, 29),
fiscal_year=2024,
fiscal_quarter=2,
month=2,
assets_published=20,
assets_started=3,
assets_completed=4,
users_activated=6,
anonymous_transcriptions=8,
transcriptions_saved=12,
tag_uses=14,
# manual present in Feb only
crowd_visits=100,
avg_visit_seconds=None,
)
# Quarter upsert should sum calc fields; manual sums only when present
q2 = KeyMetricsReport.upsert_quarter(fiscal_year=2024, fiscal_quarter=2)
self.assertIsNotNone(q2)
self.assertEqual(q2.assets_published, 30)
self.assertEqual(q2.assets_started, 5)
self.assertEqual(q2.assets_completed, 7)
self.assertEqual(q2.users_activated, 11)
self.assertEqual(q2.anonymous_transcriptions, 15)
self.assertEqual(q2.transcriptions_saved, 23)
self.assertEqual(q2.tag_uses, 27)
# Manual: only Feb had a value, so total=100, avg from Jan only
self.assertEqual(q2.crowd_visits, 100)
self.assertEqual(q2.avg_visit_seconds, Decimal("10.50"))
# Fiscal year rollup on FY2024 should equal Jan+Feb (for now)
fy = KeyMetricsReport.upsert_fiscal_year(fiscal_year=2024)
self.assertIsNotNone(fy)
self.assertEqual(fy.assets_published, 30)
self.assertEqual(fy.crowd_visits, 100)
self.assertEqual(fy.avg_visit_seconds, Decimal("10.50"))
# String and filenames
self.assertIn("FY2024 Q2", str(q2))
self.assertTrue(q2.csv_filename().startswith("key_metrics_quarterly_fy2024"))
self.assertIn("FY2024 Report", str(fy))
self.assertTrue(fy.csv_filename().startswith("key_metrics_fiscal_year_fy2024"))
def test___str___fallback_when_fields_incomplete(self):
# QUARTERLY without fiscal_quarter so fallback label path
q = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 4, 1),
period_end=date(2024, 6, 30),
fiscal_year=2024,
fiscal_quarter=None,
)
s = str(q)
self.assertIn("KeyMetricsReport QUARTERLY", s)
self.assertIn("2024-04-01", s)
self.assertIn("2024-06-30", s)
# MONTHLY without month so fallback label path
m = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 5, 1),
period_end=date(2024, 5, 31),
fiscal_year=2024,
month=None,
)
s2 = str(m)
self.assertIn("KeyMetricsReport MONTHLY", s2)
def test_quarter_helper_edges(self):
# Q3 and Q4 branches
self.assertEqual(
KeyMetricsReport.get_fiscal_quarter_for_date(date(2024, 4, 1)), 3
)
self.assertEqual(
KeyMetricsReport.get_fiscal_quarter_for_date(date(2024, 7, 1)), 4
)
def test__format_value_for_csv_variants(self):
rep = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,
period_start=date(2023, 10, 1),
period_end=date(2024, 9, 30),
fiscal_year=2024,
)
self.assertEqual(rep._format_value_for_csv("crowd_visits", None), "")
# Manual Decimal (avg_visit_seconds) to string with 2 decimals
self.assertEqual(
rep._format_value_for_csv("avg_visit_seconds", Decimal("10")),
"10.00",
)
self.assertEqual(rep._format_value_for_csv("crowd_visits", 0), 0)
self.assertEqual(rep._format_value_for_csv("assets_started", None), 0)
# Unknown field fallback: None to "", non-None to value passthrough
self.assertEqual(rep._format_value_for_csv("unknown_field", None), "")
self.assertEqual(rep._format_value_for_csv("unknown_field", "x"), "x")
def test_upsert_month_returns_none_when_no_snapshots(self):
out = KeyMetricsReport.upsert_month(year=2025, month=6)
self.assertIsNone(out)
def test_upsert_quarter_invalid_quarter_raises(self):
with self.assertRaises(ValueError):
KeyMetricsReport.upsert_quarter(fiscal_year=2024, fiscal_quarter=5)
def test_quarter_month_specs_all_quarters(self):
# Q1
q1 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2023, 10, 1),
period_end=date(2023, 12, 31),
fiscal_year=2024,
fiscal_quarter=1,
)
self.assertEqual(
q1._quarter_month_specs(), [(2023, 10), (2023, 11), (2023, 12)]
)
# Q3
q3 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 4, 1),
period_end=date(2024, 6, 30),
fiscal_year=2024,
fiscal_quarter=3,
)
self.assertEqual(q3._quarter_month_specs(), [(2024, 4), (2024, 5), (2024, 6)])
# Q4
q4 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 7, 1),
period_end=date(2024, 9, 30),
fiscal_year=2024,
fiscal_quarter=4,
)
self.assertEqual(q4._quarter_month_specs(), [(2024, 7), (2024, 8), (2024, 9)])
def test_month_bounds_handles_december(self):
first, last = KeyMetricsReport.month_bounds(date(2024, 12, 10))
self.assertEqual(first, date(2024, 12, 1))
self.assertEqual(last, date(2024, 12, 31))
def test__monthly_from_sitereports_returns_empty_dict_when_no_eom(self):
vals = KeyMetricsReport._monthly_from_sitereports(
month_start=date(2030, 5, 1),
month_end=date(2030, 5, 31),
)
self.assertEqual(vals, {}) # no snapshots at all
def test__monthly_from_sitereports_baseline_fallback_inside_month(self):
# No snapshots before month start; first snapshot inside the month
start = date(2024, 3, 1)
end = date(2024, 3, 31)
# TOTAL: baseline inside month (10) -> EOM (15)
self._mk_sr(
self._aware(2024, 3, 5, 9, 0, 0),
SiteReport.ReportName.TOTAL,
assets_published=10,
)
self._mk_sr(
self._aware(2024, 3, 31, 23, 0, 0),
SiteReport.ReportName.TOTAL,
assets_published=15,
)
# RETIRED: baseline inside month (4) -> EOM (7)
self._mk_sr(
self._aware(2024, 3, 10, 9, 0, 0),
SiteReport.ReportName.RETIRED_TOTAL,
assets_published=4,
)
self._mk_sr(
self._aware(2024, 3, 31, 23, 0, 0),
SiteReport.ReportName.RETIRED_TOTAL,
assets_published=7,
)
vals = KeyMetricsReport._monthly_from_sitereports(
month_start=start, month_end=end
)
# delta should be (15+7) - (10+4) = 8
self.assertEqual(vals["assets_published"], 8)
def test__monthly_from_sitereports_treats_missing_series_as_zero(self):
# Only TOTAL snapshots; RETIRED series absent
start = date(2024, 4, 1)
end = date(2024, 4, 30)
# baseline inside month (100) -> EOM (110)
self._mk_sr(
self._aware(2024, 4, 5, 9, 0, 0),
SiteReport.ReportName.TOTAL,
assets_published=100,
)
self._mk_sr(
self._aware(2024, 4, 30, 23, 0, 0),
SiteReport.ReportName.TOTAL,
assets_published=110,
)
vals = KeyMetricsReport._monthly_from_sitereports(
month_start=start, month_end=end
)
# RETIRED contributes 0 via the helper that treats None as 0
self.assertEqual(vals["assets_published"], 10)
def test_upsert_quarter_returns_none_when_no_monthlies_all_quarters(self):
# Q1
out1 = KeyMetricsReport.upsert_quarter(fiscal_year=2027, fiscal_quarter=1)
self.assertIsNone(out1)
# Q3
out3 = KeyMetricsReport.upsert_quarter(fiscal_year=2027, fiscal_quarter=3)
self.assertIsNone(out3)
# Q4
out4 = KeyMetricsReport.upsert_quarter(fiscal_year=2027, fiscal_quarter=4)
self.assertIsNone(out4)
def test_upsert_fiscal_year_returns_none_when_no_monthlies(self):
out = KeyMetricsReport.upsert_fiscal_year(fiscal_year=2029)
self.assertIsNone(out)
def test__calendar_year_for_month_in_fy_helper(self):
rep = KeyMetricsReport(
period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,
period_start=date(2023, 10, 1),
period_end=date(2024, 9, 30),
fiscal_year=2024,
)
# Oct in FY should map to previous calendar year
self.assertEqual(rep._calendar_year_for_month_in_fy(10, 2024), 2023)
# Jun in FY maps to the FY year
self.assertEqual(rep._calendar_year_for_month_in_fy(6, 2024), 2024)
def test_quarter_month_specs_q2(self):
q2 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
self.assertEqual(q2._quarter_month_specs(), [(2024, 1), (2024, 2), (2024, 3)])
class KeyMetricsReportCsvTestCase(TestCase):
def setUp(self):
# FY2023 FY row (for lifetime math)
self.fy23 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,
period_start=date(2022, 10, 1),
period_end=date(2023, 9, 30),
fiscal_year=2023,
assets_published=50,
assets_started=5,
assets_completed=7,
users_activated=11,
anonymous_transcriptions=13,
transcriptions_saved=17,
tag_uses=19,
crowd_visits=30,
avg_visit_seconds=Decimal("9.00"),
)
# FY2024 Q1 (for quarterly lifetime math on Q2)
self.q1_24 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2023, 10, 1),
period_end=date(2023, 12, 31),
fiscal_year=2024,
fiscal_quarter=1,
assets_published=7,
assets_started=1,
assets_completed=2,
users_activated=3,
anonymous_transcriptions=4,
transcriptions_saved=5,
tag_uses=6,
crowd_visits=None,
avg_visit_seconds=Decimal("8.00"),
)
# FY2024 monthly rows for Q2: Jan, Feb present only
self.jan24 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 1, 31),
fiscal_year=2024,
fiscal_quarter=2,
month=1,
assets_published=10,
assets_started=2,
assets_completed=3,
users_activated=5,
anonymous_transcriptions=7,
transcriptions_saved=11,
tag_uses=13,
crowd_visits=None,
)
self.feb24 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 2, 1),
period_end=date(2024, 2, 29),
fiscal_year=2024,
fiscal_quarter=2,
month=2,
assets_published=20,
assets_started=3,
assets_completed=4,
users_activated=6,
anonymous_transcriptions=8,
transcriptions_saved=12,
tag_uses=14,
crowd_visits=100,
)
# Upsert Q2 and FY2024 so we can render CSVs with proper totals
self.q2_24 = KeyMetricsReport.upsert_quarter(
fiscal_year=2024,
fiscal_quarter=2,
)
self.fy24 = KeyMetricsReport.upsert_fiscal_year(fiscal_year=2024)
def _csv_as_lines(self, rep: KeyMetricsReport) -> list[list[str]]:
raw = rep.render_csv().decode("utf-8")
return [line.split(",") for line in raw.strip().splitlines()]
def test_monthly_csv_headers_and_values(self):
# Build a synthetic single-month report to test header label only
m = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 6, 1),
period_end=date(2024, 6, 30),
fiscal_year=2024,
fiscal_quarter=3,
month=6,
assets_published=1,
)
lines = self._csv_as_lines(m)
# Header: "Metric", "
"
self.assertEqual(lines[0][0], "Metric")
self.assertEqual(lines[0][1], "June")
# One known metric row check
labels = [row[0] for row in lines[1:]]
vals = [row[1] for row in lines[1:]]
pub_idx = labels.index("Assets published")
self.assertEqual(int(vals[pub_idx]), 1)
def test_quarterly_csv_headers_totals_and_lifetime(self):
lines = self._csv_as_lines(self.q2_24)
header = lines[0]
# Months present (Jan, Feb), then "FY24 Q2 totals", "FY24 Lifetime totals"
self.assertEqual(header[0], "Metric")
self.assertIn("January", header)
self.assertIn("February", header)
self.assertIn("FY24 Q2 totals", header)
self.assertIn("FY24 Lifetime totals", header)
# Assets published row:
# Jan(10), Feb(20) => quarter total=30
# Lifetime = FY2023 FY(50) + FY2024 Q1(7) = 57
labels = [row[0] for row in lines[1:]]
ap_idx = labels.index("Assets published")
row = lines[1 + ap_idx]
# [label, Jan, Feb, Q2 total, Lifetime]
self.assertEqual(int(row[1]), 10)
self.assertEqual(int(row[2]), 20)
self.assertEqual(int(row[3]), 30)
self.assertEqual(int(row[4]), 57)
# Manual example (Crowd.loc.gov visits):
# Jan(None), Feb(100) => Q2 total=100 (not blank)
# Lifetime = FY2023 FY(30) + Q1(None) => 30
cv_idx = labels.index("Crowd.loc.gov visits")
row2 = lines[1 + cv_idx]
self.assertEqual(row2[1], "") # January empty
self.assertEqual(int(row2[2]), 100)
self.assertEqual(int(row2[3]), 100)
self.assertEqual(int(row2[4]), 30)
def test_fiscal_year_csv_headers_totals_and_lifetime(self):
# Ensure FY rows exist for lifetime (FY2023 and FY2024 already present)
lines = self._csv_as_lines(self.fy24)
header = lines[0]
# Header pattern:
# Metric | (FY24 Q1 totals if present) | Q2 totals | Q3 totals? | Q4 totals?
# | FY24 totals | FY24 Lifetime totals
self.assertEqual(header[0], "Metric")
self.assertIn("FY24 Q1 totals", header)
self.assertIn("Q2 totals", header)
self.assertIn("FY24 totals", header)
self.assertIn("FY24 Lifetime totals", header)
labels = [row[0] for row in lines[1:]]
ap_idx = labels.index("Assets published")
row = lines[1 + ap_idx]
# With our setup:
# Q1 assets_published=7 (preset), Q2=30 (from jan+feb),
# year total = 37, lifetime = FY2023 FY(50) + FY2024 FY(37) = 87
# Header columns could be: Metric, FY24 Q1 totals, Q2 totals,
# FY24 totals, FY24 Lifetime totals (Q3/Q4 absent)
# Find indices dynamically.
h = header
q1_i = h.index("FY24 Q1 totals")
q2_i = h.index("Q2 totals")
yt_i = h.index("FY24 totals")
lt_i = h.index("FY24 Lifetime totals")
self.assertEqual(int(row[q1_i]), 7)
self.assertEqual(int(row[q2_i]), 30)
self.assertEqual(int(row[yt_i]), 37)
expected_lifetime = self.fy23.assets_published + self.fy24.assets_published
self.assertEqual(int(row[lt_i]), expected_lifetime)
def test_str_formats(self):
# Monthly string covers Oct (calendar year is fy-1)
oct_row = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2023, 10, 1),
period_end=date(2023, 10, 31),
fiscal_year=2024,
fiscal_quarter=1,
month=10,
)
s = str(oct_row)
self.assertIn("FY2024M10", s)
self.assertIn("(October 2023)", s)
def test_quarterly_csv_when_no_monthlies_and_no_priors(self):
# Create a standalone quarterly row with no monthly rows in that quarter
q = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2019, 10, 1),
period_end=date(2019, 12, 31),
fiscal_year=2020,
fiscal_quarter=1,
)
lines = self._csv_as_lines(q)
# Header should be: Metric | FY25 Q3 totals | FY25 Lifetime totals
self.assertEqual(lines[0][0], "Metric")
self.assertIn("FY20 Q1 totals", lines[0])
self.assertIn("FY20 Lifetime totals", lines[0])
self.assertEqual(len(lines[0]), 3)
labels = [r[0] for r in lines[1:]]
# Calculated field: totals are numeric, lifetime is 0
ap_i = labels.index("Assets published")
ap_row = lines[1 + ap_i]
self.assertEqual(int(ap_row[1]), 0) # quarter total
self.assertEqual(int(ap_row[2]), 0) # lifetime total
# Manual field: totals should be blank when no values
cv_i = labels.index("Crowd.loc.gov visits")
cv_row = lines[1 + cv_i]
self.assertEqual(cv_row[1], "") # quarter total blank
self.assertEqual(cv_row[2], "") # lifetime total blank
def test_fiscal_year_csv_headers_when_q1_missing(self):
# Create an FY row and only Q2 and Q4 quarters for that FY
fy = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,
period_start=date(2025, 10, 1),
period_end=date(2026, 9, 30),
fiscal_year=2026,
)
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2026, 1, 1),
period_end=date(2026, 3, 31),
fiscal_year=2026,
fiscal_quarter=2,
assets_published=12,
)
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2026, 7, 1),
period_end=date(2026, 9, 30),
fiscal_year=2026,
fiscal_quarter=4,
assets_published=8,
)
lines = self._csv_as_lines(fy)
header = lines[0]
self.assertEqual(header[0], "Metric")
self.assertNotIn("FY26 Q1 totals", header)
self.assertIn("Q2 totals", header)
self.assertNotIn("Q3 totals", header)
self.assertIn("Q4 totals", header)
self.assertIn("FY26 totals", header)
self.assertIn("FY26 Lifetime totals", header)
labels = [r[0] for r in lines[1:]]
ap_i = labels.index("Assets published")
row = lines[1 + ap_i]
q2_i = header.index("Q2 totals")
q4_i = header.index("Q4 totals")
yt_i = header.index("FY26 totals")
lt_i = header.index("FY26 Lifetime totals")
self.assertEqual(int(row[q2_i]), 12)
self.assertEqual(int(row[q4_i]), 8)
self.assertEqual(int(row[yt_i]), 20)
# Lifetime sums all FY rows <= 2026 (FY2023 + FY2024 + FY2026)
expected_lifetime = (
self.fy23.assets_published + self.fy24.assets_published + 0
) # FY2026 FY row has no stored value in this test
self.assertEqual(int(row[lt_i]), expected_lifetime)
def test_format_value_for_csv_non_decimal_avg(self):
rep = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,
period_start=date(2025, 10, 1),
period_end=date(2026, 9, 30),
fiscal_year=2026,
)
self.assertEqual(
rep._format_value_for_csv("avg_visit_seconds", 12.3),
"12.3",
)
================================================
FILE: concordia/tests/test_parser.py
================================================
from types import SimpleNamespace
from unittest import mock
import requests
from django.test import TestCase
from requests.models import Response
import concordia.parser as parser_mod
from concordia.parser import extract_og_image, fetch_blog_posts, paginate_blog_posts
TITLE = "What’s New Online at the Library of Congress: May 2025"
LINK = "https://blogs.loc.gov/thesignal/2025/05/new-loc-may-2025/"
RSS = """
The Signal
-
%s
-
Volunteers Leverage OCR to Transcribe Library of Congress Digit...
""" % (
TITLE,
LINK,
)
IMAGE = "https://blogs.loc.gov/thesignal/files/2025/05/loc-2017698702.png"
HTML = """
""" % (
link1,
link2,
)
mock_response = mock.MagicMock(spec=Response)
mock_response.content = rss
mock_response.status_code = 200
mock_get.return_value = mock_response
# run the celery task
fetch_and_cache_blog_images()
mock_extract.assert_any_call(link1)
mock_extract.assert_any_call(link2)
self.assertEqual(mock_extract.call_count, 2)
@mock.patch("concordia.tasks.blog.extract_og_image")
@mock.patch("concordia.tasks.blog.fetch_blog_posts")
def test_skips_items_with_no_link(self, mock_fetch, mock_extract):
# Provide one item without a link and one with a link to make
# sure we handle no link correctly
class DummyLink:
def __init__(self, text):
self.text = text
class DummyItem:
def __init__(self, link):
self._link = link
def find(self, name):
return self._link if name == "link" else None
item_no_link = DummyItem(None)
item_with_link = DummyItem(DummyLink("https://example.invalid/post"))
mock_fetch.return_value = [item_no_link, item_with_link]
fetch_and_cache_blog_images()
mock_extract.assert_called_once_with("https://example.invalid/post")
================================================
FILE: concordia/tests/test_tasks_housekeeping.py
================================================
from unittest import mock
from django.test import TestCase
from concordia.tasks.housekeeping import clear_sessions
class ClearSessionsTaskTests(TestCase):
def test_calls_django_clearsessions_command(self):
# Verify the task invokes Django's clearsessions management command.
with mock.patch("concordia.tasks.housekeeping.call_command") as mock_call:
result = clear_sessions()
self.assertIsNone(result)
mock_call.assert_called_once_with("clearsessions")
def test_raises_when_call_command_fails(self):
# Ensure exceptions from the management command propagate.
with mock.patch(
"concordia.tasks.housekeeping.call_command",
side_effect=RuntimeError("boom"),
):
with self.assertRaises(RuntimeError):
clear_sessions()
================================================
FILE: concordia/tests/test_tasks_next_asset.py
================================================
from unittest import mock
from django.test import TestCase
from django.utils import timezone
from concordia.models import (
NextReviewableCampaignAsset,
NextReviewableTopicAsset,
NextTranscribableCampaignAsset,
NextTranscribableTopicAsset,
TranscriptionStatus,
)
from concordia.tasks.next_asset.renew import renew_next_asset_cache
from concordia.tasks.next_asset.reviewable import (
clean_next_reviewable_for_campaign,
clean_next_reviewable_for_topic,
populate_next_reviewable_for_campaign,
populate_next_reviewable_for_topic,
)
from concordia.tasks.next_asset.transcribable import (
clean_next_transcribable_for_campaign,
clean_next_transcribable_for_topic,
populate_next_transcribable_for_campaign,
populate_next_transcribable_for_topic,
)
from concordia.utils import get_anonymous_user
from .utils import (
CreateTestUsers,
create_asset,
create_topic,
create_transcription,
)
class PopulateNextAssetTasksTests(CreateTestUsers, TestCase):
def setUp(self):
self.anon = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(slug="test-asset-1", title="Test Asset 1")
self.asset2 = create_asset(
item=self.asset1.item, slug="test-asset-2", title="Test Asset 2"
)
self.topic = create_topic(project=self.asset1.item.project)
self.campaign = self.asset1.campaign
def test_populate_next_transcribable_for_campaign(self):
populate_next_transcribable_for_campaign(campaign_id=self.campaign.id)
self.assertEqual(
NextTranscribableCampaignAsset.objects.filter(
campaign=self.campaign
).count(),
2,
)
def test_populate_next_transcribable_for_topic(self):
populate_next_transcribable_for_topic(topic_id=self.topic.id)
self.assertEqual(
NextTranscribableTopicAsset.objects.filter(topic=self.topic).count(), 2
)
def test_populate_next_reviewable_for_campaign(self):
create_transcription(
asset=self.asset1, user=self.anon, submitted=timezone.now()
)
create_transcription(
asset=self.asset2, user=self.user, submitted=timezone.now()
)
populate_next_reviewable_for_campaign(campaign_id=self.campaign.id)
self.assertEqual(
NextReviewableCampaignAsset.objects.filter(campaign=self.campaign).count(),
2,
)
def test_populate_next_reviewable_for_topic(self):
create_transcription(
asset=self.asset1, user=self.anon, submitted=timezone.now()
)
create_transcription(
asset=self.asset2, user=self.user, submitted=timezone.now()
)
populate_next_reviewable_for_topic(topic_id=self.topic.id)
self.assertEqual(
NextReviewableTopicAsset.objects.filter(topic=self.topic).count(), 2
)
@mock.patch("concordia.tasks.next_asset.transcribable.logger")
def test_populate_next_transcribable_for_campaign_missing(self, mock_logger):
populate_next_transcribable_for_campaign(campaign_id=9999)
mock_logger.error.assert_called_once()
@mock.patch("concordia.tasks.next_asset.transcribable.logger")
def test_populate_next_transcribable_for_topic_missing(self, mock_logger):
populate_next_transcribable_for_topic(topic_id=9999)
mock_logger.error.assert_called_once()
@mock.patch("concordia.tasks.next_asset.reviewable.logger")
def test_populate_next_reviewable_for_campaign_missing(self, mock_logger):
populate_next_reviewable_for_campaign(campaign_id=9999)
mock_logger.error.assert_called_once()
@mock.patch("concordia.tasks.next_asset.reviewable.logger")
def test_populate_next_reviewable_for_topic_missing(self, mock_logger):
populate_next_reviewable_for_topic(topic_id=9999)
mock_logger.error.assert_called_once()
@mock.patch("concordia.tasks.next_asset.transcribable.logger")
def test_populate_next_transcribable_for_campaign_none_needed(self, mock_logger):
for i in range(3, 103):
asset = create_asset(item=self.asset1.item, slug=f"dummy-{i}")
NextTranscribableCampaignAsset.objects.create(
asset=asset,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
campaign=self.campaign,
sequence=asset.sequence,
transcription_status=asset.transcription_status,
)
populate_next_transcribable_for_campaign(campaign_id=self.campaign.id)
mock_logger.info.assert_any_call(
"Campaign %s already has %s next transcribable assets", self.campaign, 100
)
@mock.patch("concordia.tasks.next_asset.transcribable.logger")
def test_populate_next_transcribable_for_topic_none_needed(self, mock_logger):
for i in range(3, 103):
asset = create_asset(item=self.asset1.item, slug=f"dummy-{i}")
NextTranscribableTopicAsset.objects.create(
asset=asset,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
topic=self.topic,
sequence=asset.sequence,
transcription_status=asset.transcription_status,
)
populate_next_transcribable_for_topic(topic_id=self.topic.id)
mock_logger.info.assert_any_call(
"Topic %s already has %s next transcribable assets", self.topic, 100
)
@mock.patch("concordia.tasks.next_asset.reviewable.logger")
def test_populate_next_reviewable_for_campaign_none_needed(self, mock_logger):
create_transcription(
asset=self.asset1, user=self.user, submitted=timezone.now()
)
for i in range(3, 103):
asset = create_asset(item=self.asset1.item, slug=f"r-{i}")
create_transcription(asset=asset, user=self.user, submitted=timezone.now())
NextReviewableCampaignAsset.objects.create(
asset=asset,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
campaign=self.campaign,
sequence=asset.sequence,
transcriber_ids=[self.user.id],
)
populate_next_reviewable_for_campaign(campaign_id=self.campaign.id)
mock_logger.info.assert_any_call(
"Campaign %s already has %s next reviewable assets", self.campaign, 100
)
@mock.patch("concordia.tasks.next_asset.reviewable.logger")
def test_populate_next_reviewable_for_topic_none_needed(self, mock_logger):
create_transcription(
asset=self.asset1, user=self.user, submitted=timezone.now()
)
for i in range(3, 103):
asset = create_asset(item=self.asset1.item, slug=f"t-{i}")
create_transcription(asset=asset, user=self.user, submitted=timezone.now())
NextReviewableTopicAsset.objects.create(
asset=asset,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
topic=self.topic,
sequence=asset.sequence,
transcriber_ids=[self.user.id],
)
populate_next_reviewable_for_topic(topic_id=self.topic.id)
mock_logger.info.assert_any_call(
"Topic %s already has %s next reviewable assets", self.topic, 100
)
@mock.patch("concordia.tasks.next_asset.reviewable.logger")
def test_populate_next_reviewable_for_campaign_none_found(self, mock_logger):
create_transcription(
asset=self.asset1, user=self.user, submitted=timezone.now()
)
NextReviewableCampaignAsset.objects.create(
asset=self.asset1,
item=self.asset1.item,
item_item_id=self.asset1.item.item_id,
project=self.asset1.item.project,
project_slug=self.asset1.item.project.slug,
campaign=self.campaign,
sequence=self.asset1.sequence,
transcriber_ids=[self.user.id],
)
populate_next_reviewable_for_campaign(campaign_id=self.campaign.id)
mock_logger.info.assert_any_call(
"No reviewable assets found in campaign %s", self.campaign
)
@mock.patch("concordia.tasks.next_asset.reviewable.logger")
def test_populate_next_reviewable_for_topic_none_found(self, mock_logger):
create_transcription(
asset=self.asset1, user=self.user, submitted=timezone.now()
)
NextReviewableTopicAsset.objects.create(
asset=self.asset1,
item=self.asset1.item,
item_item_id=self.asset1.item.item_id,
project=self.asset1.item.project,
project_slug=self.asset1.item.project.slug,
topic=self.topic,
sequence=self.asset1.sequence,
transcriber_ids=[self.user.id],
)
populate_next_reviewable_for_topic(topic_id=self.topic.id)
mock_logger.info.assert_any_call(
"No reviewable assets found in topic %s", self.topic
)
@mock.patch("concordia.tasks.next_asset.transcribable.logger")
def test_populate_next_transcribable_for_campaign_none_found(self, mock_logger):
for asset in (self.asset1, self.asset2):
NextTranscribableCampaignAsset.objects.create(
asset=asset,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
campaign=self.campaign,
sequence=asset.sequence,
transcription_status=asset.transcription_status,
)
populate_next_transcribable_for_campaign(campaign_id=self.campaign.id)
mock_logger.info.assert_any_call(
"No transcribable assets found in campaign %s", self.campaign
)
@mock.patch("concordia.tasks.next_asset.transcribable.logger")
def test_populate_next_transcribable_for_topic_none_found(self, mock_logger):
for asset in (self.asset1, self.asset2):
NextTranscribableTopicAsset.objects.create(
asset=asset,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
topic=self.topic,
sequence=asset.sequence,
transcription_status=asset.transcription_status,
)
populate_next_transcribable_for_topic(topic_id=self.topic.id)
mock_logger.info.assert_any_call(
"No transcribable assets found in topic %s", self.topic
)
class CleanNextAssetTasksTests(TestCase):
def setUp(self):
self.asset = create_asset()
self.campaign = self.asset.campaign
self.topic = create_topic(project=self.asset.item.project)
self.campaign_transcribable = NextTranscribableCampaignAsset.objects.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.asset.item.project,
project_slug=self.asset.item.project.slug,
campaign=self.campaign,
sequence=self.asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
self.topic_transcribable = NextTranscribableTopicAsset.objects.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.asset.item.project,
project_slug=self.asset.item.project.slug,
topic=self.topic,
sequence=self.asset.sequence,
transcription_status=TranscriptionStatus.IN_PROGRESS,
)
self.campaign_reviewable = NextReviewableCampaignAsset.objects.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.asset.item.project,
project_slug=self.asset.item.project.slug,
campaign=self.campaign,
sequence=self.asset.sequence,
)
self.topic_reviewable = NextReviewableTopicAsset.objects.create(
asset=self.asset,
item=self.asset.item,
item_item_id=self.asset.item.item_id,
project=self.asset.item.project,
project_slug=self.asset.item.project.slug,
topic=self.topic,
sequence=self.asset.sequence,
)
@mock.patch(
"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_campaign.delay"
)
def test_clean_next_transcribable_for_campaign(self, mock_delay):
self.asset.transcription_status = TranscriptionStatus.COMPLETED
self.asset.save()
clean_next_transcribable_for_campaign(self.campaign.id)
self.assertFalse(
NextTranscribableCampaignAsset.objects.filter(
campaign=self.campaign
).exists()
)
mock_delay.assert_called_once_with(self.campaign.id)
@mock.patch(
"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_topic.delay"
)
def test_clean_next_transcribable_for_topic(self, mock_delay):
self.asset.transcription_status = TranscriptionStatus.COMPLETED
self.asset.save()
clean_next_transcribable_for_topic(self.topic.id)
self.assertFalse(
NextTranscribableTopicAsset.objects.filter(topic=self.topic).exists()
)
mock_delay.assert_called_once_with(self.topic.id)
@mock.patch(
"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_campaign.delay"
)
def test_clean_next_reviewable_for_campaign(self, mock_delay):
self.asset.transcription_status = TranscriptionStatus.IN_PROGRESS
self.asset.save()
clean_next_reviewable_for_campaign(self.campaign.id)
self.assertFalse(
NextReviewableCampaignAsset.objects.filter(campaign=self.campaign).exists()
)
mock_delay.assert_called_once_with(self.campaign.id)
@mock.patch(
"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_topic.delay"
)
def test_clean_next_reviewable_for_topic(self, mock_delay):
self.asset.transcription_status = TranscriptionStatus.NOT_STARTED
self.asset.save()
clean_next_reviewable_for_topic(self.topic.id)
self.assertFalse(
NextReviewableTopicAsset.objects.filter(topic=self.topic).exists()
)
mock_delay.assert_called_once_with(self.topic.id)
@mock.patch(
"concordia.tasks.next_asset.reviewable.clean_next_reviewable_for_campaign.delay"
)
@mock.patch(
"concordia.tasks.next_asset.transcribable.clean_next_transcribable_for_campaign.delay"
)
@mock.patch(
"concordia.tasks.next_asset.reviewable.clean_next_reviewable_for_topic.delay"
)
@mock.patch(
"concordia.tasks.next_asset.transcribable.clean_next_transcribable_for_topic.delay"
)
def test_renew_next_asset_cache(
self,
mock_clean_trans_topic,
mock_clean_rev_topic,
mock_clean_trans_campaign,
mock_clean_rev_campaign,
):
renew_next_asset_cache()
mock_clean_trans_campaign.assert_called_once_with(campaign_id=self.campaign.id)
mock_clean_rev_campaign.assert_called_once_with(campaign_id=self.campaign.id)
mock_clean_trans_topic.assert_called_once_with(topic_id=self.topic.id)
mock_clean_rev_topic.assert_called_once_with(topic_id=self.topic.id)
@mock.patch("concordia.tasks.next_asset.transcribable.logger")
def test_clean_next_transcribable_for_campaign_exception(self, mock_logger):
with mock.patch.object(
self.campaign_transcribable, "delete", side_effect=Exception("fail")
):
with mock.patch(
"concordia.tasks.next_asset.transcribable.find_invalid_next_transcribable_campaign_assets",
return_value=[self.campaign_transcribable],
):
clean_next_transcribable_for_campaign(self.campaign.id)
mock_logger.exception.assert_called_once()
@mock.patch("concordia.tasks.next_asset.transcribable.logger")
def test_clean_next_transcribable_for_topic_exception(self, mock_logger):
with mock.patch.object(
self.topic_transcribable, "delete", side_effect=Exception("fail")
):
with mock.patch(
"concordia.tasks.next_asset.transcribable.find_invalid_next_transcribable_topic_assets",
return_value=[self.topic_transcribable],
):
clean_next_transcribable_for_topic(self.topic.id)
mock_logger.exception.assert_called_once()
@mock.patch("concordia.tasks.next_asset.reviewable.logger")
def test_clean_next_reviewable_for_campaign_exception(self, mock_logger):
with mock.patch.object(
self.campaign_reviewable, "delete", side_effect=Exception("fail")
):
with mock.patch(
"concordia.tasks.next_asset.reviewable.find_invalid_next_reviewable_campaign_assets",
return_value=[self.campaign_reviewable],
):
clean_next_reviewable_for_campaign(self.campaign.id)
mock_logger.exception.assert_called_once()
@mock.patch("concordia.tasks.next_asset.reviewable.logger")
def test_clean_next_reviewable_for_topic_exception(self, mock_logger):
with mock.patch.object(
self.topic_reviewable, "delete", side_effect=Exception("fail")
):
with mock.patch(
"concordia.tasks.next_asset.reviewable.find_invalid_next_reviewable_topic_assets",
return_value=[self.topic_reviewable],
):
clean_next_reviewable_for_topic(self.topic.id)
mock_logger.exception.assert_called_once()
================================================
FILE: concordia/tests/test_tasks_reports_backfill.py
================================================
from django.test import TestCase
from django.utils import timezone
from concordia.models import Campaign, SiteReport, Topic
from concordia.tasks.reports.backfill import (
backfill_assets_started_for_site_reports,
)
class BackfillAssetsStartedTaskTests(TestCase):
def _dt(self, days_ago):
return timezone.now() - timezone.timedelta(days=days_ago)
def test_updates_total_and_skips_existing_by_default(self):
# Three TOTAL rows in time order. The last is already populated and
# should be skipped in default mode.
r1 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_total=100,
assets_not_started=100,
assets_published=10,
assets_started=None,
)
r2 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_total=107,
assets_not_started=92,
assets_published=17,
assets_started=None,
)
r3 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_total=100,
assets_not_started=90,
assets_published=20,
assets_started=5,
)
dt1 = self._dt(3)
dt2 = self._dt(2)
dt3 = self._dt(1)
SiteReport.objects.filter(pk=r1.pk).update(created_on=dt1)
SiteReport.objects.filter(pk=r2.pk).update(created_on=dt2)
SiteReport.objects.filter(pk=r3.pk).update(created_on=dt3)
# The TOTAL series assets_started is now expected to be rolled up from
# per-campaign series that were generated on the same reporting day.
camp1 = Campaign.objects.create(title="C1", slug="c1")
camp2 = Campaign.objects.create(title="C2", slug="c2")
c1_prev = SiteReport.objects.create(
campaign=camp1,
assets_total=50,
assets_not_started=50,
assets_published=0,
assets_started=None,
)
c1_curr = SiteReport.objects.create(
campaign=camp1,
assets_total=50,
assets_not_started=40,
assets_published=0,
assets_started=None,
)
c2_prev = SiteReport.objects.create(
campaign=camp2,
assets_total=50,
assets_not_started=50,
assets_published=0,
assets_started=None,
)
c2_curr = SiteReport.objects.create(
campaign=camp2,
assets_total=52,
assets_not_started=47,
assets_published=0,
assets_started=None,
)
SiteReport.objects.filter(pk=c1_prev.pk).update(created_on=dt1)
SiteReport.objects.filter(pk=c1_curr.pk).update(created_on=dt2)
SiteReport.objects.filter(pk=c2_prev.pk).update(created_on=dt1)
SiteReport.objects.filter(pk=c2_curr.pk).update(created_on=dt2)
updated = backfill_assets_started_for_site_reports.run()
# Campaign series (2 campaigns x 2 rows) + TOTAL series (2 rows).
self.assertEqual(updated, 6)
r1.refresh_from_db()
r2.refresh_from_db()
r3.refresh_from_db()
c1_prev.refresh_from_db()
c1_curr.refresh_from_db()
c2_prev.refresh_from_db()
c2_curr.refresh_from_db()
self.assertEqual(c1_prev.assets_started, 0)
self.assertEqual(c1_curr.assets_started, 10)
self.assertEqual(c2_prev.assets_started, 0)
self.assertEqual(c2_curr.assets_started, 5)
self.assertEqual(r1.assets_started, 0)
self.assertEqual(r2.assets_started, 15)
self.assertEqual(r3.assets_started, 5)
def test_recompute_when_skip_existing_is_false(self):
# Build a TOTAL series with two rows. Make the first row have a wrong,
# non-null assets_started so it should be recomputed when
# skip_existing is False.
#
# The TOTAL series assets_started is expected to be rolled up from
# per-campaign series generated on the same reporting day.
now = timezone.now()
prev = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_total=100,
assets_not_started=100,
assets_published=10,
)
curr = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_total=105,
assets_not_started=90,
assets_published=15,
)
dt_prev = now - timezone.timedelta(days=2)
dt_curr = now - timezone.timedelta(days=1)
# Enforce chronological order for the iterator
SiteReport.objects.filter(pk=prev.pk).update(created_on=dt_prev)
SiteReport.objects.filter(pk=curr.pk).update(created_on=dt_curr)
# Wrong non-null on first row, null on second.
SiteReport.objects.filter(pk=prev.pk).update(assets_started=5)
SiteReport.objects.filter(pk=curr.pk).update(assets_started=None)
camp = Campaign.objects.create(title="C", slug="c")
c_prev = SiteReport.objects.create(
campaign=camp,
assets_total=100,
assets_not_started=100,
assets_published=0,
assets_started=None,
)
c_curr = SiteReport.objects.create(
campaign=camp,
assets_total=105,
assets_not_started=90,
assets_published=0,
assets_started=None,
)
SiteReport.objects.filter(pk=c_prev.pk).update(created_on=dt_prev)
SiteReport.objects.filter(pk=c_curr.pk).update(created_on=dt_curr)
updated = backfill_assets_started_for_site_reports.run(skip_existing=False)
# Campaign series (2 rows) + TOTAL series (2 rows).
self.assertEqual(updated, 4)
prev_refreshed = SiteReport.objects.get(pk=prev.pk)
curr_refreshed = SiteReport.objects.get(pk=curr.pk)
c_prev.refresh_from_db()
c_curr.refresh_from_db()
self.assertEqual(c_prev.assets_started, 0)
self.assertEqual(c_curr.assets_started, 15)
self.assertEqual(prev_refreshed.assets_started, 0)
self.assertEqual(curr_refreshed.assets_started, 15)
def test_processes_retired_campaign_and_topic_series(self):
# One RETIRED_TOTAL row
rt = SiteReport.objects.create(
report_name=SiteReport.ReportName.RETIRED_TOTAL,
assets_not_started=10,
assets_published=2,
assets_started=None,
)
SiteReport.objects.filter(pk=rt.pk).update(created_on=self._dt(3))
# One per-campaign row
camp = Campaign.objects.create(title="C", slug="c")
cr = SiteReport.objects.create(
campaign=camp,
assets_not_started=7,
assets_published=1,
assets_started=None,
)
SiteReport.objects.filter(pk=cr.pk).update(created_on=self._dt(2))
# One per-topic row
topic = Topic.objects.create(title="T", slug="t")
tr = SiteReport.objects.create(
topic=topic,
assets_not_started=5,
assets_published=0,
assets_started=None,
)
SiteReport.objects.filter(pk=tr.pk).update(created_on=self._dt(1))
updated = backfill_assets_started_for_site_reports.run()
# Each single-row series sets assets_started to 0
self.assertEqual(updated, 3)
rt.refresh_from_db()
cr.refresh_from_db()
tr.refresh_from_db()
self.assertEqual(rt.assets_started, 0)
self.assertEqual(cr.assets_started, 0)
self.assertEqual(tr.assets_started, 0)
def test_skip_existing_branch_emits_heartbeat_due_to_time(self):
# First row already populated (skipped); second row needs update.
from unittest import mock
prev = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_not_started=100,
assets_published=10,
assets_started=0,
)
curr = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_not_started=90,
assets_published=15,
assets_started=None,
)
SiteReport.objects.filter(pk=prev.pk).update(created_on=self._dt(2))
SiteReport.objects.filter(pk=curr.pk).update(created_on=self._dt(1))
# Use a monotonic function that always advances time enough to trip the
# heartbeat-by-time condition, without exhausting side effects.
def make_monotonic(step=11.0):
state = {"t": 0.0}
def _mono():
state["t"] += step
return state["t"]
return _mono
with (
mock.patch("concordia.tasks.reports.backfill.structured_logger") as slog,
mock.patch(
"concordia.tasks.reports.backfill.time.monotonic",
new=make_monotonic(),
),
):
updated = backfill_assets_started_for_site_reports.run()
self.assertEqual(updated, 1)
hb_calls = [
c
for c in slog.info.call_args_list
if c.kwargs.get("event_code")
== "assets_started_backfill_series_heartbeat"
and c.kwargs.get("series") == "TOTAL"
and c.kwargs.get("scanned_rows") == 1
and c.kwargs.get("last_seen_site_report_id") == prev.id
]
self.assertTrue(hb_calls)
def test_post_scan_heartbeat_emitted_due_to_time(self):
# Single-row series where a save occurs, then a heartbeat fires due to
# elapsed time.
from unittest import mock
single = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_not_started=50,
assets_published=5,
assets_started=None,
)
SiteReport.objects.filter(pk=single.pk).update(created_on=self._dt(1))
def make_monotonic(step=11.0):
state = {"t": 0.0}
def _mono():
state["t"] += step
return state["t"]
return _mono
with (
mock.patch("concordia.tasks.reports.backfill.structured_logger") as slog,
mock.patch(
"concordia.tasks.reports.backfill.time.monotonic",
new=make_monotonic(),
),
):
updated = backfill_assets_started_for_site_reports.run()
self.assertEqual(updated, 1)
slog.info.assert_any_call(
"Scanning series...",
event_code="assets_started_backfill_series_heartbeat",
series="TOTAL",
scanned_rows=1,
updated_rows=1,
last_seen_site_report_id=single.id,
)
def test_no_update_when_equal_with_skip_existing_false(self):
# First row already equals the calculated value (zero for first snapshot),
# so no save should occur on that row when recomputing.
from unittest import mock
prev = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_not_started=100,
assets_published=10,
assets_started=0,
)
curr = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_not_started=90,
assets_published=15,
assets_started=None,
)
SiteReport.objects.filter(pk=prev.pk).update(created_on=self._dt(2))
SiteReport.objects.filter(pk=curr.pk).update(created_on=self._dt(1))
with mock.patch("concordia.tasks.reports.backfill.structured_logger") as slog:
updated = backfill_assets_started_for_site_reports.run(skip_existing=False)
self.assertEqual(updated, 1)
# Row logs should not include the first row, since it already matched.
row_logs = [
c.kwargs
for c in slog.info.call_args_list
if c.kwargs.get("event_code") == "assets_started_backfill_row"
]
self.assertTrue(any(kw.get("site_report_id") == curr.id for kw in row_logs))
self.assertFalse(
any(kw.get("site_report_id") == prev.id for kw in row_logs)
)
def test_total_assets_started_is_rolled_up_from_campaign_series(self):
# Ensure the TOTAL series does not derive assets_started from its own
# assets_total/assets_not_started deltas when campaign series data for
# the same reporting days exists.
camp = Campaign.objects.create(title="C", slug="c")
total_prev = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_total=999,
assets_not_started=999,
assets_started=None,
)
total_curr = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
assets_total=999,
assets_not_started=0,
assets_started=None,
)
c_prev = SiteReport.objects.create(
campaign=camp,
assets_total=20,
assets_not_started=20,
assets_started=None,
)
c_curr = SiteReport.objects.create(
campaign=camp,
assets_total=20,
assets_not_started=15,
assets_started=None,
)
dt_prev = self._dt(2)
dt_curr = self._dt(1)
SiteReport.objects.filter(pk=total_prev.pk).update(created_on=dt_prev)
SiteReport.objects.filter(pk=total_curr.pk).update(created_on=dt_curr)
SiteReport.objects.filter(pk=c_prev.pk).update(created_on=dt_prev)
SiteReport.objects.filter(pk=c_curr.pk).update(created_on=dt_curr)
updated = backfill_assets_started_for_site_reports.run(skip_existing=False)
self.assertEqual(updated, 4)
total_prev.refresh_from_db()
total_curr.refresh_from_db()
c_prev.refresh_from_db()
c_curr.refresh_from_db()
self.assertEqual(c_prev.assets_started, 0)
self.assertEqual(c_curr.assets_started, 5)
self.assertEqual(total_prev.assets_started, 0)
self.assertEqual(total_curr.assets_started, 5)
================================================
FILE: concordia/tests/test_tasks_reports_key_metrics.py
================================================
from datetime import date, datetime
from types import SimpleNamespace
from unittest import mock
from django.test import TestCase
from django.utils import timezone
from concordia.models import KeyMetricsReport, SiteReport
from concordia.tasks.reports.key_metrics import build_key_metrics_reports
class BuildKeyMetricsReportsTaskTests(TestCase):
def _dt(self, days_ago):
return timezone.now() - timezone.timedelta(days=days_ago)
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_recompute_all_calls_all_upserts(self, mock_localdate):
# Fix "today" to a stable mid-month date.
today = date(2024, 3, 15)
mock_localdate.return_value = today
# Earliest SiteReport in the current month so only one month is walked.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
tz = timezone.get_current_timezone()
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 3, 10, 12, 0, 0), tz)
)
fy = KeyMetricsReport.get_fiscal_year_for_date(today)
fq = KeyMetricsReport.get_fiscal_quarter_for_date(today)
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=today.replace(day=1),
period_end=today,
fiscal_year=fy,
fiscal_quarter=fq,
month=today.month,
)
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=today.replace(day=1),
period_end=today,
fiscal_year=fy,
fiscal_quarter=fq,
)
with (
mock.patch.object(KeyMetricsReport, "upsert_month") as up_m,
mock.patch.object(KeyMetricsReport, "upsert_quarter") as up_q,
mock.patch.object(KeyMetricsReport, "upsert_fiscal_year") as up_y,
):
up_m.return_value = mock.Mock(period_start=None, period_end=None)
up_q.return_value = mock.Mock(period_start=None, period_end=None)
up_y.return_value = mock.Mock(period_start=None, period_end=None)
changed = build_key_metrics_reports.run(recompute_all=True)
# One month, four quarters, one fiscal year
self.assertEqual(changed, 6)
self.assertEqual(up_m.call_count, 1)
self.assertEqual(up_q.call_count, 4)
self.assertEqual(up_y.call_count, 1)
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_incremental_refresh_and_creates(self, mock_localdate):
# Fix "today" to a stable mid-month date so the month logic is
# deterministic.
today = date(2024, 3, 15)
mock_localdate.return_value = today
# Make one SiteReport this month so the month is considered.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
tz = timezone.get_current_timezone()
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 3, 10, 12, 0, 0), tz)
)
fy = KeyMetricsReport.get_fiscal_year_for_date(today)
fq = KeyMetricsReport.get_fiscal_quarter_for_date(today)
# Existing MONTHLY row with old updated_on so it is refreshed.
monthly = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=today.replace(day=1),
period_end=today,
fiscal_year=fy,
fiscal_quarter=fq,
month=today.month,
)
KeyMetricsReport.objects.filter(pk=monthly.pk).update(
updated_on=timezone.make_aware(datetime(2024, 3, 1, 0, 0, 0), tz)
)
# Existing QUARTERLY row for the same quarter; the other three quarters
# are missing and will be created. We keep updated_on equal to the
# MONTHLY row so only the missing quarters are upserted.
quarter = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=today.replace(day=1),
period_end=today,
fiscal_year=fy,
fiscal_quarter=fq,
)
KeyMetricsReport.objects.filter(pk=quarter.pk).update(
updated_on=timezone.make_aware(datetime(2024, 3, 1, 0, 0, 0), tz)
)
# No FY row yet so it will be created in the FY stage.
with (
mock.patch.object(KeyMetricsReport, "upsert_month") as up_m,
mock.patch.object(KeyMetricsReport, "upsert_quarter") as up_q,
mock.patch.object(KeyMetricsReport, "upsert_fiscal_year") as up_y,
):
up_m.return_value = mock.Mock(period_start=None, period_end=None)
up_q.return_value = mock.Mock(period_start=None, period_end=None)
up_y.return_value = mock.Mock(period_start=None, period_end=None)
changed = build_key_metrics_reports(recompute_all=False)
# One monthly refresh, three quarterly creates (no refresh since the
# mock does not bump monthly.updated_on), and one fiscal year create.
self.assertEqual(changed, 5)
self.assertEqual(up_m.call_count, 1)
self.assertEqual(up_q.call_count, 3)
self.assertEqual(up_y.call_count, 1)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch("concordia.tasks.reports.key_metrics.SiteReport")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_early_return_after_backsteps(self, mock_local, mock_sr, slog):
# Force "today" to mid-March so last_month_start starts at Mar 1.
mock_local.return_value = date(2024, 3, 15)
# Earliest SR is mid-December so first_month_start is Dec 1.
earliest = SimpleNamespace(
created_on=timezone.make_aware(datetime(2023, 12, 15, 12, 0, 0))
)
mock_sr.objects.order_by.return_value.first.return_value = earliest
# Pretend there are no snapshots by EOM for any month we check.
mock_sr.objects.filter.return_value.exists.return_value = False
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 0)
# Ensure we logged the "no months" message.
codes = [kw.get("event_code") for _, kw in slog.info.call_args_list if kw]
self.assertIn("key_metrics_build_no_months", codes)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_recompute_all_month_upsert_and_december_rollover(
self, mock_local, upsert_month, slog
):
# Make yesterday in December so the month we process is December.
mock_local.return_value = date(2023, 12, 20)
# Create a TOTAL snapshot in December so the scan does not early-return.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2023, 12, 15, 10, 0, 0))
)
# Return a stub report so the "upserted" logging runs.
upsert_month.return_value = SimpleNamespace(
period_start=date(2023, 12, 1),
period_end=date(2023, 12, 31),
)
changed = build_key_metrics_reports(recompute_all=True)
self.assertGreaterEqual(changed, 1)
codes = [kw.get("event_code") for _, kw in slog.info.call_args_list if kw]
self.assertIn("key_metrics_month_upserted", codes)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_incremental_month_create_and_refresh(
self,
mock_local,
upsert_month,
upsert_year,
upsert_quarter,
slog,
):
mock_local.return_value = date(2024, 2, 1)
sr_jan = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr_jan.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))
)
sr_dec = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr_dec.pk).update(
created_on=timezone.make_aware(datetime(2023, 12, 20, 9, 0, 0))
)
dec_month = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2023, 12, 1),
period_end=date(2023, 12, 31),
fiscal_year=2024,
fiscal_quarter=1,
month=12,
)
KeyMetricsReport.objects.filter(pk=dec_month.pk).update(
updated_on=timezone.make_aware(datetime(2023, 12, 1, 0, 0, 0))
)
# Monthly upsert produces a stub (so it counts as 1 change per call)
upsert_month.return_value = SimpleNamespace(
period_start=date(2024, 1, 1), period_end=date(2024, 1, 31)
)
# Disable quarterly and fiscal-year increments
upsert_quarter.return_value = None
upsert_year.return_value = None
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 2)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.objects")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_quarter_recompute_all_logs(
self, mock_local, upsert_month, upsert_quarter, kmr_objects, slog
):
mock_local.return_value = date(2024, 1, 15)
# Ensure we do not early-return (one SR anywhere is fine).
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 1, 8, 0, 0))
)
# We are not using monthly upserts here.
upsert_month.return_value = None
# monthly_rows -> one fiscal year (2024)
kmr_objects.filter.return_value.values.return_value.annotate.return_value = [
{"fiscal_year": 2024}
]
# quarter_exists .first() can be anything; ignored in recompute_all.
kmr_objects.filter.return_value.first.return_value = None
# Prevent FY stage from running by returning no quarter years later.
kmr_objects.filter.return_value.values_list.return_value = []
upsert_quarter.return_value = SimpleNamespace(
period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)
)
changed = build_key_metrics_reports(recompute_all=True)
# Four quarters upserted
self.assertGreaterEqual(changed, 4)
self.assertEqual(upsert_quarter.call_count, 4)
codes = [kw.get("event_code") for _, kw in slog.info.call_args_list if kw]
self.assertIn("key_metrics_quarter_upserted", codes)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.objects")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_quarter_incremental_refresh_all_quarters(
self, mock_local, upsert_month, kmr_objects, upsert_quarter, slog
):
mock_local.return_value = date(2024, 6, 15)
# Ensure we do not early-return.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 5, 10, 8, 0, 0))
)
# No monthly creation in this test.
upsert_month.return_value = None
# Signal that we have monthly rows for fiscal_year=2024.
kmr_objects.filter.return_value.values.return_value.annotate.return_value = [
{"fiscal_year": 2024}
]
# quarter_exists present for all four quarters.
quarter_stub = SimpleNamespace(
updated_on=timezone.make_aware(datetime(2024, 1, 1, 0, 0, 0))
)
def filter_side_effect(*args, **kwargs):
# For QUARTERLY lookups with fiscal_quarter, return an object
# whose first() yields a stub so "refresh" path is taken.
class QS:
def __init__(self, exists_value=False):
self._exists = exists_value
def first(self):
return quarter_stub
def exists(self):
return self._exists
def values(self, *a, **k):
return self
def annotate(self, *a, **k):
return [{"fiscal_year": 2024}]
def values_list(self, *a, **k):
# Avoid FY stage in this test
return []
pt = kwargs.get("period_type")
if pt == KeyMetricsReport.PeriodType.MONTHLY and "updated_on__gt" in kwargs:
# Make monthly_newer_exists True
return QS(exists_value=True)
return QS()
kmr_objects.filter.side_effect = filter_side_effect
upsert_quarter.return_value = SimpleNamespace(
period_start=date(2024, 4, 1), period_end=date(2024, 6, 30)
)
changed = build_key_metrics_reports(recompute_all=False)
# Four refreshes (Q1..Q4)
self.assertGreaterEqual(changed, 4)
self.assertEqual(upsert_quarter.call_count, 4)
codes = [kw.get("event_code") for _, kw in slog.info.call_args_list if kw]
self.assertIn("key_metrics_quarter_refreshed", codes)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.objects")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_fiscal_year_recompute_all_logs(
self, mock_local, upsert_month, kmr_objects, upsert_year, slog
):
mock_local.return_value = date(2024, 1, 15)
# Ensure no early-return.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 2, 8, 0, 0))
)
upsert_month.return_value = None
# No monthlies needed; quarters present for FY 2027.
kmr_objects.filter.return_value.values.return_value.annotate.return_value = []
kmr_objects.filter.return_value.values_list.return_value = [2027]
kmr_objects.filter.return_value.first.return_value = None
upsert_year.return_value = SimpleNamespace(
period_start=date(2026, 10, 1), period_end=date(2027, 9, 30)
)
changed = build_key_metrics_reports(recompute_all=True)
self.assertGreaterEqual(changed, 1)
codes = [kw.get("event_code") for _, kw in slog.info.call_args_list if kw]
self.assertIn("key_metrics_year_upserted", codes)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.objects")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_fiscal_year_incremental_create_and_refresh(
self, mock_local, upsert_month, kmr_objects, upsert_year, slog
):
mock_local.return_value = date(2024, 5, 1)
# Ensure no early-return.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 4, 15, 8, 0, 0))
)
upsert_month.return_value = None
# First, drive "create" path: quarters exist, FY row is missing.
def filter_values_list_side_effect(*args, **kwargs):
# This handles the "fiscal_years_with_quarters" query.
class QS:
def values_list(self, *a, **k):
return [2026]
def first(self):
return None
def values(self, *a, **k):
return self
def annotate(self, *a, **k):
return []
def exists(self):
return False
return QS()
kmr_objects.filter.side_effect = filter_values_list_side_effect
upsert_year.return_value = SimpleNamespace(
period_start=date(2025, 10, 1), period_end=date(2026, 9, 30)
)
changed1 = build_key_metrics_reports(recompute_all=False)
self.assertGreaterEqual(changed1, 1)
codes1 = [kw.get("event_code") for _, kw in slog.info.call_args_list if kw]
self.assertIn("key_metrics_year_created", codes1)
# Now drive "refresh" path: FY exists, a newer quarter exists.
fy_stub = SimpleNamespace(
updated_on=timezone.make_aware(datetime(2024, 3, 1, 0, 0, 0))
)
def filter_refresh_side_effect(*args, **kwargs):
class QS:
def __init__(self, pt=None):
self.pt = pt
def values_list(self, *a, **k):
return [2026]
def first(self):
# When asking for the FY row, return a stub
return fy_stub
def values(self, *a, **k):
return self
def annotate(self, *a, **k):
return []
def exists(self):
# This is called for quarters newer than FY.updated_on
return True
return QS()
kmr_objects.filter.side_effect = filter_refresh_side_effect
upsert_year.return_value = SimpleNamespace(
period_start=date(2025, 10, 1), period_end=date(2026, 9, 30)
)
changed2 = build_key_metrics_reports(recompute_all=False)
self.assertGreaterEqual(changed2, 1)
codes2 = [kw.get("event_code") for _, kw in slog.info.call_args_list if kw]
self.assertIn("key_metrics_year_refreshed", codes2)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_recompute_all_quarter_upserts_only(
self, mock_local, mock_month, mock_quarter, mock_year, slog
):
mock_local.return_value = date(2024, 2, 1)
# Seed one site snapshot so the task has a start month (Jan 2024).
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))
)
# Seed a MONTHLY row so the quarter loop sees FY 2024 in the set.
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 1, 31),
fiscal_year=2024,
fiscal_quarter=2,
month=1,
)
# Monthly does nothing; quarter upserts return a stub; FY does nothing.
mock_month.return_value = None
mock_year.return_value = None
def quarter_stub(**kwargs):
return SimpleNamespace(
period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)
)
mock_quarter.side_effect = quarter_stub
changed = build_key_metrics_reports(recompute_all=True)
# Only quarters (4) should have counted.
self.assertEqual(changed, 4)
self.assertEqual(mock_quarter.call_count, 4)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_incremental_quarter_refresh_only(
self, mock_local, mock_month, mock_quarter, mock_year, slog
):
mock_local.return_value = date(2024, 4, 1)
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))
)
# Monthlies for Q2; newer than the quarter row we will refresh
for m in (1, 2, 3):
mr = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, m, 1),
period_end=KeyMetricsReport.month_bounds(date(2024, m, 15))[1],
fiscal_year=2024,
fiscal_quarter=2,
month=m,
)
KeyMetricsReport.objects.filter(pk=mr.pk).update(
updated_on=timezone.make_aware(datetime(2024, 3, 31, 12, 0, 0))
)
# Pre-create Q1, Q3, Q4 so they are not created by the task
for fq, ps, pe in [
(1, date(2023, 10, 1), date(2023, 12, 31)),
(3, date(2024, 4, 1), date(2024, 6, 30)),
(4, date(2024, 7, 1), date(2024, 9, 30)),
]:
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=ps,
period_end=pe,
fiscal_year=2024,
fiscal_quarter=fq,
)
# Existing Q2 with older updated_on so only this quarter refreshes
q2 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
KeyMetricsReport.objects.filter(pk=q2.pk).update(
updated_on=timezone.make_aware(datetime(2024, 1, 15, 0, 0, 0))
)
mock_month.return_value = None
mock_year.return_value = None
mock_quarter.return_value = SimpleNamespace(
period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)
)
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 1)
self.assertEqual(mock_quarter.call_count, 1)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_recompute_all_year_upsert_only(
self, mock_local, mock_month, mock_quarter, mock_year, slog
):
mock_local.return_value = date(2024, 2, 1)
# Seed snapshot to allow the task to pick a month.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))
)
# Ensure the 'fiscal_years_with_quarters' set is not empty.
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
# Monthly and quarterly stages do nothing; FY upsert returns a stub.
mock_month.return_value = None
mock_quarter.return_value = None
mock_year.return_value = SimpleNamespace(
period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)
)
changed = build_key_metrics_reports(recompute_all=True)
self.assertEqual(changed, 1)
self.assertEqual(mock_year.call_count, 1)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_incremental_year_create(
self, mock_local, mock_month, mock_quarter, mock_year, slog
):
mock_local.return_value = date(2024, 2, 1)
# Seed snapshot and a quarterly row so year loop triggers.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))
)
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
mock_month.return_value = None
mock_quarter.return_value = None
mock_year.return_value = SimpleNamespace(
period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)
)
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 1)
self.assertEqual(mock_year.call_count, 1)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_incremental_year_refresh(
self, mock_local, mock_month, mock_quarter, mock_year, slog
):
mock_local.return_value = date(2024, 4, 1)
# Seed a quarterly row with new updated_on.
q = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
KeyMetricsReport.objects.filter(pk=q.pk).update(
updated_on=timezone.make_aware(datetime(2024, 3, 31, 12, 0, 0))
)
# Create an older FY row that should be refreshed.
fy = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,
period_start=date(2023, 10, 1),
period_end=date(2024, 9, 30),
fiscal_year=2024,
)
KeyMetricsReport.objects.filter(pk=fy.pk).update(
updated_on=timezone.make_aware(datetime(2024, 1, 1, 0, 0, 0))
)
# Need a snapshot so the task can initialize months; it is not used
# further because we neutralize month and quarter stages.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))
)
mock_month.return_value = None
mock_quarter.return_value = None
mock_year.return_value = SimpleNamespace(
period_start=date(2023, 10, 1), period_end=date(2024, 9, 30)
)
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 1)
self.assertEqual(mock_year.call_count, 1)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_quarter_recompute_all_upserts_and_continue(
self,
mock_localdate,
mock_upsert_month,
mock_upsert_quarter,
mock_upsert_year,
slog,
):
# Make the "monthly" section inert (no changes).
mock_localdate.return_value = date(2024, 4, 1)
mock_upsert_month.return_value = None
mock_upsert_year.return_value = None
# Seed minimal SiteReport so the monthly stage can compute bounds safely.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))
)
# Ensure at least one fiscal_year is discovered from MONTHLY rows.
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 1, 31),
fiscal_year=2024,
fiscal_quarter=2,
month=1,
)
# Each quarter upsert returns a non-None object so rows_changed increments.
mock_upsert_quarter.return_value = SimpleNamespace(
period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)
)
changed = build_key_metrics_reports.run(recompute_all=True)
# Four quarters upserted; monthly and FY upserts return None.
self.assertEqual(changed, 4)
self.assertEqual(mock_upsert_quarter.call_count, 4)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_fiscal_year_recompute_all_upserts_and_continue(
self,
mock_localdate,
mock_upsert_month,
mock_upsert_quarter,
mock_upsert_year,
slog,
):
mock_localdate.return_value = date(2024, 4, 1)
mock_upsert_month.return_value = None
mock_upsert_quarter.return_value = None
# Seed a quarter so the FY stage finds a fiscal year to process.
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
# Earliest SiteReport so earlier stages do not error.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 5, 9, 0, 0))
)
mock_upsert_year.return_value = SimpleNamespace(
period_start=date(2023, 10, 1), period_end=date(2024, 9, 30)
)
changed = build_key_metrics_reports.run(recompute_all=True)
# Only FY upsert counts (quarter/month upserts return None).
self.assertEqual(changed, 1)
self.assertEqual(mock_upsert_year.call_count, 1)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month",
return_value=None,
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_quarter_recompute_all_non_none_continue_edge(
self, mock_localdate, mock_upsert_quarter, mock_upsert_month
):
mock_localdate.return_value = date(2024, 5, 20)
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0))
)
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 5, 1),
period_end=date(2024, 5, 31),
fiscal_year=2024,
fiscal_quarter=3,
month=5,
)
dummy = mock.MagicMock(
period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)
)
mock_upsert_quarter.return_value = dummy
changed = build_key_metrics_reports(recompute_all=True)
self.assertEqual(changed, 4)
self.assertEqual(mock_upsert_quarter.call_count, 4)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year",
return_value=mock.MagicMock(
period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)
),
)
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_quarter_incremental_refresh_monthly_newer(
self,
mock_localdate,
mock_upsert_fy,
mock_upsert_quarter,
mock_upsert_month,
):
mock_localdate.return_value = date(2024, 1, 20)
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 10, 9, 0, 0))
)
jan = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 1, 31),
fiscal_year=2024,
fiscal_quarter=2,
month=1,
)
KeyMetricsReport.objects.filter(pk=jan.pk).update(updated_on=timezone.now())
now = timezone.now()
older = now - timezone.timedelta(days=10)
q1 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2023, 10, 1),
period_end=date(2023, 12, 31),
fiscal_year=2024,
fiscal_quarter=1,
)
q2 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
q3 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 4, 1),
period_end=date(2024, 6, 30),
fiscal_year=2024,
fiscal_quarter=3,
)
q4 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 7, 1),
period_end=date(2024, 9, 30),
fiscal_year=2024,
fiscal_quarter=4,
)
KeyMetricsReport.objects.filter(pk=q1.pk).update(updated_on=now)
KeyMetricsReport.objects.filter(pk=q2.pk).update(updated_on=older)
KeyMetricsReport.objects.filter(pk=q3.pk).update(updated_on=now)
KeyMetricsReport.objects.filter(pk=q4.pk).update(updated_on=now)
fy = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,
period_start=date(2023, 10, 1),
period_end=date(2024, 9, 30),
fiscal_year=2024,
)
KeyMetricsReport.objects.filter(pk=fy.pk).update(updated_on=now)
mock_upsert_quarter.return_value = mock.MagicMock(
period_start=date(2024, 1, 1), period_end=date(2024, 3, 31)
)
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 1)
self.assertGreaterEqual(mock_upsert_quarter.call_count, 1)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year",
return_value=mock.MagicMock(
period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)
),
)
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_fiscal_year_recompute_all_non_none_continue_edge(
self, mock_localdate, mock_upsert_fy, mock_upsert_quarter, mock_upsert_month
):
mock_localdate.return_value = date(2024, 5, 20)
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0))
)
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
changed = build_key_metrics_reports(recompute_all=True)
self.assertEqual(changed, 1)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year",
return_value=mock.MagicMock(
period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)
),
)
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_fiscal_year_incremental_create_missing(
self,
mock_localdate,
mock_upsert_fy,
mock_upsert_quarter,
mock_upsert_month,
):
mock_localdate.return_value = date(2024, 5, 20)
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
tz = timezone.get_current_timezone()
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0), tz)
)
for qn, start, end in [
(1, date(2023, 10, 1), date(2023, 12, 31)),
(2, date(2024, 1, 1), date(2024, 3, 31)),
(3, date(2024, 4, 1), date(2024, 6, 30)),
(4, date(2024, 7, 1), date(2024, 9, 30)),
]:
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=start,
period_end=end,
fiscal_year=2024,
fiscal_quarter=qn,
)
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 1)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year",
return_value=mock.MagicMock(
period_start=date(2024, 10, 1), period_end=date(2025, 9, 30)
),
)
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_fiscal_year_incremental_refresh_when_quarter_newer(
self, mock_localdate, mock_upsert_fy, mock_upsert_quarter, mock_upsert_month
):
mock_localdate.return_value = date(2024, 5, 20)
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0))
)
older = timezone.now() - timezone.timedelta(days=7)
newer = timezone.now()
fy = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,
period_start=date(2023, 10, 1),
period_end=date(2024, 9, 30),
fiscal_year=2024,
)
KeyMetricsReport.objects.filter(pk=fy.pk).update(updated_on=older)
q2 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
KeyMetricsReport.objects.filter(pk=q2.pk).update(updated_on=newer)
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 1)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter",
return_value=None,
)
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_quarter_recompute_all_none_branch_continue(
self, mock_localdate, upsert_quarter, upsert_month, upsert_year, slog
):
# Keep the monthly scan minimal and stable
mock_localdate.return_value = date(2024, 2, 10)
# Seed a site snapshot so the task computes month bounds
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(
datetime(2024, 2, 9, 12, 0, 0), timezone.get_current_timezone()
)
)
# Ensure the quarterly stage iterates a fiscal year by having a MONTHLY
# row
fy = KeyMetricsReport.get_fiscal_year_for_date(mock_localdate.return_value)
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 2, 1),
period_end=date(2024, 2, 29),
fiscal_year=fy,
fiscal_quarter=2,
month=2,
)
# upsert_quarter returns None -> branch falls through to 'continue'
changed = build_key_metrics_reports(recompute_all=True)
# No rows changed because monthly and FY are neutralized and quarter
# upserts return None (hitting the continue path each time).
self.assertEqual(changed, 0)
self.assertEqual(upsert_quarter.call_count, 4)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter",
return_value=None,
)
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_quarter_incremental_refresh_none_branch_continue(
self,
mock_localdate,
mock_upsert_quarter,
mock_upsert_month,
mock_upsert_year,
slog,
):
# Ensure monthly scan has a valid window
mock_localdate.return_value = date(2024, 2, 10)
# Seed one site snapshot so month range can be computed
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
tz = timezone.get_current_timezone()
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 5, 12, 0, 0), tz)
)
# Provide a MONTHLY row in FY 2024; make it "newer" than Q2
jan = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.MONTHLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 1, 31),
fiscal_year=2024,
fiscal_quarter=2,
month=1,
)
KeyMetricsReport.objects.filter(pk=jan.pk).update(updated_on=timezone.now())
# Create quarter rows so the incremental branch runs.
# Only Q2 should be older than the monthly row to trigger refresh.
now = timezone.now()
older = now - timezone.timedelta(days=10)
quarters = {
1: ((date(2023, 10, 1), date(2023, 12, 31)), now),
2: ((date(2024, 1, 1), date(2024, 3, 31)), older),
3: ((date(2024, 4, 1), date(2024, 6, 30)), now),
4: ((date(2024, 7, 1), date(2024, 9, 30)), now),
}
for fq, val in quarters.items():
(ps, pe), updated = val
q = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=ps,
period_end=pe,
fiscal_year=2024,
fiscal_quarter=fq,
)
KeyMetricsReport.objects.filter(pk=q.pk).update(updated_on=updated)
# upsert_quarter is mocked to return None, so when the code reaches the
# monthly_newer_exists refresh path for Q2 it will take the "is None"
# branch and continue without incrementing rows_changed.
changed = build_key_metrics_reports(recompute_all=False)
# No rows changed: month and year upserts return None, and Q2 refresh
# returned None (so branch continued). Only one refresh attempt expected.
self.assertEqual(changed, 0)
self.assertEqual(mock_upsert_quarter.call_count, 1)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month",
return_value=None,
)
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_fiscal_year_recompute_all_none_branch_continue(
self,
mock_localdate,
mock_upsert_month,
mock_upsert_quarter,
mock_upsert_year,
slog,
):
mock_localdate.return_value = date(2024, 5, 20)
# Ensure monthly scan can initialize.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
tz = timezone.get_current_timezone()
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0), tz)
)
# Ensure at least one fiscal year is present for the FY stage.
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
# Month/quarter upserts are mocked to None; FY upsert also None.
changed = build_key_metrics_reports(recompute_all=True)
# Nothing should be counted since FY upsert returned None and the code
# immediately continued the loop without incrementing or logging.
self.assertEqual(changed, 0)
self.assertEqual(mock_upsert_year.call_count, 1)
codes = [kw.get("event_code") for _, kw in slog.info.call_args_list if kw]
self.assertNotIn("key_metrics_year_upserted", codes)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter",
return_value=None,
)
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month",
return_value=None,
)
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_fiscal_year_incremental_refresh_none_branch_continue(
self,
mock_localdate,
mock_upsert_month,
mock_upsert_quarter,
mock_upsert_year,
slog,
):
mock_localdate.return_value = date(2024, 5, 20)
# Make monthly stage computable.
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
tz = timezone.get_current_timezone()
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 5, 10, 12, 0, 0), tz)
)
# Existing FY row with older updated_on so a newer quarter will
# trigger the refresh path.
fy = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,
period_start=date(2023, 10, 1),
period_end=date(2024, 9, 30),
fiscal_year=2024,
)
KeyMetricsReport.objects.filter(pk=fy.pk).update(
updated_on=timezone.make_aware(datetime(2024, 3, 1, 0, 0, 0), tz)
)
# Quarter newer than the FY row to make quarter_newer_exists True.
q2 = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
KeyMetricsReport.objects.filter(pk=q2.pk).update(
updated_on=timezone.make_aware(datetime(2024, 3, 15, 0, 0, 0), tz)
)
# FY upsert returns None so the branch is skipped and loop continues.
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 0)
self.assertEqual(mock_upsert_year.call_count, 1)
codes = [kw.get("event_code") for _, kw in slog.info.call_args_list if kw]
self.assertNotIn("key_metrics_year_refreshed", codes)
self.assertNotIn("key_metrics_year_created", codes)
self.assertNotIn("key_metrics_year_upserted", codes)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_incremental_fiscal_year_created_branch(
self,
mock_localdate,
mock_upsert_month,
mock_upsert_quarter,
mock_upsert_year,
slog,
):
mock_localdate.return_value = date(2024, 4, 1)
mock_upsert_month.return_value = None
mock_upsert_quarter.return_value = None
# Quarter exists for FY discovery; no FY row exists yet.
KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 2, 9, 0, 0))
)
mock_upsert_year.return_value = SimpleNamespace(
period_start=date(2023, 10, 1), period_end=date(2024, 9, 30)
)
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 1)
self.assertEqual(mock_upsert_year.call_count, 1)
@mock.patch("concordia.tasks.reports.key_metrics.structured_logger")
@mock.patch(
"concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_fiscal_year"
)
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_quarter")
@mock.patch("concordia.tasks.reports.key_metrics.KeyMetricsReport.upsert_month")
@mock.patch("concordia.tasks.reports.key_metrics.timezone.localdate")
def test_incremental_fiscal_year_refresh_due_to_newer_quarter(
self,
mock_localdate,
mock_upsert_month,
mock_upsert_quarter,
mock_upsert_year,
slog,
):
mock_localdate.return_value = date(2024, 4, 1)
mock_upsert_month.return_value = None
mock_upsert_quarter.return_value = None
# Existing FY row with earlier updated_on.
fy = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.FISCAL_YEAR,
period_start=date(2023, 10, 1),
period_end=date(2024, 9, 30),
fiscal_year=2024,
)
KeyMetricsReport.objects.filter(pk=fy.pk).update(
updated_on=timezone.make_aware(datetime(2024, 3, 1, 0, 0, 0))
)
# Quarter with newer updated_on to trigger the refresh path.
q = KeyMetricsReport.objects.create(
period_type=KeyMetricsReport.PeriodType.QUARTERLY,
period_start=date(2024, 1, 1),
period_end=date(2024, 3, 31),
fiscal_year=2024,
fiscal_quarter=2,
)
KeyMetricsReport.objects.filter(pk=q.pk).update(
updated_on=timezone.make_aware(datetime(2024, 3, 15, 0, 0, 0))
)
sr = SiteReport.objects.create(report_name=SiteReport.ReportName.TOTAL)
SiteReport.objects.filter(pk=sr.pk).update(
created_on=timezone.make_aware(datetime(2024, 1, 3, 9, 0, 0))
)
mock_upsert_year.return_value = SimpleNamespace(
period_start=date(2023, 10, 1), period_end=date(2024, 9, 30)
)
changed = build_key_metrics_reports(recompute_all=False)
self.assertEqual(changed, 1)
self.assertEqual(mock_upsert_year.call_count, 1)
================================================
FILE: concordia/tests/test_tasks_reports_sitereport.py
================================================
from datetime import timedelta
from django.test import TestCase
from django.utils import timezone
from concordia.models import Asset, Campaign, SiteReport, Transcription
from concordia.tasks.reports.sitereport import (
_daily_active_users,
campaign_report,
retired_total_report,
site_report,
)
from concordia.utils import get_anonymous_user
from .utils import (
CreateTestUsers,
create_asset,
create_campaign,
create_item,
create_project,
create_tag,
create_tag_collection,
create_topic,
create_transcription,
)
class SiteReportTestCase(CreateTestUsers, TestCase):
@classmethod
def setUpTestData(cls):
# We use setUpTestData instead of setUp so the database is only set
# up once rather than for each individual test in this test case
cls.user1 = cls.create_user(username="tester1")
cls.user2 = cls.create_user(username="tester2")
cls.user3 = cls.create_user(username="tester3")
cls.anonymous_user = get_anonymous_user()
cls.asset1 = create_asset()
cls.item1 = cls.asset1.item
cls.project1 = cls.item1.project
cls.campaign1 = cls.project1.campaign
cls.asset1_transcription1 = create_transcription(
asset=cls.asset1, user=cls.user1, accepted=timezone.now()
)
cls.asset1_transcription2 = create_transcription(
asset=cls.asset1,
user=cls.anonymous_user,
rejected=timezone.now(),
reviewed_by=cls.user1,
)
cls.topic1 = create_topic(project=cls.project1)
cls.tag1 = create_tag()
cls.tag_collection1 = create_tag_collection(
tag=cls.tag1, asset=cls.asset1, user=cls.user1
)
cls.campaign2 = create_campaign(slug="test-campaign-slug-2")
cls.project2 = create_project(
campaign=cls.campaign2, slug="test-project-slug-2"
)
cls.item2 = create_item(project=cls.project2, item_id="2")
cls.asset2 = create_asset(item=cls.item2, slug="test-asset-slug-2")
cls.topic2 = create_topic(
project=cls.asset2.item.project, slug="test-topic-slug-2"
)
cls.tag_collection2 = create_tag_collection(
tag=cls.tag1, asset=cls.asset2, user=cls.user1
)
cls.campaign3 = create_campaign(slug="test-campaign-slug-3")
cls.project3 = create_project(
campaign=cls.campaign3, slug="test-project-slug-3"
)
cls.item3 = create_item(project=cls.project3, item_id="3")
cls.asset3 = create_asset(item=cls.item3, slug="test-asset-slug-3")
cls.asset4 = create_asset(
item=cls.item3, slug="test-asset-slug-4", published=False
)
cls.item4 = create_item(project=cls.project3, item_id="4", published=False)
cls.asset5 = create_asset(
item=cls.item4, slug="test-asset-slug-5", published=False
)
cls.project3.topics.add(cls.topic1)
cls.project3.topics.add(cls.topic2)
cls.retired_campaign = create_campaign(slug="retired-campaign-slug")
cls.retired_project = create_project(
campaign=cls.retired_campaign, slug="retired-project-slug"
)
cls.retired_item = create_item(project=cls.retired_project)
cls.retired_asset = create_asset(
item=cls.retired_item, slug="retired-asset-slug"
)
time = timezone.now() - timedelta(days=1, hours=1)
cls.retired_asset_transcription1 = create_transcription(
asset=cls.retired_asset, user=cls.user1, accepted=time
)
# Done like this to override auto_now_add and auto_now
Transcription.objects.filter(pk=cls.retired_asset_transcription1.pk).update(
created_on=time, updated_on=time
)
time = timezone.now() - timedelta(days=1, seconds=1)
cls.retired_asset_transcription2 = create_transcription(
asset=cls.retired_asset,
user=cls.user2,
rejected=time,
reviewed_by=cls.user1,
)
# Done like this to override auto_now_add and auto_now
Transcription.objects.filter(pk=cls.retired_asset_transcription2.pk).update(
created_on=time, updated_on=time
)
# Generate the campaign report before "retiring" the campaign to populate
# the retired total report
cls.retired_campaign_report = campaign_report(campaign=cls.retired_campaign)
cls.retired_asset.delete()
cls.retired_item.delete()
cls.retired_project.delete()
cls.retired_campaign.status = Campaign.Status.RETIRED
cls.retired_campaign.save()
site_report()
cls.site_report = SiteReport.objects.filter(
report_name=SiteReport.ReportName.TOTAL
).first()
cls.retired_site_report = SiteReport.objects.filter(
report_name=SiteReport.ReportName.RETIRED_TOTAL
).first()
cls.campaign1_report = SiteReport.objects.filter(campaign=cls.campaign1).first()
cls.topic1_report = SiteReport.objects.filter(topic=cls.topic1).first()
def test_daily_active_users(self):
self.assertEqual(_daily_active_users(), 2)
def test_site_report(self):
self.assertEqual(self.site_report.assets_total, 5)
self.assertEqual(self.site_report.assets_published, 3)
self.assertEqual(self.site_report.assets_not_started, 4)
self.assertEqual(self.site_report.assets_in_progress, 1)
self.assertEqual(self.site_report.assets_waiting_review, 0)
self.assertEqual(self.site_report.assets_completed, 0)
self.assertEqual(self.site_report.assets_unpublished, 2)
self.assertEqual(self.site_report.items_published, 3)
self.assertEqual(self.site_report.items_unpublished, 1)
self.assertEqual(self.site_report.projects_published, 3)
self.assertEqual(self.site_report.projects_unpublished, 0)
self.assertEqual(self.site_report.anonymous_transcriptions, 1)
self.assertEqual(self.site_report.transcriptions_saved, 2)
self.assertEqual(self.site_report.daily_review_actions, 2)
self.assertEqual(self.site_report.distinct_tags, 1)
self.assertEqual(self.site_report.tag_uses, 2)
self.assertEqual(self.site_report.campaigns_published, 4)
self.assertEqual(self.site_report.campaigns_unpublished, 0)
self.assertEqual(self.site_report.users_registered, 4)
self.assertEqual(self.site_report.users_activated, 4)
self.assertEqual(self.site_report.daily_active_users, 2)
def test_retired_site_report(self):
self.assertEqual(self.retired_site_report.assets_total, 1)
self.assertEqual(self.retired_site_report.assets_published, 1)
self.assertEqual(self.retired_site_report.assets_not_started, 0)
self.assertEqual(self.retired_site_report.assets_in_progress, 1)
self.assertEqual(self.retired_site_report.assets_waiting_review, 0)
self.assertEqual(self.retired_site_report.assets_completed, 0)
self.assertEqual(self.retired_site_report.assets_unpublished, 0)
self.assertEqual(self.retired_site_report.items_published, 1)
self.assertEqual(self.retired_site_report.items_unpublished, 0)
self.assertEqual(self.retired_site_report.projects_published, 1)
self.assertEqual(self.retired_site_report.projects_unpublished, 0)
self.assertEqual(self.retired_site_report.anonymous_transcriptions, 0)
self.assertEqual(self.retired_site_report.transcriptions_saved, 2)
self.assertEqual(self.retired_site_report.daily_review_actions, 0)
self.assertEqual(self.retired_site_report.distinct_tags, 0)
self.assertEqual(self.retired_site_report.tag_uses, 0)
self.assertEqual(self.retired_site_report.registered_contributors, 2)
def test_campaign_report(self):
self.assertEqual(self.campaign1_report.assets_total, 1)
self.assertEqual(self.campaign1_report.assets_published, 1)
self.assertEqual(self.campaign1_report.assets_not_started, 0)
self.assertEqual(self.campaign1_report.assets_in_progress, 1)
self.assertEqual(self.campaign1_report.assets_waiting_review, 0)
self.assertEqual(self.campaign1_report.assets_completed, 0)
self.assertEqual(self.campaign1_report.assets_unpublished, 0)
self.assertEqual(self.campaign1_report.items_published, 1)
self.assertEqual(self.campaign1_report.items_unpublished, 0)
self.assertEqual(self.campaign1_report.projects_published, 1)
self.assertEqual(self.campaign1_report.projects_unpublished, 0)
self.assertEqual(self.campaign1_report.anonymous_transcriptions, 1)
self.assertEqual(self.campaign1_report.transcriptions_saved, 2)
self.assertEqual(self.campaign1_report.daily_review_actions, 2)
self.assertEqual(self.campaign1_report.distinct_tags, 1)
self.assertEqual(self.campaign1_report.tag_uses, 1)
self.assertEqual(self.campaign1_report.registered_contributors, 2)
def test_topic_report(self):
self.assertEqual(self.topic1_report.assets_total, 4)
self.assertEqual(self.topic1_report.assets_published, 2)
self.assertEqual(self.topic1_report.assets_not_started, 3)
self.assertEqual(self.topic1_report.assets_in_progress, 1)
self.assertEqual(self.topic1_report.assets_waiting_review, 0)
self.assertEqual(self.topic1_report.assets_completed, 0)
self.assertEqual(self.topic1_report.assets_unpublished, 2)
self.assertEqual(self.topic1_report.items_published, 2)
self.assertEqual(self.topic1_report.items_unpublished, 1)
self.assertEqual(self.topic1_report.projects_published, 2)
self.assertEqual(self.topic1_report.projects_unpublished, 0)
self.assertEqual(self.topic1_report.anonymous_transcriptions, 1)
self.assertEqual(self.topic1_report.transcriptions_saved, 2)
self.assertEqual(self.topic1_report.daily_review_actions, 2)
self.assertEqual(self.topic1_report.distinct_tags, 1)
self.assertEqual(self.topic1_report.tag_uses, 1)
def test_topic_report_zero_assets_emits_warning(self):
# Create a new topic attached to a project with no items/assets so the
# topic report computes zero total assets and emits a warning.
from unittest import mock
empty_campaign = create_campaign(slug="sr-empty-c")
empty_project = create_project(campaign=empty_campaign, slug="sr-empty-p")
empty_topic = create_topic(project=empty_project, slug="sr-empty-t")
with mock.patch("concordia.tasks.reports.sitereport.structured_logger") as slog:
site_report()
warn_calls = [
c
for c in slog.warning.call_args_list
if c.kwargs.get("event_code") == "topic_report_zero_assets"
and c.kwargs.get("topic") == empty_topic
]
self.assertTrue(warn_calls)
class SiteReportAssetsStartedRollupTests(CreateTestUsers, TestCase):
def test_total_assets_started_rolls_up_campaign_deltas_ignoring_retirements(
self,
):
"""
The TOTAL assets_started value should be derived from per-campaign
daily deltas for the same reporting day.
This protects the site-wide daily count from being suppressed when a
campaign retires and its already-started assets are removed from the
active asset tables.
"""
from unittest import mock
active_campaign = create_campaign(slug="rollup-active-c")
retiring_campaign = create_campaign(slug="rollup-retiring-c")
active_project = create_project(
campaign=active_campaign, slug="rollup-active-p"
)
active_item = create_item(project=active_project, item_id="ra")
active_asset = create_asset(item=active_item, slug="rollup-active-a")
retiring_project = create_project(
campaign=retiring_campaign, slug="rollup-retiring-p"
)
retiring_item = create_item(project=retiring_project, item_id="rb")
retiring_asset = create_asset(item=retiring_item, slug="rollup-retiring-a")
# Day 1 snapshot: one not-started asset in the active campaign and one
# already-started asset in the campaign that will retire.
Asset.objects.filter(pk=active_asset.pk).update(
transcription_status="not_started"
)
Asset.objects.filter(pk=retiring_asset.pk).update(
transcription_status="in_progress"
)
base_now = timezone.now()
day1 = base_now - timedelta(days=2)
day2 = base_now - timedelta(days=1)
with mock.patch("django.utils.timezone.now", return_value=day1):
site_report()
# Between snapshots, the active campaign starts its asset.
Asset.objects.filter(pk=active_asset.pk).update(
transcription_status="in_progress"
)
# The other campaign is retired and its content is removed.
retiring_asset.delete()
retiring_item.delete()
retiring_project.delete()
retiring_campaign.status = Campaign.Status.RETIRED
retiring_campaign.save()
with mock.patch("django.utils.timezone.now", return_value=day2):
site_report()
total_day2 = (
SiteReport.objects.filter(
report_name=SiteReport.ReportName.TOTAL,
campaign__isnull=True,
topic__isnull=True,
created_on__date=day2.date(),
)
.order_by("-created_on", "-pk")
.first()
)
active_day2 = (
SiteReport.objects.filter(
campaign=active_campaign, created_on__date=day2.date()
)
.order_by("-created_on", "-pk")
.first()
)
self.assertIsNotNone(total_day2)
self.assertIsNotNone(active_day2)
self.assertEqual(active_day2.assets_started, 1)
self.assertEqual(total_day2.assets_started, 1)
def test_retired_total_assets_started_is_always_zero(self):
retired_campaign = create_campaign(slug="rollup-retired-c")
retired_campaign.status = Campaign.Status.RETIRED
retired_campaign.save()
r1 = SiteReport.objects.create(
campaign=retired_campaign,
assets_total=10,
assets_not_started=0,
assets_started=4,
)
r2 = SiteReport.objects.create(
campaign=retired_campaign,
assets_total=10,
assets_not_started=0,
assets_started=7,
)
now = timezone.now()
SiteReport.objects.filter(pk=r1.pk).update(created_on=now - timedelta(days=2))
SiteReport.objects.filter(pk=r2.pk).update(created_on=now - timedelta(days=1))
self.assertEqual(SiteReport.objects.get(pk=r2.pk).assets_started, 7)
retired_total_report()
retired_total = (
SiteReport.objects.filter(report_name=SiteReport.ReportName.RETIRED_TOTAL)
.order_by("-created_on", "-pk")
.first()
)
self.assertIsNotNone(retired_total)
self.assertEqual(retired_total.assets_started, 0)
================================================
FILE: concordia/tests/test_tasks_retirement.py
================================================
from unittest import mock
from django.core.exceptions import ObjectDoesNotExist
from django.test import TestCase
from concordia.models import Asset, Project
from concordia.tasks.retirement import (
assets_removal_success,
delete_asset,
item_removal_success,
project_removal_success,
remove_next_assets,
remove_next_item,
remove_next_project,
retire_campaign,
)
from .utils import (
create_asset,
create_campaign,
create_campaign_retirement_progress,
create_item,
create_project,
)
class RetirementTasksTests(TestCase):
def test_retire_campaign_initializes_totals_and_sets_status_and_triggers(self):
# Build a campaign with 2 projects, 2 items, 3 assets.
camp = create_campaign(slug="ret-c1")
p1 = create_project(campaign=camp, slug="ret-p1")
p2 = create_project(campaign=camp, slug="ret-p2")
i1 = create_item(project=p1, item_id="ret-i1")
i2 = create_item(project=p2, item_id="ret-i2")
a1 = create_asset(item=i1, slug="ret-a1")
a2 = create_asset(item=i1, slug="ret-a2")
a3 = create_asset(item=i2, slug="ret-a3")
self.assertTrue(all([a1.pk, a2.pk, a3.pk]))
with mock.patch(
"concordia.tasks.retirement.remove_next_project.delay"
) as m_delay:
prog = retire_campaign(camp.id)
prog.refresh_from_db()
self.assertEqual(prog.project_total, 2)
self.assertEqual(prog.item_total, 2)
self.assertEqual(prog.asset_total, 3)
camp.refresh_from_db()
# Status must be set to RETIRED.
self.assertEqual(camp.status, camp.Status.RETIRED) # type: ignore[attr-defined]
m_delay.assert_called_once_with(camp.id)
def test_retire_campaign_existing_progress_and_already_retired(self):
camp = create_campaign(slug="ret-c2")
# Pre-create progress so the totals branch is skipped.
prog = create_campaign_retirement_progress(campaign=camp)
prog.project_total = 7
prog.item_total = 8
prog.asset_total = 9
prog.save()
# Mark campaign retired to skip status change.
camp.status = camp.Status.RETIRED # type: ignore[attr-defined]
camp.save()
with mock.patch(
"concordia.tasks.retirement.remove_next_project.delay"
) as m_delay:
retire_campaign(camp.id)
prog.refresh_from_db()
self.assertEqual(prog.project_total, 7)
self.assertEqual(prog.item_total, 8)
self.assertEqual(prog.asset_total, 9)
camp.refresh_from_db()
self.assertEqual(camp.status, camp.Status.RETIRED) # type: ignore[attr-defined]
m_delay.assert_called_once_with(camp.id)
def test_remove_next_project_calls_remove_next_item_when_project_exists(self):
camp = create_campaign(slug="ret-c3")
proj = create_project(campaign=camp, slug="ret-p3")
create_campaign_retirement_progress(campaign=camp)
with mock.patch("concordia.tasks.retirement.remove_next_item.delay") as m_delay:
remove_next_project(camp.id)
m_delay.assert_called_once_with(proj.id)
def test_remove_next_project_marks_complete_when_no_projects(self):
camp = create_campaign(slug="ret-c4")
prog = create_campaign_retirement_progress(campaign=camp)
with mock.patch("concordia.tasks.retirement.remove_next_item.delay") as m_delay:
remove_next_project(camp.id)
prog.refresh_from_db()
self.assertTrue(prog.complete)
self.assertIsNotNone(prog.completed_on)
m_delay.assert_not_called()
def test_project_removal_success_increments_and_triggers_next(self):
camp = create_campaign(slug="ret-c5")
prog = create_campaign_retirement_progress(campaign=camp)
self.assertEqual(prog.projects_removed, 0)
with mock.patch(
"concordia.tasks.retirement.remove_next_project.delay"
) as m_delay:
project_removal_success(project_id=123, campaign_id=camp.id)
prog.refresh_from_db()
self.assertEqual(prog.projects_removed, 1)
self.assertTrue(any(e.get("id") == 123 for e in prog.removal_log))
m_delay.assert_called_once_with(camp.id)
def test_remove_next_item_calls_remove_next_assets_when_item_exists(self):
camp = create_campaign(slug="ret-c6")
proj = create_project(campaign=camp, slug="ret-p6")
itm = create_item(project=proj, item_id="ret-i6")
with mock.patch(
"concordia.tasks.retirement.remove_next_assets.delay"
) as m_delay:
remove_next_item(proj.id)
m_delay.assert_called_once_with(itm.id)
def test_remove_next_item_deletes_project_and_triggers_when_no_items(self):
camp = create_campaign(slug="ret-c7")
proj = create_project(campaign=camp, slug="ret-p7")
with mock.patch(
"concordia.tasks.retirement.project_removal_success.delay"
) as m_delay:
remove_next_item(proj.id)
with self.assertRaises(ObjectDoesNotExist):
Project.objects.get(pk=proj.id)
m_delay.assert_called_once_with(proj.id, camp.id)
def test_assets_removal_success_updates_counts_and_triggers_next(self):
camp = create_campaign(slug="ret-c8")
prog = create_campaign_retirement_progress(campaign=camp)
self.assertEqual(prog.assets_removed, 0)
with mock.patch(
"concordia.tasks.retirement.remove_next_assets.delay"
) as m_delay:
assets_removal_success([10, 11, 12], campaign_id=camp.id, item_id=55)
prog.refresh_from_db()
self.assertEqual(prog.assets_removed, 3)
self.assertTrue(any(e.get("id") == 10 for e in prog.removal_log))
self.assertTrue(any(e.get("id") == 11 for e in prog.removal_log))
self.assertTrue(any(e.get("id") == 12 for e in prog.removal_log))
m_delay.assert_called_once_with(55)
def test_remove_next_assets_when_no_assets_deletes_item_and_triggers(self):
camp = create_campaign(slug="ret-c9")
proj = create_project(campaign=camp, slug="ret-p9")
itm = create_item(project=proj, item_id="ret-i9")
with mock.patch(
"concordia.tasks.retirement.item_removal_success.delay"
) as m_delay:
remove_next_assets(itm.id)
with self.assertRaises(ObjectDoesNotExist):
# Item should be deleted.
type(itm).objects.get(pk=itm.id) # type: ignore[attr-defined]
m_delay.assert_called_once_with(itm.id, camp.id, proj.id)
def test_remove_next_assets_with_assets_uses_chord_in_chunks_of_10(self):
camp = create_campaign(slug="ret-c10")
proj = create_project(campaign=camp, slug="ret-p10")
itm = create_item(project=proj, item_id="ret-i10")
# Create 12 assets; only 10 should be in the chord header.
ids = []
for n in range(12):
a = create_asset(item=itm, slug=f"ret-a10-{n}", sequence=n)
ids.append(a.id)
first_ten = list(
Asset.objects.filter(item=itm)
.order_by("id")
.values_list("id", flat=True)[:10]
)
with (
mock.patch("concordia.tasks.retirement.chord") as m_chord,
mock.patch("concordia.tasks.retirement.delete_asset.s") as m_del_sig,
mock.patch(
"concordia.tasks.retirement.assets_removal_success.s"
) as m_body_sig,
):
runner = mock.MagicMock()
m_chord.return_value = runner
m_del_sig.side_effect = lambda aid: f"S({aid})"
m_body_sig.return_value = "BODY"
remove_next_assets(itm.id)
# Header should contain exactly 10 signatures, matching first ten ids.
header_iter = m_chord.call_args[0][0]
header_list = list(header_iter)
self.assertEqual(header_list, [f"S({aid})" for aid in first_ten])
# The body signature should be called with campaign and item ids.
m_body_sig.assert_called_once_with(camp.id, itm.id)
runner.assert_called_once_with("BODY")
def test_delete_asset_deletes_storage_and_model_and_returns_id(self):
itm = create_item(item_id="ret-i11")
a = create_asset(item=itm, slug="ret-a11", sequence=11)
with mock.patch("django.core.files.storage.FileSystemStorage.delete") as m_del:
ret_id = delete_asset(a.id)
self.assertEqual(ret_id, a.id)
self.assertFalse(Asset.objects.filter(pk=a.id).exists())
m_del.assert_called()
def test_item_removal_success_increments_and_triggers_next(self):
camp = create_campaign(slug="ret-c12")
proj = create_project(campaign=camp, slug="ret-p12")
itm = create_item(project=proj, item_id="ret-i12")
prog = create_campaign_retirement_progress(campaign=camp)
self.assertEqual(prog.items_removed, 0)
with mock.patch("concordia.tasks.retirement.remove_next_item.delay") as m_delay:
item_removal_success(
item_id=itm.id, campaign_id=camp.id, project_id=proj.id
)
prog.refresh_from_db()
self.assertEqual(prog.items_removed, 1)
self.assertTrue(
any(
entry.get("type") == "item" and entry.get("id") == itm.id
for entry in prog.removal_log
)
)
m_delay.assert_called_once_with(proj.id)
================================================
FILE: concordia/tests/test_tasks_search_index.py
================================================
from unittest import mock
from django.test import TestCase
from concordia.tasks.search_index import (
create_opensearch_indices,
delete_opensearch_indices,
populate_opensearch_assets_indices,
populate_opensearch_indices,
populate_opensearch_users_indices,
rebuild_opensearch_indices,
)
class SearchIndexTasksTests(TestCase):
def test_create_opensearch_indices_calls_management_command(self):
with mock.patch("concordia.tasks.search_index.call_command") as m_call:
result = create_opensearch_indices()
self.assertIsNone(result)
m_call.assert_called_once_with(
"opensearch",
"index",
"create",
verbosity=2,
force=True,
ignore_error=True,
)
def test_delete_opensearch_indices_calls_management_command(self):
with mock.patch("concordia.tasks.search_index.call_command") as m_call:
result = delete_opensearch_indices()
self.assertIsNone(result)
m_call.assert_called_once_with(
"opensearch", "index", "delete", force=True, ignore_error=True
)
def test_rebuild_opensearch_indices_calls_management_command(self):
with mock.patch("concordia.tasks.search_index.call_command") as m_call:
result = rebuild_opensearch_indices()
self.assertIsNone(result)
m_call.assert_called_once_with(
"opensearch",
"index",
"rebuild",
verbosity=2,
force=True,
ignore_error=True,
)
def test_populate_users_indices_calls_management_command(self):
with mock.patch("concordia.tasks.search_index.call_command") as m_call:
result = populate_opensearch_users_indices()
self.assertIsNone(result)
m_call.assert_called_once_with(
"opensearch",
"document",
"index",
"--indices",
"users",
"--force",
"--parallel",
)
def test_populate_assets_indices_calls_management_command(self):
with mock.patch("concordia.tasks.search_index.call_command") as m_call:
result = populate_opensearch_assets_indices()
self.assertIsNone(result)
m_call.assert_called_once_with(
"opensearch",
"document",
"index",
"--indices",
"assets",
"--force",
"--parallel",
)
def test_populate_all_indices_calls_management_command(self):
with mock.patch("concordia.tasks.search_index.call_command") as m_call:
result = populate_opensearch_indices()
self.assertIsNone(result)
m_call.assert_called_once_with(
"opensearch", "document", "index", "--force", "--parallel"
)
================================================
FILE: concordia/tests/test_tasks_thumbnails.py
================================================
from unittest import mock
from django.test import TestCase
from concordia.tasks.thumbnails import (
download_item_thumbnail_task,
download_missing_thumbnails_task,
)
from .utils import create_campaign, create_item, create_project
class ThumbnailsTasksTests(TestCase):
def test_download_item_thumbnail_task_returns_skip_when_no_url(self):
# Item has no thumbnail_url; task should return skip message.
proj = create_project(campaign=create_campaign(slug="t-c1"), slug="t-p1")
item = create_item(project=proj, item_id="t-i1", thumbnail_url="")
with mock.patch("importer.tasks.items.download_and_set_item_thumbnail") as m_dl:
result = download_item_thumbnail_task.run(item.id, force=False)
self.assertEqual(result, "No thumbnail URL available.")
m_dl.assert_not_called()
def test_download_item_thumbnail_task_calls_helper_with_force(self):
# Item has a thumbnail_url; helper should be called with force flag.
proj = create_project(campaign=create_campaign(slug="t-c2"), slug="t-p2")
item = create_item(
project=proj,
item_id="t-i2",
thumbnail_url="https://ex.invalid/t.jpg",
thumbnail_image="",
)
with mock.patch("importer.tasks.items.download_and_set_item_thumbnail") as m_dl:
m_dl.return_value = "stored/path/t.jpg"
result = download_item_thumbnail_task.run(item.id, force=True)
self.assertEqual(result, "stored/path/t.jpg")
m_dl.assert_called_once()
# First arg is the Item instance, second the source URL.
args, kwargs = m_dl.call_args
self.assertEqual(args[0].id, item.id)
self.assertEqual(args[1], "https://ex.invalid/t.jpg")
self.assertTrue(kwargs.get("force"))
def test_download_missing_thumbnails_task_returns_zero_when_none(self):
# No items meet the filter; should log and return 0, no group calls.
with mock.patch("concordia.tasks.thumbnails.group") as m_group:
count = download_missing_thumbnails_task.run()
self.assertEqual(count, 0)
m_group.assert_not_called()
def test_download_missing_thumbnails_task_filters_and_batches_once(self):
from unittest import mock
camp = create_campaign(slug="t-c3")
proj_a = create_project(campaign=camp, slug="t-p3a")
proj_b = create_project(campaign=camp, slug="t-p3b")
i1 = create_item(
project=proj_a,
item_id="t-i3-1",
thumbnail_url="http://example.com/img1.jpg",
thumbnail_image="",
)
i2 = create_item(
project=proj_a,
item_id="t-i3-2",
thumbnail_url="http://example.com/img2.jpg",
thumbnail_image="",
)
create_item( # wrong project -> not eligible
project=proj_b,
item_id="t-i3-3",
thumbnail_url="http://example.com/img3.jpg",
thumbnail_image="",
)
create_item( # already has image -> not eligible
project=proj_a,
item_id="t-i3-4",
thumbnail_url="http://example.com/img4.jpg",
thumbnail_image="has-file",
)
with (
mock.patch("concordia.tasks.thumbnails.group") as m_group,
mock.patch(
"concordia.tasks.thumbnails.download_item_thumbnail_task.s"
) as m_sig,
):
runner = mock.MagicMock()
runner.apply_async.return_value.get.return_value = None
def fake_group(header_iter):
# Force generator evaluation so .s(...) is actually called.
list(header_iter)
return runner
m_group.side_effect = fake_group
count = download_missing_thumbnails_task.run(
project_id=proj_a.id, batch_size=2, limit=10, force=True
)
self.assertEqual(count, 2)
m_sig.assert_has_calls(
[mock.call(i1.id, force=True), mock.call(i2.id, force=True)],
any_order=False,
)
m_group.assert_called_once()
runner.apply_async.assert_called_once()
runner.apply_async.return_value.get.assert_called_once_with(
disable_sync_subtasks=False
)
def test_download_missing_thumbnails_task_multiple_waves(self):
from unittest import mock
camp = create_campaign(slug="t-c4")
proj = create_project(campaign=camp, slug="t-p4")
items = [
create_item(
project=proj,
item_id=f"t-i4-{n}",
thumbnail_url=f"http://example.com/{n}.jpg",
thumbnail_image="",
)
for n in range(5)
]
with (
mock.patch("concordia.tasks.thumbnails.group") as m_group,
mock.patch(
"concordia.tasks.thumbnails.download_item_thumbnail_task.s"
) as m_sig,
):
runners = [mock.MagicMock() for _ in range(3)]
for r in runners:
r.apply_async.return_value.get.return_value = None
it = iter(runners)
def fake_group(header_iter):
# Force generator consumption each wave.
list(header_iter)
return next(it)
m_group.side_effect = fake_group
count = download_missing_thumbnails_task.run(
project_id=proj.id, batch_size=2, limit=None, force=False
)
self.assertEqual(count, 5)
self.assertEqual(m_group.call_count, 3)
expected = [mock.call(itm.id, force=False) for itm in items]
self.assertEqual(m_sig.call_args_list, expected)
================================================
FILE: concordia/tests/test_tasks_unusualactivity.py
================================================
from datetime import UTC, datetime, timedelta
from unittest import mock
from django.test import TestCase, override_settings
from django.utils import timezone
from concordia.tasks.unusualactivity import unusual_activity
class UnusualActivityTaskTests(TestCase):
@override_settings(CONCORDIA_ENVIRONMENT="development")
def test_noop_when_not_production_and_not_ignored(self):
# Should not render templates or send mail.
with (
mock.patch(
"concordia.tasks.unusualactivity.loader.get_template"
) as m_get_tmpl,
mock.patch(
"concordia.tasks.unusualactivity.EmailMultiAlternatives"
) as m_email,
mock.patch(
"concordia.tasks.unusualactivity.Transcription.objects"
) as m_mgr,
):
unusual_activity(ignore_env=False)
m_get_tmpl.assert_not_called()
m_email.assert_not_called()
# Manager methods should not be called either.
self.assertFalse(m_mgr.transcribe_incidents.called)
self.assertFalse(m_mgr.review_incidents.called)
@override_settings(
CONCORDIA_ENVIRONMENT="production",
DEFAULT_FROM_EMAIL="noreply@example.com",
DEFAULT_TO_EMAIL="",
)
def test_runs_in_production_without_default_to(self):
# Executes, builds subject without env suffix, and sends to one addr.
fixed_now_dt = timezone.make_aware(datetime(2025, 1, 1, 12, 0), timezone=UTC)
expected_one_day_ago = fixed_now_dt - timedelta(days=1)
with (
mock.patch(
"concordia.tasks.unusualactivity.Site.objects.get_current"
) as m_site,
mock.patch(
"concordia.tasks.unusualactivity.timezone.localtime"
) as m_localtime,
mock.patch(
"concordia.tasks.unusualactivity.timezone.now",
return_value=fixed_now_dt,
),
mock.patch(
"concordia.tasks.unusualactivity.loader.get_template"
) as m_get_tmpl,
mock.patch(
"concordia.tasks.unusualactivity.EmailMultiAlternatives"
) as m_email,
mock.patch(
"concordia.tasks.unusualactivity.Transcription.objects"
) as m_mgr,
):
lt = mock.Mock()
lt.strftime.return_value = "STAMP"
m_localtime.return_value = lt
m_site.return_value = mock.Mock(domain="example.com")
txt_tmpl = mock.Mock()
html_tmpl = mock.Mock()
txt_tmpl.render.return_value = "TEXT"
html_tmpl.render.return_value = "HTML"
m_get_tmpl.side_effect = lambda name: (
txt_tmpl if name.endswith(".txt") else html_tmpl
)
m_mgr.transcribe_incidents.return_value = []
m_mgr.review_incidents.return_value = []
msg = mock.Mock()
m_email.return_value = msg
unusual_activity(ignore_env=False)
expected_subject = "Unusual User Activity Report for STAMP"
args, kwargs = m_email.call_args
self.assertEqual(kwargs["subject"], expected_subject)
self.assertEqual(kwargs["from_email"], "noreply@example.com")
self.assertEqual(kwargs["to"], ["rsar@loc.gov"])
self.assertEqual(kwargs["reply_to"], ["noreply@example.com"])
txt_tmpl.render.assert_called_once()
html_tmpl.render.assert_called_once()
self.assertEqual(
m_mgr.transcribe_incidents.call_args[0][0], expected_one_day_ago
)
self.assertEqual(m_mgr.review_incidents.call_args[0][0], expected_one_day_ago)
msg.attach_alternative.assert_called_once_with("HTML", "text/html")
msg.send.assert_called_once()
@override_settings(
CONCORDIA_ENVIRONMENT="test",
DEFAULT_FROM_EMAIL="notify@example.com",
DEFAULT_TO_EMAIL="extra@example.com",
)
def test_ignore_env_appends_suffix_and_includes_default_to(self):
fixed_now_dt = timezone.make_aware(datetime(2025, 1, 2, 9, 30), timezone=UTC)
expected_one_day_ago = fixed_now_dt - timedelta(days=1)
with (
mock.patch(
"concordia.tasks.unusualactivity.Site.objects.get_current"
) as m_site,
mock.patch(
"concordia.tasks.unusualactivity.timezone.localtime"
) as m_localtime,
mock.patch(
"concordia.tasks.unusualactivity.timezone.now",
return_value=fixed_now_dt,
),
mock.patch(
"concordia.tasks.unusualactivity.loader.get_template"
) as m_get_tmpl,
mock.patch(
"concordia.tasks.unusualactivity.EmailMultiAlternatives"
) as m_email,
mock.patch(
"concordia.tasks.unusualactivity.Transcription.objects"
) as m_mgr,
):
lt = mock.Mock()
lt.strftime.return_value = "STAMP2"
m_localtime.return_value = lt
m_site.return_value = mock.Mock(domain="example.net")
txt_tmpl = mock.Mock()
html_tmpl = mock.Mock()
txt_tmpl.render.return_value = "TEXT2"
html_tmpl.render.return_value = "HTML2"
m_get_tmpl.side_effect = lambda name: (
txt_tmpl if name.endswith(".txt") else html_tmpl
)
m_mgr.transcribe_incidents.return_value = []
m_mgr.review_incidents.return_value = []
msg = mock.Mock()
m_email.return_value = msg
unusual_activity(ignore_env=True)
expected_subject = "Unusual User Activity Report for STAMP2 [TEST]"
args, kwargs = m_email.call_args
self.assertEqual(kwargs["subject"], expected_subject)
self.assertEqual(kwargs["to"], ["rsar@loc.gov", "extra@example.com"])
self.assertEqual(kwargs["from_email"], "notify@example.com")
self.assertEqual(kwargs["reply_to"], ["notify@example.com"])
txt_tmpl.render.assert_called_once()
html_tmpl.render.assert_called_once()
self.assertEqual(
m_mgr.transcribe_incidents.call_args[0][0], expected_one_day_ago
)
self.assertEqual(m_mgr.review_incidents.call_args[0][0], expected_one_day_ago)
msg.attach_alternative.assert_called_once_with("HTML2", "text/html")
msg.send.assert_called_once()
================================================
FILE: concordia/tests/test_tasks_useractivity.py
================================================
from unittest import mock
from django.core import mail
from django.core.cache import cache
from django.test import TestCase
from concordia.exceptions import CacheLockedError
from concordia.models import Campaign, Transcription, UserProfileActivity
from concordia.tasks.unusualactivity import unusual_activity
from concordia.tasks.useractivity import (
populate_active_campaign_counts,
populate_completed_campaign_counts,
update_useractivity_cache,
update_userprofileactivity_from_cache,
)
from concordia.utils import get_anonymous_user
from .utils import (
CreateTestUsers,
create_asset,
create_campaign,
create_item,
create_project,
create_tag,
create_tag_collection,
create_transcription,
)
class UserActivityTaskTestCase(CreateTestUsers, TestCase):
def setUp(self):
cache.clear()
self.user = self.create_test_user()
self.campaign = create_campaign()
self.key = f"userprofileactivity_{self.campaign.pk}"
@mock.patch("concordia.tasks.useractivity.update_userprofileactivity_table")
def test_update_userprofileactivity_from_cache_no_updates(self, mock_update_table):
cache.set(self.key, None)
with mock.patch("concordia.logging.ConcordiaLogger.debug") as mock_debug:
update_userprofileactivity_from_cache()
self.assertEqual(mock_debug.call_count, 2)
mock_debug.assert_called_with(
"Cache contained no updates for key. Skipping",
event_code="update_userprofileactivity_from_cache_no_updates",
key=self.key,
)
self.assertEqual(mock_update_table.call_count, 0)
@mock.patch("concordia.tasks.useractivity.update_userprofileactivity_table")
def test_update_userprofileactivity_from_cache_update(self, mock_update_table):
cache.set(self.key, {self.user.pk: (1, 0)})
update_userprofileactivity_from_cache()
self.assertEqual(mock_update_table.call_count, 2)
mock_update_table.assert_has_calls(
[
mock.call(self.user, self.campaign.id, "transcribe_count", 1),
mock.call(self.user, self.campaign.id, "review_count", 0),
]
)
self.assertIsNone(cache.get(self.key))
@mock.patch("concordia.tasks.unusualactivity.Transcription.objects")
def test_unusual_activity(self, mock_transcription):
mock_transcription.transcribe_incidents.return_value = (
Transcription.objects.none()
)
mock_transcription.review_incidents.return_value = Transcription.objects.none()
unusual_activity(ignore_env=True)
self.assertEqual(len(mail.outbox), 1)
expected_subject = "Unusual User Activity Report"
self.assertIn(expected_subject, mail.outbox[0].subject)
@mock.patch("django.core.cache.cache.add")
@mock.patch("django.core.cache.cache.delete")
@mock.patch("concordia.tasks.useractivity._update_useractivity_cache")
def test_update_useractivity_cache(self, mock_update, mock_delete, mock_add):
user = self.user
campaign = self.campaign
mock_add.return_value = False
with self.assertRaises(CacheLockedError):
update_useractivity_cache(user.id, campaign.id, "transcribe")
self.assertEqual(mock_update.call_count, 0)
self.assertEqual(mock_delete.call_count, 0)
mock_add.return_value = True
update_useractivity_cache(user.id, campaign.id, "transcribe")
self.assertEqual(mock_update.call_count, 1)
mock_update.assert_called_with(user.id, campaign.id, "transcribe")
self.assertEqual(mock_delete.call_count, 1)
mock_delete.assert_called_with("userprofileactivity_cache_lock")
update_useractivity_cache(user.id, campaign.id, "review")
self.assertEqual(mock_update.call_count, 2)
mock_update.assert_called_with(user.id, campaign.id, "review")
self.assertEqual(mock_delete.call_count, 2)
mock_delete.assert_called_with("userprofileactivity_cache_lock")
def test_populate_active_campaign_counts_computes_user_and_anon_rows(self):
camp = create_campaign(slug="ua-camp-a")
proj = create_project(campaign=camp, slug="ua-proj-a")
item = create_item(project=proj, item_id="ua-item-a")
a1 = create_asset(item=item, slug="ua-a1", sequence=1)
a2 = create_asset(item=item, slug="ua-a2", sequence=2)
u1 = self.create_test_user("ua-u1")
u2 = self.create_test_user("ua-u2")
anon = get_anonymous_user()
create_transcription(asset=a1, user=u1, reviewed_by=u2)
create_transcription(asset=a2, user=u2, reviewed_by=u1)
create_transcription(asset=a1, user=anon)
create_transcription(asset=a2, user=u2, reviewed_by=anon)
t1 = create_tag(value="ua-t1")
t2 = create_tag(value="ua-t2")
create_tag_collection(tag=t1, asset=a1, user=u1)
create_tag_collection(tag=t2, asset=a2, user=u1)
create_tag_collection(tag=t1, asset=a2, user=anon)
populate_active_campaign_counts.run()
rows = UserProfileActivity.objects.filter(campaign=camp)
self.assertEqual(rows.count(), 3)
r_u1 = rows.get(user=u1)
r_u2 = rows.get(user=u2)
r_an = rows.get(user=anon)
self.assertEqual(r_u1.asset_count, 2)
self.assertEqual(r_u1.asset_tag_count, 2)
self.assertEqual(r_u1.transcribe_count, 1)
self.assertEqual(r_u1.review_count, 1)
self.assertEqual(r_u2.asset_count, 2)
self.assertEqual(r_u2.asset_tag_count, 0)
self.assertEqual(r_u2.transcribe_count, 2)
self.assertEqual(r_u2.review_count, 1)
self.assertEqual(r_an.asset_count, 2)
self.assertEqual(r_an.asset_tag_count, 1)
self.assertEqual(r_an.transcribe_count, 1)
self.assertEqual(r_an.review_count, 1)
def test_populate_completed_campaign_counts_processes_non_active_only(self):
active = create_campaign(slug="ua-act-1")
p1 = create_project(campaign=active, slug="ua-act-proj")
it1 = create_item(project=p1, item_id="ua-act-item")
a_act = create_asset(item=it1, slug="ua-act-a")
u_act = self.create_test_user("ua-act-u")
create_transcription(asset=a_act, user=u_act)
retired = create_campaign(slug="ua-ret-1", status=Campaign.Status.RETIRED)
p2 = create_project(campaign=retired, slug="ua-ret-proj")
it2 = create_item(project=p2, item_id="ua-ret-item")
a_ret = create_asset(item=it2, slug="ua-ret-a")
u_ret = self.create_test_user("ua-ret-u")
create_transcription(asset=a_ret, user=u_ret)
populate_completed_campaign_counts.run()
self.assertFalse(UserProfileActivity.objects.filter(campaign=active).exists())
self.assertTrue(UserProfileActivity.objects.filter(campaign=retired).exists())
def test_update_useractivity_cache_lock_max_retries_sends_email(self):
with (
mock.patch("django.core.cache.cache.add", return_value=False),
mock.patch.object(update_useractivity_cache, "max_retries", 0, create=True),
mock.patch("concordia.tasks.useractivity.send_mail") as m_send,
mock.patch("concordia.logging.ConcordiaLogger.warning") as m_warn,
mock.patch("concordia.logging.ConcordiaLogger.exception") as m_exc,
mock.patch("concordia.tasks.useractivity.logger.error") as m_err,
):
with self.assertRaises(CacheLockedError):
update_useractivity_cache.run(
self.user.id, self.campaign.id, "transcribe"
)
# Structured logs were emitted
self.assertTrue(m_warn.called)
self.assertTrue(m_exc.called)
# Email sent with expected subject
self.assertTrue(m_send.called)
sent_args, sent_kwargs = m_send.call_args
self.assertEqual(
sent_args[0],
"Task update_useractivity_cache failed: cache is locked.",
)
# Unstructured error log emitted
self.assertTrue(m_err.called)
def test_update_useractivity_cache_update_exception_releases_lock(self):
with (
mock.patch("concordia.tasks.useractivity.cache.add", return_value=True),
mock.patch("concordia.tasks.useractivity.cache.delete") as m_del,
mock.patch(
"concordia.tasks.useractivity._update_useractivity_cache",
side_effect=RuntimeError("boom"),
),
mock.patch("concordia.tasks.useractivity.send_mail") as m_mail,
):
with self.assertRaises(RuntimeError):
update_useractivity_cache.run(self.user.id, self.campaign.id, "review")
m_del.assert_called_once_with("userprofileactivity_cache_lock")
m_mail.assert_not_called()
class UpdateUserprofileactivityFromCacheTestCase(CreateTestUsers, TestCase):
def setUp(self):
cache.clear()
self.user = self.create_test_user()
self.campaign = create_campaign()
self.key = f"userprofileactivity_{self.campaign.pk}"
@mock.patch("concordia.tasks.useractivity.update_userprofileactivity_table")
def test_no_updates(self, mock_update_table):
cache.set(self.key, None)
with mock.patch("concordia.logging.ConcordiaLogger.debug") as mock_debug:
update_userprofileactivity_from_cache()
self.assertEqual(mock_debug.call_count, 2)
mock_debug.assert_called_with(
"Cache contained no updates for key. Skipping",
event_code="update_userprofileactivity_from_cache_no_updates",
key=self.key,
)
self.assertEqual(mock_update_table.call_count, 0)
@mock.patch("concordia.tasks.useractivity.update_userprofileactivity_table")
def test_update(self, mock_update_table):
cache.set(self.key, {self.user.pk: (1, 0)})
update_userprofileactivity_from_cache()
self.assertEqual(mock_update_table.call_count, 2)
mock_update_table.assert_has_calls(
[
mock.call(self.user, self.campaign.id, "transcribe_count", 1),
mock.call(self.user, self.campaign.id, "review_count", 0),
]
)
self.assertIsNone(cache.get(self.key))
================================================
FILE: concordia/tests/test_tasks_visualizations.py
================================================
from datetime import timedelta
from unittest import mock
from django.core.cache import caches
from django.test import TestCase, override_settings
from django.utils import timezone
from concordia.models import Campaign, SiteReport, TranscriptionStatus
from concordia.tasks.visualizations import (
populate_asset_status_visualization_cache,
populate_daily_activity_visualization_cache,
)
from .utils import create_asset, create_campaign, create_item, create_project
@override_settings(
CACHES={
"default": {
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
},
"visualization_cache": {
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
},
}
)
class VisualizationCacheTasksTests(TestCase):
class _UploadFailed(Exception):
pass
def setUp(self):
self.cache = caches["visualization_cache"]
self.cache.clear()
def test_populate_asset_status_visualization_cache(self):
c1 = create_campaign(status=Campaign.Status.ACTIVE, title="Alpha")
c2 = create_campaign(status=Campaign.Status.ACTIVE, title="Beta")
p1 = create_project(campaign=c1)
i1 = create_item(project=p1)
p2 = create_project(campaign=c2)
i2 = create_item(project=p2)
create_asset(item=i1, transcription_status=TranscriptionStatus.NOT_STARTED)
create_asset(
item=i2,
slug="test-asset-2",
transcription_status=TranscriptionStatus.IN_PROGRESS,
)
create_asset(
item=i2,
slug="test-asset-3",
transcription_status=TranscriptionStatus.SUBMITTED,
)
create_asset(
item=i2,
slug="test-asset-4",
transcription_status=TranscriptionStatus.COMPLETED,
)
populate_asset_status_visualization_cache.run()
overview = self.cache.get("asset-status-overview")
expected_labels = [
TranscriptionStatus.CHOICE_MAP[key]
for key, _ in TranscriptionStatus.CHOICES
]
self.assertEqual(overview["status_labels"], expected_labels)
# Totals: 1 not_started, 1 in_progress, 1 submitted, 1 completed
self.assertEqual(overview["total_counts"], [1, 1, 1, 1])
def test_populate_daily_activity_visualization_cache(self):
today = timezone.localdate()
date1 = today - timedelta(days=2)
date2 = today - timedelta(days=1)
sr1 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
transcriptions_saved=5,
daily_review_actions=1,
)
sr2 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
transcriptions_saved=10,
daily_review_actions=2,
)
# Set specific created_on dates directly in DB
SiteReport.objects.filter(pk=sr1.pk).update(created_on=date1)
SiteReport.objects.filter(pk=sr2.pk).update(created_on=date2)
populate_daily_activity_visualization_cache.run()
result = self.cache.get("daily-transcription-activity-last-28-days")
self.assertIsNotNone(result)
expected_labels = [(date2 - timedelta(days=1)), date2]
expected_labels = [d.strftime("%Y-%m-%d") for d in expected_labels]
# Extract the two datasets
datasets = result["transcription_datasets"]
self.assertEqual(len(datasets), 2)
trans = next(ds for ds in datasets if ds["label"] == "Transcriptions")
reviews = next(ds for ds in datasets if ds["label"] == "Reviews")
# transcriptions = 5 on date1, 10 - 5 = 5 on date2
# reviews = 1 on date1, 2 on date2
self.assertEqual(trans["data"][-2:], [5, 5]) # last two days in the data range
self.assertEqual(reviews["data"][-2:], [1, 2])
def test_negative_daily_saved_clamps_to_zero(self):
today = timezone.localdate()
date1 = today - timedelta(days=2)
date2 = today - timedelta(days=1)
sr1 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
transcriptions_saved=10,
daily_review_actions=0,
)
sr2 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
transcriptions_saved=5, # decreased total, which should not happen
daily_review_actions=0,
)
SiteReport.objects.filter(pk=sr1.pk).update(created_on=date1)
SiteReport.objects.filter(pk=sr2.pk).update(created_on=date2)
populate_daily_activity_visualization_cache.run()
result = self.cache.get("daily-transcription-activity-last-28-days")
self.assertIsNotNone(result)
datasets = result["transcription_datasets"]
trans = next(ds for ds in datasets if ds["label"] == "Transcriptions")
# Should clamp the second day to 0
self.assertEqual(trans["data"][-2:], [10, 0])
def test_asset_status_unchanged_skips_upload_and_cache_update(self):
campaign = create_campaign(status=Campaign.Status.ACTIVE, title="Only")
project = create_project(campaign=campaign)
item = create_item(project=project)
create_asset(item=item, transcription_status=TranscriptionStatus.NOT_STARTED)
create_asset(
item=item, slug="a2", transcription_status=TranscriptionStatus.IN_PROGRESS
)
create_asset(
item=item, slug="a3", transcription_status=TranscriptionStatus.SUBMITTED
)
create_asset(
item=item, slug="a4", transcription_status=TranscriptionStatus.COMPLETED
)
expected_counts = [1, 1, 1, 1]
existing_payload = {
"status_labels": [
TranscriptionStatus.CHOICE_MAP[key]
for key, _ in TranscriptionStatus.CHOICES
],
"total_counts": expected_counts,
"csv_url": "https://old.example/asset-status.csv",
}
self.cache.set("asset-status-overview", existing_payload, None)
with (
mock.patch(
"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save"
) as mock_save,
mock.patch("concordia.tasks.visualizations.structured_logger") as mock_log,
):
populate_asset_status_visualization_cache.run()
mock_save.assert_not_called()
# Cache should remain as-is
self.assertEqual(self.cache.get("asset-status-overview"), existing_payload)
# Logged unchanged
self.assertTrue(mock_log.info.called)
self.assertEqual(
mock_log.info.call_args.kwargs.get("event_code"),
"asset_status_vis_unchanged",
)
def test_asset_status_upload_failure_with_prior_url_falls_back(self):
campaign = create_campaign(status=Campaign.Status.ACTIVE, title="Only")
project = create_project(campaign=campaign)
item = create_item(project=project)
create_asset(item=item, transcription_status=TranscriptionStatus.NOT_STARTED)
# Ensure "existing" differs so code takes the non-unchanged path
self.cache.set(
"asset-status-overview",
{
"status_labels": [],
"total_counts": [0, 0, 0, 0],
"csv_url": "https://old.example/asset-status.csv",
},
None,
)
with (
mock.patch(
"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save",
side_effect=self._UploadFailed("test exception"),
),
mock.patch("concordia.tasks.visualizations.structured_logger") as mock_log,
):
# Should not raise because we have a prior CSV URL to fall back to
populate_asset_status_visualization_cache.run()
updated = self.cache.get("asset-status-overview")
# Counts should reflect the new data (1 in NOT_STARTED; others 0)
expected = [
1 if key == TranscriptionStatus.NOT_STARTED else 0
for key, _ in TranscriptionStatus.CHOICES
]
self.assertEqual(updated["total_counts"], expected)
# URL should remain the old one
self.assertEqual(updated["csv_url"], "https://old.example/asset-status.csv")
# Logged exception with the non-missing-url code
self.assertTrue(mock_log.exception.called)
self.assertEqual(
mock_log.exception.call_args.kwargs.get("event_code"),
"asset_status_vis_csv_error",
)
def test_asset_status_upload_failure_without_prior_url_raises(self):
campaign = create_campaign(status=Campaign.Status.ACTIVE, title="Only")
project = create_project(campaign=campaign)
item = create_item(project=project)
create_asset(item=item, transcription_status=TranscriptionStatus.NOT_STARTED)
# No existing cache entry, so no prior URL
with (
mock.patch(
"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save",
side_effect=self._UploadFailed("test exception"),
),
mock.patch("concordia.tasks.visualizations.structured_logger") as mock_log,
):
with self.assertRaises(self._UploadFailed):
populate_asset_status_visualization_cache.run()
self.assertTrue(mock_log.exception.called)
self.assertEqual(
mock_log.exception.call_args.kwargs.get("event_code"),
"asset_status_vis_csv_missing_url_error",
)
def test_daily_activity_unchanged_skips_upload_and_cache_update(self):
# With no SiteReports, both series are 28 zeros; pre-populate matching cache
zeros = [0] * 28
existing = {
"labels": [], # labels do not matter for the dedupe
"transcription_datasets": [
{"label": "Transcriptions", "data": zeros},
{"label": "Reviews", "data": zeros},
],
"csv_url": "https://old.example/daily.csv",
}
self.cache.set("daily-transcription-activity-last-28-days", existing, None)
with (
mock.patch(
"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save"
) as mock_save,
mock.patch("concordia.tasks.visualizations.structured_logger") as mock_log,
):
populate_daily_activity_visualization_cache.run()
mock_save.assert_not_called()
self.assertEqual(
self.cache.get("daily-transcription-activity-last-28-days"), existing
)
self.assertTrue(mock_log.info.called)
self.assertEqual(
mock_log.info.call_args.kwargs.get("event_code"),
"daily_activity_vis_unchanged",
)
def test_daily_activity_upload_failure_with_prior_url_falls_back(self):
# Build reports so new data will not be all zeros (ensures "changed" path)
today = timezone.localdate()
date1 = today - timedelta(days=2)
date2 = today - timedelta(days=1)
sr1 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
transcriptions_saved=3,
daily_review_actions=1,
)
sr2 = SiteReport.objects.create(
report_name=SiteReport.ReportName.TOTAL,
transcriptions_saved=5,
daily_review_actions=2,
)
SiteReport.objects.filter(pk=sr1.pk).update(created_on=date1)
SiteReport.objects.filter(pk=sr2.pk).update(created_on=date2)
# Prior cache with different series and a CSV URL to fall back to
self.cache.set(
"daily-transcription-activity-last-28-days",
{
"labels": [],
"transcription_datasets": [
{"label": "Transcriptions", "data": [0] * 28},
{"label": "Reviews", "data": [0] * 28},
],
"csv_url": "https://old.example/daily.csv",
},
None,
)
with (
mock.patch(
"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save",
side_effect=self._UploadFailed("test exception"),
),
mock.patch("concordia.tasks.visualizations.structured_logger") as mock_log,
):
# Should not raise because we have a prior CSV URL
populate_daily_activity_visualization_cache.run()
updated = self.cache.get("daily-transcription-activity-last-28-days")
self.assertIsNotNone(updated)
# Still using the old URL
self.assertEqual(updated["csv_url"], "https://old.example/daily.csv")
# Logged exception with the non-missing-url code
self.assertTrue(mock_log.exception.called)
self.assertEqual(
mock_log.exception.call_args.kwargs.get("event_code"),
"daily_activity_vis_csv_error",
)
def test_daily_activity_upload_failure_without_prior_url_raises(self):
# No existing cache entry -> csv_url is None
with (
mock.patch(
"concordia.tasks.visualizations.VISUALIZATION_STORAGE.save",
side_effect=self._UploadFailed("test exception"),
),
mock.patch("concordia.tasks.visualizations.structured_logger") as mock_log,
):
with self.assertRaises(self._UploadFailed):
populate_daily_activity_visualization_cache.run()
self.assertTrue(mock_log.exception.called)
self.assertEqual(
mock_log.exception.call_args.kwargs.get("event_code"),
"daily_activity_vis_csv_missing_url_error",
)
================================================
FILE: concordia/tests/test_templatetags.py
================================================
from django.http import QueryDict
from django.template import Context, Template
from django.templatetags.static import static
from django.test import TestCase, override_settings
from django.utils.html import escape, format_html
from concordia.models import TranscriptionStatus
from concordia.templatetags.concordia_filtering_tags import transcription_status_filters
from concordia.templatetags.concordia_text_tags import reprchar
from concordia.templatetags.custom_math import multiply
from concordia.templatetags.reject_filter import reject
from concordia.templatetags.truncation import (
WordBreakTruncator,
truncatechars_on_word_break,
)
from concordia.templatetags.visualization import concordia_visualization
class TestTemplateTags(TestCase):
def test_truncatechars_on_word_break(self):
test_string = "Lorem ipsum \u0317 dolor sit amet, consectetur adipiscing elit"
self.assertEqual(truncatechars_on_word_break(test_string, 0), "[…]")
self.assertEqual(truncatechars_on_word_break(test_string, 1), "[…]")
self.assertEqual(truncatechars_on_word_break(test_string, 10), "Lorem[…]")
self.assertEqual(
truncatechars_on_word_break(test_string, 30),
"Lorem ipsum \u0317 dolor sit[…]",
)
self.assertEqual(truncatechars_on_word_break(test_string, 1000), test_string)
self.assertEqual(
truncatechars_on_word_break(test_string, "badvalue"), test_string
)
self.assertEqual(
WordBreakTruncator(test_string).word_break(30, "[\u0317]"),
"Lorem ipsum \u0317 dolor sit[\u0317]",
)
def test_multiply(self):
self.assertEqual(multiply(5, 5), 5 * 5)
self.assertEqual(multiply(0, 5), 0 * 5)
self.assertEqual(multiply(1, 2), 1 * 2)
def test_transcription_status_filters(self):
status_counts = []
for choice in TranscriptionStatus.CHOICES:
status_counts.append((choice, 0, 1))
transcription_status_filters(status_counts, "")
transcription_status_filters(status_counts, "", reversed_order=True)
transcription_status_filters(status_counts, TranscriptionStatus.CHOICES[0][0])
def test_qs_alter(self):
base_template = "{% load concordia_querystring %}"
out = Template(
base_template + "{% qs_alter 'bar=baz&baz=taz' foo='bar' %}"
).render(Context())
self.assertEqual(out, "bar=baz&baz=taz&foo=bar")
data = QueryDict("bar=baz&baz=taz&bar=foo")
out = Template(base_template + "{% qs_alter data foo='bar' %}").render(
Context({"data": data})
)
self.assertEqual(out, "bar=baz&bar=foo&baz=taz&foo=bar")
out = Template(base_template + "{% qs_alter data delete:bar %}").render(
Context({"data": data})
)
self.assertEqual(out, "baz=taz")
out = Template(base_template + "{% qs_alter data delete:taz %}").render(
Context({"data": data})
)
self.assertEqual(out, "bar=baz&bar=foo&baz=taz")
out = Template(
base_template + "{% qs_alter data delete_value:\"bar\",'foo' %}"
).render(Context({"data": data}))
self.assertEqual(out, "bar=baz&baz=taz")
out = Template(
base_template + "{% qs_alter data delete_value:'bar','taz' %}"
).render(Context({"data": data}))
self.assertEqual(out, "bar=baz&bar=foo&baz=taz")
out = Template(
base_template + "{% qs_alter data foo='bar' as new_data %}" "{{ new_data }}"
).render(Context({"data": data}))
self.assertEqual(out, "bar=baz&bar=foo&baz=taz&foo=bar")
# Test add_if_missing when the key already exists (should not overwrite)
out = Template(
base_template + "{% qs_alter data add_if_missing:bar='newvalue' %}"
).render(Context({"data": data}))
self.assertEqual(out, "bar=baz&bar=foo&baz=taz")
def test_reprchar_variants(self):
cases = [
("A", "A"),
("\n", "\\n"),
("\x00", "\\x00"),
("\u200b", "\\u200b"),
("\\", "\\\\"),
]
for ch, expected in cases:
self.assertEqual(reprchar(ch), expected)
class RejectFilterTests(TestCase):
def test_returns_input_when_falsy(self):
self.assertEqual(reject("", "x"), "")
self.assertEqual(reject([], "x"), [])
self.assertIsNone(reject(None, "x"))
self.assertEqual(reject((), "x"), ())
def test_string_single_reject(self):
self.assertEqual(
reject("error warn marked-safe", "marked-safe"),
"error warn",
)
def test_string_multiple_rejects(self):
self.assertEqual(
reject("error warn marked-safe", "marked-safe,warn"),
"error",
)
def test_string_no_match(self):
self.assertEqual(reject("one two", "three"), "one two")
def test_string_empty_args(self):
self.assertEqual(reject("one two", ""), "one two")
def test_string_whitespace_split_and_join(self):
self.assertEqual(reject("a b\tc", "b"), "a c")
def test_string_case_sensitivity(self):
self.assertEqual(reject("A a", "a"), "A")
def test_iterable_list(self):
self.assertEqual(
reject(["ok", "deprecated", "x", "hidden"], "deprecated,hidden"),
["ok", "x"],
)
def test_iterable_tuple_and_duplicates(self):
self.assertEqual(reject(("a", "b", "c", "b"), "b"), ["a", "c"])
def test_iterable_no_match(self):
self.assertEqual(reject(["one", "two"], "three"), ["one", "two"])
def test_iterable_empty_args(self):
self.assertEqual(reject(["one", "two"], ""), ["one", "two"])
@override_settings(
STORAGES={
"default": {"BACKEND": "django.core.files.storage.FileSystemStorage"},
"staticfiles": {
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
},
},
STATICFILES_STORAGE="django.contrib.staticfiles.storage.StaticFilesStorage",
)
class VisualizationTagsTests(TestCase):
def test_without_attrs_renders_section_and_script(self):
# No attributes: should render a plain and matching '
script_src = static(f"js/visualizations/{name}.js")
script_html = format_html(
'', script_src
)
result = concordia_visualization(name) + script_html
# The id attribute must have the name escaped
escaped_id = escape(name)
self.assertIn(f'id="{escaped_id}"', result)
# The script src must also be escaped
raw_src = static(f"js/visualizations/{name}.js")
escaped_src = escape(raw_src)
self.assertIn(f'src="{escaped_src}"', result)
================================================
FILE: concordia/tests/test_top_level_views.py
================================================
"""
Tests for for the top-level & “CMS” views
"""
from unittest.mock import patch
from django.core.cache import cache
from django.test import RequestFactory, TestCase
from django.urls import reverse
from maintenance_mode.core import get_maintenance_mode, set_maintenance_mode
from concordia.models import (
Banner,
CarouselSlide,
Guide,
OverlayPosition,
SimplePage,
SiteReport,
)
from concordia.views.simple_pages import simple_page
from .utils import (
CacheControlAssertions,
CreateTestUsers,
JSONAssertMixin,
create_guide,
create_site_report,
)
class TopLevelViewTests(
JSONAssertMixin, CreateTestUsers, CacheControlAssertions, TestCase
):
def setUp(self):
cache.clear()
def tearDown(self):
cache.clear()
def test_healthz(self):
data = self.assertValidJSON(self.client.get("/healthz"))
for k in (
"current_time",
"load_average",
"debug",
"database_has_data",
"application_version",
):
self.assertIn(k, data)
def test_homepage(self):
response = self.client.get(reverse("homepage"))
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(response, "home.html")
banner = Banner.objects.create(
slug="test-banner", text="Test Banner", active=True
)
response = self.client.get(reverse("homepage"))
context = response.context
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(response, "home.html")
self.assertIn("banner", context)
self.assertEqual(context["banner"].text, banner.text)
banner.delete()
slide = CarouselSlide.objects.create(
published=True,
overlay_position=OverlayPosition.LEFT,
headline="Test Headline",
)
response = self.client.get(reverse("homepage"))
context = response.context
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(response, "home.html")
self.assertIn("firstslide", context)
self.assertEqual(context["firstslide"].headline, slide.headline)
slide.delete()
def test_contact_us_redirect(self):
response = self.client.get(reverse("contact"))
self.assertEqual(response.status_code, 302)
self.assertEqual(response["Location"], "https://ask.loc.gov/crowd")
def test_simple_page(self):
s = SimplePage.objects.create(
title="Get Started 123",
body="not the real body",
path=reverse("welcome-guide"),
)
s2 = SimplePage.objects.create(
title="Get Started Spanish 123",
body="not the real spanish body",
path=reverse("welcome-guide-spanish"),
)
resp = self.client.get(reverse("welcome-guide"))
self.assertEqual(200, resp.status_code)
self.assertEqual(s.title, resp.context["title"])
self.assertEqual(
[(reverse("welcome-guide"), s.title)], resp.context["breadcrumbs"]
)
self.assertEqual(resp.context["body"], f"{s.body}
")
request = RequestFactory().get("/")
resp = simple_page(request, path=reverse("welcome-guide"))
self.assertEqual(200, resp.status_code)
resp = self.client.get(reverse("welcome-guide-spanish"))
self.assertEqual(200, resp.status_code)
self.assertEqual(s2.title, resp.context["title"])
self.assertEqual("es", resp.context["language_code"])
self.assertEqual(
[(reverse("welcome-guide-spanish"), s2.title)], resp.context["breadcrumbs"]
)
self.assertEqual(resp.context["body"], f"{s2.body}
")
def test_nested_simple_page(self):
Guide.objects.create(title="How to Tag")
l1 = SimplePage.objects.create(
title="Get Started",
body="not the real body",
path=reverse("welcome-guide"),
)
l2 = SimplePage.objects.create(
title="How to Tag",
body="This is _not_ the real page",
path=reverse("how-to-tag"),
)
resp = self.client.get(reverse("how-to-tag"))
self.assertEqual(200, resp.status_code)
self.assertEqual(l2.title, resp.context["title"])
self.assertEqual(
resp.context["breadcrumbs"],
[(reverse("welcome-guide"), l1.title), (reverse("how-to-tag"), l2.title)],
)
self.assertHTMLEqual(
resp.context["body"], "This is not the real page
"
)
create_guide(page=l1)
resp = self.client.get(reverse("welcome-guide"))
self.assertEqual(200, resp.status_code)
def test_simple_page_with_context(self):
path = reverse("about")
page_body = (
"{{ assets_published}}
"
"{{ campaigns_published }}
"
"{{ assets_completed }}
"
"{{ assets_waiting_review }}
"
"{{ users_activated }}
"
)
about_page = SimplePage.objects.create(
title="About",
body=page_body,
path=reverse("about"),
)
# Test with no SiteReports
response = self.client.get(path)
context = response.context
self.assertEqual(200, response.status_code)
self.assertEqual(about_page.title, context["title"])
self.assertEqual([(path, about_page.title)], context["breadcrumbs"])
self.assertEqual(
context["body"], "0
\n0
\n0
\n0
\n0
"
)
# Test with only active SiteReport
cache.clear()
create_site_report(
report_name=SiteReport.ReportName.TOTAL,
campaigns_published=1,
assets_published=1,
assets_completed=1,
assets_waiting_review=1,
users_activated=1,
)
response = self.client.get(path)
context = response.context
self.assertEqual(
context["body"], "1
\n1
\n1
\n1
\n1
"
)
# Test with both SiteReports, but with cached values from above
# So we should expect the retired SiteReport to not be included in data
create_site_report(
report_name=SiteReport.ReportName.RETIRED_TOTAL,
assets_published=1,
assets_completed=1,
assets_waiting_review=1,
)
response = self.client.get(path)
context = response.context
self.assertEqual(
context["body"], "1
\n1
\n1
\n1
\n1
"
)
# Test without bad cached data
cache.clear()
response = self.client.get(path)
context = response.context
self.assertEqual(
context["body"], "2
\n1
\n2
\n2
\n1
"
)
class HelpCenterRedirectTests(TestCase):
def test_HelpCenterRedirectView(self):
SimplePage.objects.create(
title="Get Started Page",
body="Page Body",
path="/get-started/page/",
)
self.assertRedirects(
self.client.get("/help-center/page/"), "/get-started/page/"
)
def test_HelpCenterSpanishRedirectView(self):
SimplePage.objects.create(
title="Get Started Page",
body="Page Body",
path="/get-started-esp/page-esp/",
)
self.assertRedirects(
self.client.get("/help-center/page-esp/"), "/get-started-esp/page-esp/"
)
class MaintenanceModeTests(TestCase, CreateTestUsers):
def setUp(self):
cache.clear()
self.timestamp_value = 1
self.user = None
def tearDown(self):
cache.clear()
def test_maintenance_mode_off(self):
self.user = self.create_super_user()
self.login_user()
set_maintenance_mode(True)
with patch("concordia.views.maintenance_mode.time") as mock:
mock.return_value = self.timestamp_value
self.assertRedirects(
self.client.get(reverse("maintenance_mode_off")),
f"/?t={self.timestamp_value}",
)
self.assertEqual(get_maintenance_mode(), False)
self.user = self.create_test_user()
self.login_user()
set_maintenance_mode(True)
with patch("concordia.views.maintenance_mode.time") as mock:
mock.return_value = self.timestamp_value
self.assertRedirects(
self.client.get(reverse("maintenance_mode_off")),
f"/?t={self.timestamp_value}",
target_status_code=503,
)
self.assertEqual(get_maintenance_mode(), True)
def test_maintenance_mode_on_without_frontend(self):
cache.set("maintenance_mode_frontend_available", False, None)
self.user = self.create_super_user()
self.login_user()
set_maintenance_mode(False)
with patch("concordia.views.maintenance_mode.time") as mock:
mock.return_value = self.timestamp_value
self.assertRedirects(
self.client.get(reverse("maintenance_mode_on")),
f"/?t={self.timestamp_value}",
target_status_code=503,
)
self.assertEqual(get_maintenance_mode(), True)
self.user = self.create_test_user()
self.login_user()
set_maintenance_mode(False)
with patch("concordia.views.maintenance_mode.time") as mock:
mock.return_value = self.timestamp_value
self.assertRedirects(
self.client.get(reverse("maintenance_mode_on")),
f"/?t={self.timestamp_value}",
)
self.assertEqual(get_maintenance_mode(), False)
def test_maintenance_mode_on_with_frontend(self):
cache.set("maintenance_mode_frontend_available", True, None)
self.user = self.create_super_user()
self.login_user()
set_maintenance_mode(False)
with patch("concordia.views.maintenance_mode.time") as mock:
mock.return_value = self.timestamp_value
self.assertRedirects(
self.client.get(reverse("maintenance_mode_on")),
f"/?t={self.timestamp_value}",
)
self.assertEqual(get_maintenance_mode(), True)
self.user = self.create_test_user()
self.login_user()
set_maintenance_mode(False)
with patch("concordia.views.maintenance_mode.time") as mock:
mock.return_value = self.timestamp_value
self.assertRedirects(
self.client.get(reverse("maintenance_mode_on")),
f"/?t={self.timestamp_value}",
)
self.assertEqual(get_maintenance_mode(), False)
def test_maintenance_mode_frontend_available(self):
self.user = self.create_super_user()
self.login_user()
cache.set("maintenance_mode_frontend_available", False, None)
with patch("concordia.views.maintenance_mode.time") as mock:
mock.return_value = self.timestamp_value
self.assertRedirects(
self.client.get(reverse("maintenance_mode_frontend_available")),
f"/?t={self.timestamp_value}",
)
self.assertEqual(cache.get("maintenance_mode_frontend_available"), True)
self.user = self.create_test_user()
self.login_user()
cache.set("maintenance_mode_frontend_available", False, None)
with patch("concordia.views.maintenance_mode.time") as mock:
mock.return_value = self.timestamp_value
self.assertRedirects(
self.client.get(reverse("maintenance_mode_frontend_available")),
f"/?t={self.timestamp_value}",
)
self.assertEqual(cache.get("maintenance_mode_frontend_available"), False)
def test_maintenance_mode_frontend_unavailable(self):
self.user = self.create_super_user()
self.login_user()
cache.set("maintenance_mode_frontend_available", True, None)
with patch("concordia.views.maintenance_mode.time") as mock:
mock.return_value = self.timestamp_value
self.assertRedirects(
self.client.get(reverse("maintenance_mode_frontend_unavailable")),
f"/?t={self.timestamp_value}",
)
self.assertEqual(cache.get("maintenance_mode_frontend_available"), False)
self.user = self.create_test_user()
self.login_user()
cache.set("maintenance_mode_frontend_available", True, None)
with patch("concordia.views.maintenance_mode.time") as mock:
mock.return_value = self.timestamp_value
self.assertRedirects(
self.client.get(reverse("maintenance_mode_frontend_unavailable")),
f"/?t={self.timestamp_value}",
)
self.assertEqual(cache.get("maintenance_mode_frontend_available"), True)
================================================
FILE: concordia/tests/test_utils_celery.py
================================================
from types import SimpleNamespace
from unittest import mock
from django.test import TestCase
from concordia.utils.celery import get_registered_task
class CeleryUtilsTests(TestCase):
def test_get_registered_task_returns_task_from_registry(self):
name = "pkg.tasks.do_thing"
dummy_task = object()
app = SimpleNamespace(tasks={name: dummy_task}, send_task=mock.Mock())
with mock.patch(
"concordia.utils.celery.concordia_celery_app",
app,
):
got = get_registered_task(name)
self.assertIs(got, dummy_task)
app.send_task.assert_not_called()
def test_get_registered_task_raises_runtime_error_with_cause(self):
name = "pkg.tasks.missing"
app = SimpleNamespace(tasks={}, send_task=mock.Mock())
with mock.patch(
"concordia.utils.celery.concordia_celery_app",
app,
):
with self.assertRaises(RuntimeError) as ctx:
get_registered_task(name)
message = str(ctx.exception)
self.assertIn(f"Task {name} is not registered.", message)
self.assertIn("Did you typo it?", message)
self.assertIsInstance(ctx.exception.__cause__, KeyError)
app.send_task.assert_not_called()
================================================
FILE: concordia/tests/test_utils_logging.py
================================================
from types import SimpleNamespace
from django.test import TestCase
from concordia.logging import get_logging_user_id
from concordia.utils import get_anonymous_user
from .utils import CreateTestUsers
class LoggingTests(CreateTestUsers, TestCase):
def test_get_logging_user_id_authenticated_user(self):
user = self.create_test_user()
self.assertEqual(get_logging_user_id(user), str(user.id))
def test_get_logging_user_id_anonymous_user(self):
anon = get_anonymous_user()
self.assertEqual(get_logging_user_id(anon), "anonymous")
def test_get_logging_user_id_missing_auth_attribute(self):
mock_user = object()
self.assertEqual(get_logging_user_id(mock_user), "anonymous")
def test_get_logging_user_id_authenticated_no_id(self):
user = SimpleNamespace(is_authenticated=True, username="someuser")
self.assertEqual(get_logging_user_id(user), "anonymous")
================================================
FILE: concordia/tests/test_utils_next_asset_reviewable_campaign.py
================================================
from unittest.mock import MagicMock, patch
from django.test import TestCase
from django.utils.timezone import now
from concordia.models import (
AssetTranscriptionReservation,
NextReviewableCampaignAsset,
)
from concordia.utils import get_anonymous_user
from concordia.utils.next_asset import (
find_new_reviewable_campaign_assets,
find_next_reviewable_campaign_asset,
find_reviewable_campaign_asset,
)
from concordia.utils.next_asset.reviewable.campaign import (
_eligible_reviewable_base_qs,
_find_reviewable_in_item,
_find_reviewable_in_project,
_next_seq_after,
_reserved_asset_ids_subq,
find_and_order_potential_reviewable_campaign_assets,
find_invalid_next_reviewable_campaign_assets,
)
from .utils import (
CreateTestUsers,
create_asset,
create_campaign,
create_item,
create_project,
create_transcription,
)
class NextReviewableCampaignAssetTests(CreateTestUsers, TestCase):
def setUp(self):
self.anon = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(sequence=1)
self.asset2 = create_asset(
item=self.asset1.item, sequence=2, slug="test-asset-2"
)
self.campaign = self.asset1.campaign
def test_find_new_reviewable_campaign_assets_filters_correctly(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
queryset = find_new_reviewable_campaign_assets(self.campaign, self.user)
self.assertIn(self.asset1, queryset)
def test_find_new_reviewable_campaign_assets_without_user(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
# Covers lines 28–30
queryset = find_new_reviewable_campaign_assets(self.campaign, None)
self.assertIn(self.asset1, queryset)
def test_find_reviewable_campaign_asset_from_next_table(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
NextReviewableCampaignAsset.objects.create(
asset=self.asset1,
campaign=self.campaign,
item=self.asset1.item,
item_item_id=self.asset1.item.item_id,
project=self.asset1.item.project,
project_slug=self.asset1.item.project.slug,
sequence=self.asset1.sequence,
transcriber_ids=[],
)
asset = find_reviewable_campaign_asset(self.campaign, self.user)
self.assertEqual(asset, self.asset1)
@patch("concordia.utils.next_asset.reviewable.campaign.get_registered_task")
def test_find_reviewable_campaign_asset_falls_back_and_spawns_task(
self, mock_get_task
):
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset = find_reviewable_campaign_asset(self.campaign, self.user)
self.assertEqual(asset, self.asset2)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.campaign.get_registered_task")
def test_find_next_reviewable_campaign_asset_orders_and_falls_back(
self, mock_get_task
):
"""
With short-circuiting: project-level returns the eligible asset
and we do not spawn a task.
"""
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(asset, self.asset1)
# Short-circuit satisfied -> no cache fallback -> no task spawned
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.campaign.get_registered_task")
def test_find_next_reviewable_campaign_asset_when_next_asset_exists(
self, mock_get_task
):
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
NextReviewableCampaignAsset.objects.create(
asset=self.asset2,
campaign=self.campaign,
item=self.asset2.item,
item_item_id=self.asset2.item.item_id,
project=self.asset2.item.project,
project_slug=self.asset2.item.project.slug,
sequence=self.asset2.sequence,
transcriber_ids=[],
)
asset = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug=self.asset2.item.project.slug,
item_id=self.asset2.item.item_id,
original_asset_id=self.asset2.id - 1,
)
self.assertEqual(asset, self.asset2)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
def test_short_circuit_same_item_excludes_users_own_work(self):
"""
Reviewable item short-circuit must not return assets
transcribed by the requesting user.
"""
# Two submitted in same item: one by self.user, one by anon
mine = self.asset1
other = self.asset2
create_transcription(asset=mine, user=self.user, submitted=now())
create_transcription(asset=other, user=self.anon, submitted=now())
chosen = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug=mine.item.project.slug,
item_id=mine.item.item_id,
original_asset_id=mine.id,
)
self.assertEqual(chosen, other)
def test_item_short_circuit_reviewable_respects_after_sequence_and_reservations(
self,
):
"""
Item reviewable short-circuit should choose next by sequence
and skip reserved assets.
"""
# Three submitted in the same item (none by self.user)
asset1 = self.asset1
asset2 = self.asset2
asset3 = create_asset(item=asset1.item, sequence=3, slug="rev-a3")
for asset in (asset1, asset2, asset3):
create_transcription(asset=asset, user=self.anon, submitted=now())
# After asset1, pick asset2
chosen = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug=asset1.item.project.slug,
item_id=asset1.item.item_id,
original_asset_id=asset1.id,
)
self.assertEqual(chosen, asset2)
# Reserve asset2, so should pick asset3
AssetTranscriptionReservation.objects.create(
asset=asset2, reservation_token="rv" # nosec
)
chosen2 = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug=asset1.item.project.slug,
item_id=asset1.item.item_id,
original_asset_id=asset1.id,
)
self.assertEqual(chosen2, asset3)
def test_project_short_circuit_when_item_has_only_users_work(self):
"""
If the only SUBMITTED assets in the item were transcribed by the user,
the function should fall back to other assets in the same project.
"""
# Item-level: only user's work
create_transcription(asset=self.asset1, user=self.user, submitted=now())
create_transcription(asset=self.asset2, user=self.user, submitted=now())
# Project-level: someone else's work
other_item = create_item(project=self.asset1.item.project, item_id="p2")
project_asset = create_asset(item=other_item, slug="rev-p-asset")
create_transcription(asset=project_asset, user=self.anon, submitted=now())
chosen = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, project_asset)
@patch("concordia.utils.next_asset.reviewable.campaign.get_registered_task")
def test_cache_excludes_user_and_triggers_spawn_task(self, mock_get_task):
"""
When the only cached asset is excluded via transcriber_ids containing the user,
the function should skip cache, fall back to manual and spawn a populate task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Cached entry references the user's own work (excluded by contains)
create_transcription(asset=self.asset1, user=self.user, submitted=now())
NextReviewableCampaignAsset.objects.create(
asset=self.asset1,
campaign=self.campaign,
item=self.asset1.item,
item_item_id=self.asset1.item.item_id,
project=self.asset1.item.project,
project_slug=self.asset1.item.project.slug,
sequence=self.asset1.sequence,
transcriber_ids=[self.user.id],
)
# A valid reviewable exists elsewhere
other = create_asset(item=self.asset1.item, sequence=3, slug="rev-other")
create_transcription(asset=other, user=self.anon, submitted=now())
# Pass empty project/item to ensure we hit cache (and thus spawn task)
chosen = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug="",
item_id="",
original_asset_id=None,
)
self.assertEqual(chosen, other)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
class ReviewableCampaignInternalsTests(CreateTestUsers, TestCase):
def setUp(self):
self.anon = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(sequence=1, slug="rc-a1")
# same item, higher sequence
self.asset2 = create_asset(item=self.asset1.item, sequence=2, slug="rc-a2")
self.campaign = self.asset1.campaign
def test_reserved_asset_ids_subq_filters_to_campaign(self):
AssetTranscriptionReservation.objects.create(
asset=self.asset1, reservation_token="r1" # nosec
)
other_campaign = create_campaign(slug="rc-camp-a", title="rc-camp-a")
other_project = create_project(
campaign=other_campaign, slug="rc-proj-a", title="rc-proj-a"
)
other_item = create_item(project=other_project, item_id="rc-other-item")
other_campaign_asset = create_asset(item=other_item, slug="rc-other-camp-a")
AssetTranscriptionReservation.objects.create(
asset=other_campaign_asset, reservation_token="r2" # nosec
)
id_set = set(
_reserved_asset_ids_subq(self.campaign).values_list("asset_id", flat=True)
)
self.assertIn(self.asset1.id, id_set)
self.assertNotIn(other_campaign_asset.id, id_set)
def test_eligible_reviewable_base_qs_excludes_user_and_requires_submitted(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.user, submitted=now())
asset3 = create_asset(item=self.asset1.item, sequence=3, slug="rc-a3")
queryset_user = _eligible_reviewable_base_qs(self.campaign, user=self.user)
self.assertIn(self.asset1, queryset_user)
self.assertNotIn(self.asset2, queryset_user)
self.assertNotIn(asset3, queryset_user)
queryset_none = _eligible_reviewable_base_qs(self.campaign, user=None)
self.assertIn(self.asset1, queryset_none)
self.assertIn(self.asset2, queryset_none)
self.assertNotIn(asset3, queryset_none)
def test_next_seq_after_none_missing_and_valid(self):
self.assertIsNone(_next_seq_after(None))
self.assertIsNone(_next_seq_after(99999999))
self.assertEqual(_next_seq_after(self.asset2.pk), self.asset2.sequence)
def test_find_reviewable_in_item_after_none_returns_first(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
chosen = _find_reviewable_in_item(
self.campaign,
self.user,
item_id=self.asset1.item.item_id,
after_asset_pk=None,
)
self.assertEqual(chosen, self.asset1)
def test_find_reviewable_in_item_after_asset_in_other_item_ignores_gate(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
other_item = create_item(
project=self.asset1.item.project, item_id="rc-other-item"
)
other_asset = create_asset(item=other_item, slug="rc-other-asset")
create_transcription(asset=other_asset, user=self.anon, submitted=now())
chosen = _find_reviewable_in_item(
self.campaign,
self.user,
item_id=self.asset1.item.item_id,
after_asset_pk=other_asset.id,
)
self.assertEqual(chosen, self.asset1)
def test_find_reviewable_in_item_after_asset_missing_ignores_gate(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
chosen = _find_reviewable_in_item(
self.campaign,
self.user,
item_id=self.asset1.item.item_id,
after_asset_pk=987654321,
)
self.assertEqual(chosen, self.asset1)
def test_find_reviewable_in_item_after_asset_sidc_ignores_gate(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
other_campaign = create_campaign(slug="rc-camp-b", title="rc-camp-b")
other_project = create_project(
campaign=other_campaign, slug="rc-proj-b", title="rc-proj-b"
)
other_item = create_item(
project=other_project, item_id=self.asset1.item.item_id
)
other_campaign_asset = create_asset(item=other_item, slug="rc-cross-camp")
create_transcription(
asset=other_campaign_asset, user=self.anon, submitted=now()
)
chosen = _find_reviewable_in_item(
self.campaign,
self.user,
item_id=self.asset1.item.item_id,
after_asset_pk=other_campaign_asset.id,
)
self.assertEqual(chosen, self.asset1)
def test_find_reviewable_in_project_orders_and_excludes_user(self):
other_item = create_item(project=self.asset1.item.project, item_id="rc-p-item")
mine = create_asset(item=other_item, sequence=1, slug="rc-p-mine")
theirs = create_asset(item=other_item, sequence=2, slug="rc-p-theirs")
create_transcription(asset=mine, user=self.user, submitted=now())
create_transcription(asset=theirs, user=self.anon, submitted=now())
chosen = _find_reviewable_in_project(
self.campaign,
self.user,
project_slug=self.asset1.item.project.slug,
after_asset_pk=self.asset1.id,
)
self.assertEqual(chosen, theirs)
def test_find_reviewable_in_project_returns_none_when_only_users_work(self):
other_item = create_item(project=self.asset1.item.project, item_id="rc-p2")
mine = create_asset(item=other_item, sequence=1, slug="rc-p2-mine")
create_transcription(asset=mine, user=self.user, submitted=now())
chosen = _find_reviewable_in_project(
self.campaign,
self.user,
project_slug=self.asset1.item.project.slug,
after_asset_pk=self.asset1.id,
)
self.assertIsNone(chosen)
def test_find_new_reviewable_campaign_assets_excludes_reserved_and_next_table(
self,
):
asset_reserved = create_asset(
item=self.asset1.item, sequence=3, slug="rc-a-res"
)
asset_cached = create_asset(
item=self.asset1.item, sequence=4, slug="rc-a-cached"
)
for asset in (asset_reserved, asset_cached):
create_transcription(asset=asset, user=self.anon, submitted=now())
AssetTranscriptionReservation.objects.create(
asset=asset_reserved, reservation_token="rv" # nosec
)
from concordia.models import NextReviewableCampaignAsset
NextReviewableCampaignAsset.objects.create(
asset=asset_cached,
campaign=self.campaign,
item=asset_cached.item,
item_item_id=asset_cached.item.item_id,
project=asset_cached.item.project,
project_slug=asset_cached.item.project.slug,
sequence=asset_cached.sequence,
transcriber_ids=[],
)
queryset = find_new_reviewable_campaign_assets(self.campaign, self.user)
self.assertNotIn(asset_reserved, queryset)
self.assertNotIn(asset_cached, queryset)
def test_find_and_order_potential_reviewable_campaign_assets_ordering(self):
base_item = self.asset1.item
same_item_next = create_asset(item=base_item, sequence=10, slug="rc-ci-next")
other_item_same_project = create_asset(
item=create_item(project=base_item.project, item_id="rc-it-2"),
sequence=5,
slug="rc-p-next",
)
other_project = create_project(
campaign=self.campaign, slug="rc-proj", title="rc-proj"
)
other_project_item = create_item(project=other_project, item_id="rc-it-3")
other_project_asset = create_asset(
item=other_project_item, sequence=1, slug="rc-op"
)
for asset in (same_item_next, other_item_same_project, other_project_asset):
create_transcription(asset=asset, user=self.anon, submitted=now())
def cache_row(asset):
return NextReviewableCampaignAsset.objects.create(
asset=asset,
campaign=self.campaign,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
sequence=asset.sequence,
transcriber_ids=[],
)
cache_row(same_item_next)
cache_row(other_item_same_project)
cache_row(other_project_asset)
after_primary_key = self.asset1.id
ordered = find_and_order_potential_reviewable_campaign_assets(
self.campaign,
self.user,
project_slug=base_item.project.slug,
item_id=base_item.item_id,
asset_pk=after_primary_key,
).values_list("asset_id", flat=True)
ordered = list(ordered)
self.assertEqual(ordered[0], same_item_next.id)
self.assertEqual(ordered[1], other_item_same_project.id)
self.assertIn(other_project_asset.id, ordered[2:])
@patch("concordia.utils.next_asset.reviewable.campaign.get_registered_task")
def test_find_reviewable_campaign_asset_no_eligible_spawns_task_and_returns_none(
self, mock_get_task
):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset = find_reviewable_campaign_asset(self.campaign, self.user)
self.assertIsNone(asset)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.campaign.get_registered_task")
def test_manual_fallback_orders_and_spawns_task(self, mock_get_task):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset_x = create_asset(item=self.asset1.item, sequence=7, slug="rc-mf-x")
asset_y = create_asset(item=self.asset1.item, sequence=8, slug="rc-mf-y")
create_transcription(asset=asset_x, user=self.anon, submitted=now())
create_transcription(asset=asset_y, user=self.anon, submitted=now())
chosen = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug="",
item_id="",
original_asset_id=asset_x.id, # makes asset_y the "next" by id
)
self.assertEqual(chosen, asset_y)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
def test_find_invalid_next_reviewable_campaign_assets_reserved_and_wrong_status(
self,
):
from concordia.models import NextReviewableCampaignAsset
reserved_asset = create_asset(
item=self.asset1.item, sequence=30, slug="rc-inv-res"
)
create_transcription(asset=reserved_asset, user=self.anon, submitted=now())
AssetTranscriptionReservation.objects.create(
asset=reserved_asset, reservation_token="rv" # nosec
)
NextReviewableCampaignAsset.objects.create(
asset=reserved_asset,
campaign=self.campaign,
item=reserved_asset.item,
item_item_id=reserved_asset.item.item_id,
project=reserved_asset.item.project,
project_slug=reserved_asset.item.project.slug,
sequence=reserved_asset.sequence,
transcriber_ids=[],
)
wrong_status_asset = create_asset(
item=self.asset1.item, sequence=31, slug="rc-inv-wrong"
)
create_transcription(asset=wrong_status_asset, user=self.anon) # IN_PROGRESS
NextReviewableCampaignAsset.objects.create(
asset=wrong_status_asset,
campaign=self.campaign,
item=wrong_status_asset.item,
item_item_id=wrong_status_asset.item.item_id,
project=wrong_status_asset.item.project,
project_slug=wrong_status_asset.item.project.slug,
sequence=wrong_status_asset.sequence,
transcriber_ids=[],
)
invalid = list(
find_invalid_next_reviewable_campaign_assets(self.campaign.id).values_list(
"asset_id", flat=True
)
)
self.assertIn(reserved_asset.id, invalid)
self.assertIn(wrong_status_asset.id, invalid)
def test_item_short_circuit_internal_applies_after_and_skips_reserved(self):
asset1 = self.asset1
asset2 = self.asset2
asset3 = create_asset(item=asset1.item, sequence=3, slug="rc-int-a3")
for asset in (asset1, asset2, asset3):
create_transcription(asset=asset, user=self.anon, submitted=now())
AssetTranscriptionReservation.objects.create(
asset=asset2, reservation_token="rv-int" # nosec
)
chosen = _find_reviewable_in_item(
self.campaign,
self.user,
item_id=asset1.item.item_id,
after_asset_pk=asset1.id,
)
self.assertEqual(chosen, asset3)
def test_item_short_circuit_internal_excludes_users_own_work(self):
mine = self.asset1
other = self.asset2
create_transcription(asset=mine, user=self.user, submitted=now())
create_transcription(asset=other, user=self.anon, submitted=now())
chosen = _find_reviewable_in_item(
self.campaign,
self.user,
item_id=mine.item.item_id,
after_asset_pk=None,
)
self.assertEqual(chosen, other)
def test_project_short_circuit_internal_skips_reserved_first(self):
project = self.asset1.item.project
item2 = create_item(project=project, item_id="rc-proj-int")
first = create_asset(item=item2, sequence=1, slug="rc-proj-int-1")
second = create_asset(item=item2, sequence=2, slug="rc-proj-int-2")
for asset in (first, second):
create_transcription(asset=asset, user=self.anon, submitted=now())
AssetTranscriptionReservation.objects.create(
asset=first, reservation_token="rv-proj-int" # nosec
)
chosen = _find_reviewable_in_project(
self.campaign,
self.user,
project_slug=project.slug,
after_asset_pk=self.asset1.id,
)
self.assertEqual(chosen, second)
def test_order_potential_without_after_prefers_item_then_project(self):
base_item = self.asset1.item
same_item = create_asset(item=base_item, sequence=9, slug="rc-ci-none")
same_project = create_asset(
item=create_item(project=base_item.project, item_id="rc-it-np"),
sequence=2,
slug="rc-p-none",
)
other_project = create_asset(
item=create_item(
project=create_project(
campaign=self.campaign, slug="rc-proj-none", title="rc-proj-none"
),
item_id="rc-it-op-none",
),
sequence=1,
slug="rc-op-none",
)
for asset in (same_item, same_project, other_project):
create_transcription(asset=asset, user=self.anon, submitted=now())
NextReviewableCampaignAsset.objects.create(
asset=same_item,
campaign=self.campaign,
item=same_item.item,
item_item_id=same_item.item.item_id,
project=same_item.item.project,
project_slug=same_item.item.project.slug,
sequence=same_item.sequence,
transcriber_ids=[],
)
NextReviewableCampaignAsset.objects.create(
asset=same_project,
campaign=self.campaign,
item=same_project.item,
item_item_id=same_project.item.item_id,
project=same_project.item.project,
project_slug=same_project.item.project.slug,
sequence=same_project.sequence,
transcriber_ids=[],
)
NextReviewableCampaignAsset.objects.create(
asset=other_project,
campaign=self.campaign,
item=other_project.item,
item_item_id=other_project.item.item_id,
project=other_project.item.project,
project_slug=other_project.item.project.slug,
sequence=other_project.sequence,
transcriber_ids=[],
)
ordered = find_and_order_potential_reviewable_campaign_assets(
self.campaign,
self.user,
project_slug=base_item.project.slug,
item_id=base_item.item_id,
asset_pk=None, # ensure next_asset==0
).values_list("asset_id", flat=True)
ordered = list(ordered)
self.assertEqual(ordered[0], same_item.id)
self.assertEqual(ordered[1], same_project.id)
self.assertIn(other_project.id, ordered[2:])
@patch("concordia.utils.next_asset.reviewable.campaign.get_registered_task")
def test_next_reviewable_manual_fallback_no_after_spawns_and_picks_lowest_seq(
self, mock_get_task
):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset1 = self.asset1
asset2 = self.asset2
create_transcription(asset=asset1, user=self.anon, submitted=now())
create_transcription(asset=asset2, user=self.anon, submitted=now())
chosen = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug="",
item_id="",
original_asset_id=None, # triggers Value(0) annotation branch
)
self.assertEqual(chosen, asset1)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.campaign.get_registered_task")
def test_next_reviewable_manual_fallback_invalid_after_str(self, mock_get_task):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset1 = self.asset1
asset2 = self.asset2
create_transcription(asset=asset1, user=self.anon, submitted=now())
create_transcription(asset=asset2, user=self.anon, submitted=now())
chosen = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug="",
item_id="",
original_asset_id="not-an-int",
)
self.assertEqual(chosen, asset1)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.campaign.get_registered_task")
def test_next_reviewable_cached_path_when_short_circuits_fail(self, mock_get_task):
"""
Provide item_id and project_slug so short-circuits run but fail
(only user's SUBMITTED work in that scope), then ensure we use
the cached table (no task spawned).
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
create_transcription(asset=self.asset1, user=self.user, submitted=now())
create_transcription(asset=self.asset2, user=self.user, submitted=now())
cached_project = create_project(
campaign=self.campaign, slug="rc-cached-proj", title="rc-cached-proj"
)
cached_item = create_item(project=cached_project, item_id="rc-cached-item")
cached_asset = create_asset(item=cached_item, slug="rc-cached-asset")
create_transcription(asset=cached_asset, user=self.anon, submitted=now())
NextReviewableCampaignAsset.objects.create(
asset=cached_asset,
campaign=self.campaign,
item=cached_asset.item,
item_item_id=cached_asset.item.item_id,
project=cached_asset.item.project,
project_slug=cached_asset.item.project.slug,
sequence=cached_asset.sequence,
transcriber_ids=[],
)
chosen = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, cached_asset)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.campaign.get_registered_task")
def test_next_reviewable_uses_cache_when_bypassing_short_circuits(
self, mock_get_task
):
"""
Skip both short-circuits by passing blanks; ensure we return from
the cache table directly (no task spawned).
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
cached_project = create_project(
campaign=self.campaign,
slug="rc-cached-proj-2",
title="rc-cached-proj-2",
)
cached_item = create_item(project=cached_project, item_id="rc-cached-item-2")
cached_asset = create_asset(item=cached_item, slug="rc-cached-asset-2")
create_transcription(asset=cached_asset, user=self.anon, submitted=now())
NextReviewableCampaignAsset.objects.create(
asset=cached_asset,
campaign=self.campaign,
item=cached_asset.item,
item_item_id=cached_asset.item.item_id,
project=cached_asset.item.project,
project_slug=cached_asset.item.project.slug,
sequence=cached_asset.sequence,
transcriber_ids=[],
)
chosen = find_next_reviewable_campaign_asset(
self.campaign,
self.user,
project_slug="",
item_id="",
original_asset_id=None,
)
self.assertEqual(chosen, cached_asset)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
================================================
FILE: concordia/tests/test_utils_next_asset_reviewable_topic.py
================================================
from unittest.mock import MagicMock, patch
from django.test import TestCase
from django.utils.timezone import now
from concordia.models import (
AssetTranscriptionReservation,
NextReviewableTopicAsset,
)
from concordia.utils import get_anonymous_user
from concordia.utils.next_asset import (
find_new_reviewable_topic_assets,
find_next_reviewable_topic_asset,
find_reviewable_topic_asset,
)
from concordia.utils.next_asset.reviewable.topic import (
_eligible_reviewable_base_qs as topic_eligible_reviewable_base_qs,
)
from concordia.utils.next_asset.reviewable.topic import (
_find_reviewable_in_item as topic_find_reviewable_in_item,
)
from concordia.utils.next_asset.reviewable.topic import (
_find_reviewable_in_project as topic_find_reviewable_in_project,
)
from concordia.utils.next_asset.reviewable.topic import (
_next_seq_after as topic_next_seq_after,
)
from concordia.utils.next_asset.reviewable.topic import (
_reserved_asset_ids_subq as topic_reserved_asset_ids_subq,
)
from concordia.utils.next_asset.reviewable.topic import (
find_and_order_potential_reviewable_topic_assets,
find_invalid_next_reviewable_topic_assets,
)
from concordia.utils.next_asset.reviewable.topic import (
find_next_reviewable_topic_assets as find_cached_reviewable_topic_assets,
)
from .utils import (
CreateTestUsers,
create_asset,
create_campaign,
create_item,
create_project,
create_topic,
create_transcription,
)
class NextReviewableTopicAssetTests(CreateTestUsers, TestCase):
def setUp(self):
self.anon = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(sequence=1)
self.asset2 = create_asset(
item=self.asset1.item, sequence=2, slug="test-asset-2"
)
self.topic = create_topic(project=self.asset1.item.project)
def test_find_new_reviewable_topic_assets_filters_correctly(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
queryset = find_new_reviewable_topic_assets(self.topic, self.user)
self.assertIn(self.asset1, queryset)
def test_find_new_reviewable_topic_assets_without_user(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
queryset = find_new_reviewable_topic_assets(self.topic, None)
self.assertIn(self.asset1, queryset)
def test_find_reviewable_topic_asset_from_next_table(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
NextReviewableTopicAsset.objects.create(
asset=self.asset1,
topic=self.topic,
item=self.asset1.item,
item_item_id=self.asset1.item.item_id,
project=self.asset1.item.project,
project_slug=self.asset1.item.project.slug,
sequence=self.asset1.sequence,
transcriber_ids=[],
)
asset = find_reviewable_topic_asset(self.topic, self.user)
self.assertEqual(asset, self.asset1)
@patch("concordia.utils.next_asset.reviewable.topic.get_registered_task")
def test_find_reviewable_topic_asset_falls_back_and_spawns_task(
self, mock_get_task
):
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset = find_reviewable_topic_asset(self.topic, self.user)
self.assertEqual(asset, self.asset2)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.topic.get_registered_task")
def test_find_next_reviewable_topic_asset_orders_and_falls_back(
self, mock_get_task
):
"""
With short-circuiting: project-level returns
the eligible asset and we do not spawn a task.
"""
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset = find_next_reviewable_topic_asset(
self.topic,
self.user,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(asset, self.asset1)
# Short-circuit satisfied -> no cache fallback -> no task spawned
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.topic.get_registered_task")
def test_find_next_reviewable_topic_asset_when_next_asset_exists(
self, mock_get_task
):
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
NextReviewableTopicAsset.objects.create(
asset=self.asset2,
topic=self.topic,
item=self.asset2.item,
item_item_id=self.asset2.item.item_id,
project=self.asset2.item.project,
project_slug=self.asset2.item.project.slug,
sequence=self.asset2.sequence,
transcriber_ids=[],
)
asset = find_next_reviewable_topic_asset(
self.topic,
self.user,
project_slug=self.asset2.item.project.slug,
item_id=self.asset2.item.item_id,
original_asset_id=self.asset2.id - 1,
)
self.assertEqual(asset, self.asset2)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
def test_short_circuit_same_item_topic_excludes_users_own_work(self):
mine = self.asset1
other = self.asset2
create_transcription(asset=mine, user=self.user, submitted=now())
create_transcription(asset=other, user=self.anon, submitted=now())
chosen = find_next_reviewable_topic_asset(
self.topic,
self.user,
project_slug=mine.item.project.slug,
item_id=mine.item.item_id,
original_asset_id=mine.id,
)
self.assertEqual(chosen, other)
def test_item_short_circuit_topic_reviewable_respects_after_and_reservations(self):
asset3 = create_asset(item=self.asset1.item, sequence=3, slug="rev-topic-a3")
for asset in (self.asset1, self.asset2, asset3):
create_transcription(asset=asset, user=self.anon, submitted=now())
chosen = find_next_reviewable_topic_asset(
self.topic,
self.user,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, self.asset2)
AssetTranscriptionReservation.objects.create(
asset=self.asset2, reservation_token="rv" # nosec
)
chosen2 = find_next_reviewable_topic_asset(
self.topic,
self.user,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen2, asset3)
@patch("concordia.utils.next_asset.reviewable.topic.get_registered_task")
def test_cache_excludes_user_and_triggers_spawn_task_topic(self, mock_get_task):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
create_transcription(asset=self.asset1, user=self.user, submitted=now())
NextReviewableTopicAsset.objects.create(
asset=self.asset1,
topic=self.topic,
item=self.asset1.item,
item_item_id=self.asset1.item.item_id,
project=self.asset1.item.project,
project_slug=self.asset1.item.project.slug,
sequence=self.asset1.sequence,
transcriber_ids=[self.user.id],
)
other = create_asset(item=self.asset1.item, sequence=3, slug="rev-topic-other")
create_transcription(asset=other, user=self.anon, submitted=now())
chosen = find_next_reviewable_topic_asset(
self.topic,
self.user,
project_slug="",
item_id="",
original_asset_id=None,
)
self.assertEqual(chosen, other)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
def test_find_next_reviewable_topic_assets_excludes_user(self):
create_transcription(asset=self.asset1, user=self.user, submitted=now())
NextReviewableTopicAsset.objects.create(
asset=self.asset1,
topic=self.topic,
item=self.asset1.item,
item_item_id=self.asset1.item.item_id,
project=self.asset1.item.project,
project_slug=self.asset1.item.project.slug,
sequence=self.asset1.sequence,
transcriber_ids=[self.user.id],
)
NextReviewableTopicAsset.objects.create(
asset=self.asset2,
topic=self.topic,
item=self.asset2.item,
item_item_id=self.asset2.item.item_id,
project=self.asset2.item.project,
project_slug=self.asset2.item.project.slug,
sequence=self.asset2.sequence,
transcriber_ids=[],
)
queryset = find_cached_reviewable_topic_assets(self.topic, self.user)
self.assertNotIn(self.asset1.id, queryset.values_list("asset_id", flat=True))
self.assertIn(self.asset2.id, queryset.values_list("asset_id", flat=True))
@patch("concordia.utils.next_asset.reviewable.topic.get_registered_task")
def test_next_reviewable_cached_path_when_short_circuits_fail_topic(
self, mock_get_task
):
"""
Item+project short-circuits fail (only user's work), so we should pull
from the cached table and not spawn a task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Only user's submitted work in current item/project
create_transcription(asset=self.asset1, user=self.user, submitted=now())
create_transcription(asset=self.asset2, user=self.user, submitted=now())
# Cached eligible asset in another project (not reachable via short-circuit)
cached_project = create_project(
campaign=self.asset1.campaign,
slug="topic-cached-proj",
title="topic-cached-proj",
)
cached_item = create_item(project=cached_project, item_id="topic-cached-item")
cached_asset = create_asset(item=cached_item, slug="topic-cached-asset")
create_transcription(asset=cached_asset, user=self.anon, submitted=now())
NextReviewableTopicAsset.objects.create(
asset=cached_asset,
topic=self.topic,
item=cached_asset.item,
item_item_id=cached_asset.item.item_id,
project=cached_asset.item.project,
project_slug=cached_asset.item.project.slug,
sequence=cached_asset.sequence,
transcriber_ids=[],
)
chosen = find_next_reviewable_topic_asset(
self.topic,
self.user,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, cached_asset)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.topic.get_registered_task")
def test_next_reviewable_uses_cache_when_bypassing_short_circuits_topic(
self, mock_get_task
):
"""
Pass blanks for project/item so we bypass short-circuits and hit cache.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
cached_project = create_project(
campaign=self.asset1.campaign,
slug="topic-cached-proj-2",
title="topic-cached-proj-2",
)
cached_item = create_item(project=cached_project, item_id="topic-cached-item-2")
cached_asset = create_asset(item=cached_item, slug="topic-cached-asset-2")
create_transcription(asset=cached_asset, user=self.anon, submitted=now())
NextReviewableTopicAsset.objects.create(
asset=cached_asset,
topic=self.topic,
item=cached_asset.item,
item_item_id=cached_asset.item.item_id,
project=cached_asset.item.project,
project_slug=cached_asset.item.project.slug,
sequence=cached_asset.sequence,
transcriber_ids=[],
)
chosen = find_next_reviewable_topic_asset(
self.topic,
self.user,
project_slug="",
item_id="",
original_asset_id=None,
)
self.assertEqual(chosen, cached_asset)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.topic.get_registered_task")
def test_next_reviewable_manual_fallback_no_after_spawns_and_picks_lowest_seq_topic(
self, mock_get_task
):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
other_item = create_item(project=self.asset1.item.project, item_id="t-mf-item")
asset_x = create_asset(item=other_item, sequence=7, slug="t-mf-x")
asset_y = create_asset(item=other_item, sequence=8, slug="t-mf-y")
create_transcription(asset=asset_x, user=self.anon, submitted=now())
create_transcription(asset=asset_y, user=self.anon, submitted=now())
chosen = find_next_reviewable_topic_asset(
self.topic,
self.user,
project_slug="",
item_id="",
original_asset_id=None,
)
self.assertEqual(chosen, asset_x)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.reviewable.topic.get_registered_task")
def test_next_reviewable_manual_fallback_invalid_after_str_topic(
self, mock_get_task
):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
other_item = create_item(
project=self.asset1.item.project, item_id="t-mf-item-2"
)
asset_a = create_asset(item=other_item, sequence=1, slug="t-mf-a")
asset_b = create_asset(item=other_item, sequence=2, slug="t-mf-b")
create_transcription(asset=asset_a, user=self.anon, submitted=now())
create_transcription(asset=asset_b, user=self.anon, submitted=now())
chosen = find_next_reviewable_topic_asset(
self.topic,
self.user,
project_slug="",
item_id="",
original_asset_id="not-an-int",
)
self.assertEqual(chosen, asset_a)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
class ReviewableTopicInternalsTests(CreateTestUsers, TestCase):
def setUp(self):
self.anon = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(sequence=1, slug="rt-a1")
self.asset2 = create_asset(item=self.asset1.item, sequence=2, slug="rt-a2")
self.topic = create_topic(project=self.asset1.item.project)
def test_topic_reserved_asset_ids_subq_unfiltered(self):
# Reservation tied to this test topic
AssetTranscriptionReservation.objects.create(
asset=self.asset1,
reservation_token="rt-r1", # nosec
)
# Reservation in entirely different campaign/project
other_campaign = create_campaign(slug="rt-camp-x", title="rt-camp-x")
other_project = create_project(
campaign=other_campaign, slug="rt-proj-x", title="rt-proj-x"
)
other_item = create_item(project=other_project, item_id="rt-item-x")
other_asset = create_asset(item=other_item, slug="rt-asset-x")
AssetTranscriptionReservation.objects.create(
asset=other_asset,
reservation_token="rt-r2", # nosec
)
id_set = set(topic_reserved_asset_ids_subq().values_list("asset_id", flat=True))
self.assertIn(self.asset1.id, id_set)
self.assertIn(other_asset.id, id_set)
def test_topic_eligible_reviewable_base_qs_excludes_user_and_requires_submitted(
self,
):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.user, submitted=now())
asset3 = create_asset(item=self.asset1.item, sequence=3, slug="rt-a3")
# asset3 has no submitted timestamp -> not SUBMITTED
queryset_user = topic_eligible_reviewable_base_qs(self.topic, user=self.user)
self.assertIn(self.asset1, queryset_user)
self.assertNotIn(self.asset2, queryset_user)
self.assertNotIn(asset3, queryset_user)
queryset_none = topic_eligible_reviewable_base_qs(self.topic, user=None)
self.assertIn(self.asset1, queryset_none)
self.assertIn(self.asset2, queryset_none)
self.assertNotIn(asset3, queryset_none)
def test_topic_next_seq_after_none_missing_and_valid(self):
self.assertIsNone(topic_next_seq_after(None))
self.assertIsNone(topic_next_seq_after(987654321))
self.assertEqual(topic_next_seq_after(self.asset2.pk), self.asset2.sequence)
def test_topic_find_reviewable_in_item_after_none_returns_first(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
chosen = topic_find_reviewable_in_item(
self.topic,
self.user,
item_id=self.asset1.item.item_id,
after_asset_pk=None,
)
self.assertEqual(chosen, self.asset1)
def test_topic_find_reviewable_in_item_after_asset_in_other_item_ignores_gate(
self,
):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
other_item = create_item(
project=self.asset1.item.project, item_id="rt-other-item"
)
other_asset = create_asset(item=other_item, slug="rt-other-asset")
create_transcription(asset=other_asset, user=self.anon, submitted=now())
chosen = topic_find_reviewable_in_item(
self.topic,
self.user,
item_id=self.asset1.item.item_id,
after_asset_pk=other_asset.id,
)
self.assertEqual(chosen, self.asset1)
def test_topic_find_reviewable_in_item_after_asset_missing_ignores_gate(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
chosen = topic_find_reviewable_in_item(
self.topic,
self.user,
item_id=self.asset1.item.item_id,
after_asset_pk=123456789,
)
self.assertEqual(chosen, self.asset1)
def test_topic_find_reviewable_in_item_after_asset_sidc_ignores_gate(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
other_campaign = create_campaign(slug="rt-camp-b", title="rt-camp-b")
other_project = create_project(
campaign=other_campaign, slug="rt-proj-b", title="rt-proj-b"
)
other_item = create_item(
project=other_project, item_id=self.asset1.item.item_id
)
cross_asset = create_asset(item=other_item, slug="rt-cross")
create_transcription(asset=cross_asset, user=self.anon, submitted=now())
chosen = topic_find_reviewable_in_item(
self.topic,
self.user,
item_id=self.asset1.item.item_id,
after_asset_pk=cross_asset.id,
)
self.assertEqual(chosen, self.asset1)
def test_topic_find_reviewable_in_project_orders_and_excludes_user(self):
project = self.asset1.item.project
other_item = create_item(project=project, item_id="rt-p-item")
mine = create_asset(item=other_item, sequence=1, slug="rt-p-mine")
theirs = create_asset(item=other_item, sequence=2, slug="rt-p-theirs")
create_transcription(asset=mine, user=self.user, submitted=now())
create_transcription(asset=theirs, user=self.anon, submitted=now())
chosen = topic_find_reviewable_in_project(
self.topic,
self.user,
project_slug=project.slug,
after_asset_pk=self.asset1.id,
)
self.assertEqual(chosen, theirs)
def test_topic_find_reviewable_in_project_returns_none_when_only_users_work(self):
project = self.asset1.item.project
other_item = create_item(project=project, item_id="rt-p2")
mine = create_asset(item=other_item, sequence=1, slug="rt-p2-mine")
create_transcription(asset=mine, user=self.user, submitted=now())
chosen = topic_find_reviewable_in_project(
self.topic,
self.user,
project_slug=project.slug,
after_asset_pk=self.asset1.id,
)
self.assertIsNone(chosen)
def test_find_and_order_potential_reviewable_topic_assets_ordering(self):
base_item = self.asset1.item
same_item_next = create_asset(item=base_item, sequence=10, slug="rt-ci-next")
other_item_same_project = create_asset(
item=create_item(project=base_item.project, item_id="rt-it-2"),
sequence=5,
slug="rt-p-next",
)
other_project = create_project(
campaign=self.asset1.campaign, slug="rt-proj", title="rt-proj"
)
other_project_item = create_item(project=other_project, item_id="rt-it-3")
other_project_asset = create_asset(
item=other_project_item, sequence=1, slug="rt-op"
)
for asset in (same_item_next, other_item_same_project, other_project_asset):
create_transcription(asset=asset, user=self.anon, submitted=now())
NextReviewableTopicAsset.objects.create(
asset=asset,
topic=self.topic,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
sequence=asset.sequence,
transcriber_ids=[],
)
ordered = find_and_order_potential_reviewable_topic_assets(
self.topic,
self.user,
project_slug=base_item.project.slug,
item_id=base_item.item_id,
asset_pk=self.asset1.id,
).values_list("asset_id", flat=True)
ordered = list(ordered)
self.assertEqual(ordered[0], same_item_next.id)
self.assertEqual(ordered[1], other_item_same_project.id)
self.assertIn(other_project_asset.id, ordered[2:])
def test_order_potential_without_after_prefers_item_then_project_topic(self):
base_item = self.asset1.item
same_item = create_asset(item=base_item, sequence=9, slug="rt-ci-none")
same_project = create_asset(
item=create_item(project=base_item.project, item_id="rt-it-np"),
sequence=2,
slug="rt-p-none",
)
other_project = create_project(
campaign=self.asset1.campaign, slug="rt-proj-none", title="rt-proj-none"
)
other_project_item = create_item(project=other_project, item_id="rt-it-op-none")
other_project_asset = create_asset(
item=other_project_item, sequence=1, slug="rt-op-none"
)
for asset in (same_item, same_project, other_project_asset):
create_transcription(asset=asset, user=self.anon, submitted=now())
NextReviewableTopicAsset.objects.create(
asset=asset,
topic=self.topic,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
sequence=asset.sequence,
transcriber_ids=[],
)
ordered = find_and_order_potential_reviewable_topic_assets(
self.topic,
self.user,
project_slug=base_item.project.slug,
item_id=base_item.item_id,
asset_pk=None, # next_asset==0 branch
).values_list("asset_id", flat=True)
ordered = list(ordered)
self.assertEqual(ordered[0], same_item.id)
self.assertEqual(ordered[1], same_project.id)
self.assertIn(other_project_asset.id, ordered[2:])
def test_find_invalid_next_reviewable_topic_assets_reserved_and_wrong_status(
self,
):
# Reserved
reserved_asset = create_asset(
item=self.asset1.item, sequence=30, slug="rt-inv-res"
)
create_transcription(asset=reserved_asset, user=self.anon, submitted=now())
AssetTranscriptionReservation.objects.create(
asset=reserved_asset, reservation_token="rt-rv" # nosec
)
NextReviewableTopicAsset.objects.create(
asset=reserved_asset,
topic=self.topic,
item=reserved_asset.item,
item_item_id=reserved_asset.item.item_id,
project=reserved_asset.item.project,
project_slug=reserved_asset.item.project.slug,
sequence=reserved_asset.sequence,
transcriber_ids=[],
)
# Wrong status (IN_PROGRESS)
wrong_status_asset = create_asset(
item=self.asset1.item, sequence=31, slug="rt-inv-wrong"
)
create_transcription(asset=wrong_status_asset, user=self.anon)
NextReviewableTopicAsset.objects.create(
asset=wrong_status_asset,
topic=self.topic,
item=wrong_status_asset.item,
item_item_id=wrong_status_asset.item.item_id,
project=wrong_status_asset.item.project,
project_slug=wrong_status_asset.item.project.slug,
sequence=wrong_status_asset.sequence,
transcriber_ids=[],
)
invalid_ids = list(
find_invalid_next_reviewable_topic_assets(self.topic.id).values_list(
"asset_id", flat=True
)
)
self.assertIn(reserved_asset.id, invalid_ids)
self.assertIn(wrong_status_asset.id, invalid_ids)
def test_topic_project_short_circuit_internal_skips_reserved_first(self):
project = self.asset1.item.project
item2 = create_item(project=project, item_id="rt-proj-int")
first = create_asset(item=item2, sequence=1, slug="rt-proj-int-1")
second = create_asset(item=item2, sequence=2, slug="rt-proj-int-2")
for asset in (first, second):
create_transcription(asset=asset, user=self.anon, submitted=now())
AssetTranscriptionReservation.objects.create(
asset=first, reservation_token="rt-proj-int" # nosec
)
chosen = topic_find_reviewable_in_project(
self.topic,
self.user,
project_slug=project.slug,
after_asset_pk=self.asset1.id,
)
self.assertEqual(chosen, second)
def test_topic_item_short_circuit_internal_excludes_users_own_work(self):
mine = self.asset1
other = self.asset2
create_transcription(asset=mine, user=self.user, submitted=now())
create_transcription(asset=other, user=self.anon, submitted=now())
chosen = topic_find_reviewable_in_item(
self.topic,
self.user,
item_id=mine.item.item_id,
after_asset_pk=None,
)
self.assertEqual(chosen, other)
def test_topic_item_short_circuit_internal_applies_after_and_skips_reserved(self):
asset3 = create_asset(item=self.asset1.item, sequence=3, slug="rt-int-a3")
for asset in (self.asset1, self.asset2, asset3):
create_transcription(asset=asset, user=self.anon, submitted=now())
AssetTranscriptionReservation.objects.create(
asset=self.asset2, reservation_token="rt-int-rv" # nosec
)
chosen = topic_find_reviewable_in_item(
self.topic,
self.user,
item_id=self.asset1.item.item_id,
after_asset_pk=self.asset1.id,
)
self.assertEqual(chosen, asset3)
================================================
FILE: concordia/tests/test_utils_next_asset_transcribable_campaign.py
================================================
from unittest.mock import MagicMock, patch
from django.test import TestCase
from django.utils.timezone import now
from concordia.models import (
Asset,
AssetTranscriptionReservation,
NextTranscribableCampaignAsset,
TranscriptionStatus,
)
from concordia.utils import get_anonymous_user
from concordia.utils.next_asset import (
find_new_transcribable_campaign_assets,
find_next_transcribable_campaign_asset,
find_transcribable_campaign_asset,
)
from concordia.utils.next_asset.transcribable.campaign import (
_eligible_transcribable_base_qs as tc_eligible_base_qs,
)
from concordia.utils.next_asset.transcribable.campaign import (
_find_transcribable_in_item as tc_find_in_item,
)
from concordia.utils.next_asset.transcribable.campaign import (
_find_transcribable_not_started_in_project as tc_find_ns_in_proj,
)
from concordia.utils.next_asset.transcribable.campaign import (
_next_seq_after as tc_next_seq_after,
)
from concordia.utils.next_asset.transcribable.campaign import (
_order_unstarted_first as tc_order_unstarted_first,
)
from concordia.utils.next_asset.transcribable.campaign import (
_reserved_asset_ids_subq as tc_reserved_ids_subq,
)
from concordia.utils.next_asset.transcribable.campaign import (
find_and_order_potential_transcribable_campaign_assets,
find_invalid_next_transcribable_campaign_assets,
)
from concordia.utils.next_asset.transcribable.campaign import (
find_next_transcribable_campaign_assets as find_cached_transcribable_assets,
)
from .utils import (
CreateTestUsers,
create_asset,
create_campaign,
create_item,
create_project,
create_transcription,
)
class NextTranscribableCampaignAssetTests(CreateTestUsers, TestCase):
def setUp(self):
self.anon = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(
sequence=1, slug="test-asset-1", title="Test Asset 1"
)
self.asset2 = create_asset(
item=self.asset1.item, sequence=2, slug="test-asset-2", title="Test Asset 2"
)
self.campaign = self.asset1.campaign
def test_find_new_transcribable_campaign_assets_filters_correctly(self):
create_transcription(
asset=self.asset1,
user=self.anon,
submitted=now(),
)
queryset = find_new_transcribable_campaign_assets(self.campaign)
self.assertNotIn(self.asset1, queryset)
self.assertIn(self.asset2, queryset)
def test_find_transcribable_campaign_asset_from_next_table(self):
NextTranscribableCampaignAsset.objects.create(
asset=self.asset1,
campaign=self.campaign,
item=self.asset1.item,
item_item_id=self.asset1.item.item_id,
project=self.asset1.item.project,
project_slug=self.asset1.item.project.slug,
sequence=self.asset1.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
asset = find_transcribable_campaign_asset(self.campaign)
self.assertEqual(asset, self.asset1)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_find_transcribable_campaign_asset_falls_back_and_spawns_task(
self, mock_get_task
):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset = find_transcribable_campaign_asset(self.campaign)
self.assertEqual(asset, self.asset1)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_find_next_transcribable_campaign_asset_orders_and_falls_back(
self, mock_get_task
):
"""
With short-circuiting: item-level returns the next asset
and we do not spawn a task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(asset, self.asset2)
# Short-circuit satisfied -> no cache fallback -> no task spawned
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_find_next_transcribable_campaign_asset_when_next_asset_exists(
self, mock_get_task
):
# Make asset2 eligible (IN_PROGRESS)
create_transcription(
asset=self.asset2,
user=self.anon,
)
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Cache has asset2
NextTranscribableCampaignAsset.objects.create(
asset=self.asset2,
campaign=self.campaign,
item=self.asset2.item,
item_item_id=self.asset2.item.item_id,
project=self.asset2.item.project,
project_slug=self.asset2.item.project.slug,
sequence=self.asset2.sequence,
transcription_status=TranscriptionStatus.IN_PROGRESS,
)
# Bypass item/project short-circuits so we hit the cache
asset = find_next_transcribable_campaign_asset(
self.campaign,
project_slug="",
item_id="",
original_asset_id=None,
)
self.assertEqual(asset, self.asset2)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
def test_short_circuit_same_item_respects_after_sequence_and_reservations(self):
"""
Same item short-circuit:
- Picks the next by sequence (> original)
- Skips reserved assets
"""
asset3 = create_asset(
item=self.asset1.item, sequence=3, slug="test-asset-3", title="Test Asset 3"
)
# Normal: after asset1 => choose asset2
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, self.asset2)
# Reserve asset2 => should skip to asset3
AssetTranscriptionReservation.objects.create(
asset=self.asset2, reservation_token="tkn" # nosec
)
chosen2 = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen2, asset3)
def test_project_short_circuit_prefers_not_started_over_in_progress(self):
"""
When item-level has no eligible assets, project-level should:
- Prefer NOT_STARTED over IN_PROGRESS
- Order by (item_id, sequence) within same status
"""
# Exhaust item: mark both item assets submitted
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
other_item = create_item(
project=self.asset1.item.project, item_id="proj-only-item"
)
in_progress_asset = create_asset(
item=other_item, slug="proj-inprog", title="Proj InProg"
)
create_transcription(asset=in_progress_asset, user=self.anon) # IN_PROGRESS
not_started_asset = create_asset(
item=other_item, slug="proj-notstarted", title="Proj NotStarted"
)
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id, # same item, but it's exhausted
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, not_started_asset)
def test_project_short_circuit_when_item_id_empty_string(self):
"""
If item_id is '', skip item short-circuit and use project-level.
"""
other_item = create_item(project=self.asset1.item.project, item_id="proj2")
project_asset = create_asset(
item=other_item, slug="proj-asset", title="Proj Asset"
)
# Make current item ineligible
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id="", # empty skips item short-circuit
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, project_asset)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_find_transcribable_campaign_asset_none_spawns(self, mock_get_task):
"""
When no NOT_STARTED/IN_PROGRESS exist, return None and trigger populate.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Make both assets SUBMITTED (no transcribable remain)
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
asset = find_transcribable_campaign_asset(self.campaign)
self.assertIsNone(asset)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_next_transcribable_manual_no_after_prefers_not_started(
self, mock_get_task
):
"""
With no short-circuit and empty cache, pick NOT_STARTED and spawn task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
campaign2 = create_campaign(slug="tc-na-camp", title="tc-na-camp")
project2 = create_project(
campaign=campaign2, slug="tc-na-proj", title="tc-na-proj"
)
item2 = create_item(project=project2, item_id="tc-na-item")
not_started_asset = create_asset(
item=item2, sequence=2, slug="tc-na-ns", title="TC NA NS"
)
in_progress_asset = create_asset(
item=item2, sequence=1, slug="tc-na-ip", title="TC NA IP"
)
create_transcription(asset=in_progress_asset, user=self.anon) # IN_PROGRESS
chosen = find_next_transcribable_campaign_asset(
campaign2, project_slug="", item_id="", original_asset_id=None
)
self.assertEqual(chosen, not_started_asset)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_next_transcribable_manual_invalid_after_str(self, mock_get_task):
"""
Treat a non-integer "after" like None: choose NOT_STARTED and spawn task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Make existing setup assets ineligible for selection.
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
other_item = create_item(
project=self.asset1.item.project, item_id="tc-mf-item-2"
)
asset_a = create_asset(
item=other_item, sequence=1, slug="tc-mf-a", title="TC MF A"
)
asset_b = create_asset(
item=other_item, sequence=2, slug="tc-mf-b", title="TC MF B"
)
create_transcription(asset=asset_b, user=self.anon) # IN_PROGRESS
chosen = find_next_transcribable_campaign_asset(
self.campaign, project_slug="", item_id="", original_asset_id=None
)
self.assertEqual(chosen, asset_a)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_next_transcribable_none_anywhere_spawns(self, mock_get_task):
"""
With no cache and no manual candidates: return None; do not spawn a task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
campaign2 = create_campaign(slug="tc-none-camp", title="tc-none-camp")
chosen = find_next_transcribable_campaign_asset(
campaign2, project_slug="", item_id="", original_asset_id=None
)
self.assertIsNone(chosen)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
def test_item_short_circuit_missing_after_pk_treated_as_none_top(self):
# Both assets are NOT_STARTED in the same item. Give a missing "after".
missing_pk = 987_654_321
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=missing_pk,
)
# With no valid "after" seq, returns the first NOT_STARTED (asset1).
self.assertEqual(chosen, self.asset1)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_cache_excludes_original_pk_and_chooses_next(self, mock_get_task):
# Two cached rows; original points at the first -> second should be chosen.
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
other_item = create_item(
project=self.asset1.item.project, item_id="tc-cache-pk-item"
)
first = create_asset(item=other_item, sequence=1, slug="tc-cache-first")
second = create_asset(item=other_item, sequence=2, slug="tc-cache-second")
for asset in (first, second):
NextTranscribableCampaignAsset.objects.create(
asset=asset,
campaign=self.campaign,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
sequence=asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug="",
item_id="",
original_asset_id=first.id,
)
self.assertEqual(chosen, second)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
def test_project_short_circuit_without_original_id(self):
# Exhaust current item; ensure project-level returns NOT_STARTED with
# original_asset_id=None.
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
other_item = create_item(
project=self.asset1.item.project, item_id="tc-proj-no-orig"
)
pick = create_asset(item=other_item, sequence=5, slug="tc-proj-pick")
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=None,
)
self.assertEqual(chosen, pick)
def test_item_short_circuit_after_pk_in_other_item_ignores_gate(self):
# Original PK exists but belongs to a different item; treat as no "after".
other_item = create_item(
project=self.asset1.item.project, item_id="tc-oth-it-ignores-gate"
)
other_asset = create_asset(item=other_item, slug="tc-oth-a-ignores-gate")
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=other_asset.id,
)
# With no valid "after" in this item, pick first NOT_STARTED by sequence.
self.assertEqual(chosen, self.asset1)
def test_next_transcribable_after_pk_missing_treats_as_no_after(self):
"""
Missing original_asset_id -> ignore 'after' gate and pick first NS in item.
"""
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=987654321, # missing
)
self.assertEqual(chosen, self.asset1)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_no_ns_anywhere_and_no_ip_in_item_returns_none(self, mock_get_task):
"""
With item_id present: no NOT_STARTED anywhere and no same-item IN_PROGRESS
so return None and do not spawn a task.
"""
# Exhaust the only item in the project/campaign.
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
got = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertIsNone(got)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
def test_after_pk_digit_string_missing_treats_as_no_after(self):
"""
original_asset_id is a digit string for a missing PK -> treat like no 'after'.
Covers the DoesNotExist branch distinct from non-digit ValueError.
"""
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id="987654321", # digit string, no such Asset
)
self.assertEqual(chosen, self.asset1)
def test_same_item_inprogress_selected_when_no_ns_and_no_after(self):
"""
With item_id present, no NOT_STARTED anywhere and original_asset_id=None,
pick same-item IN_PROGRESS (after_seq is None path).
"""
create_transcription(asset=self.asset2, user=self.anon) # IN_PROGRESS
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
got = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=None, # after_seq is None in IP fallback
)
self.assertEqual(got, self.asset2)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_manual_invalid_after_str_campaign_valueerror_branch(self, mock_get_task):
"""
original_asset_id is a non-digit string -> ValueError path.
Bypass short-circuits and empty cache => manual picks first NOT_STARTED
and spawns populate task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug="",
item_id="",
original_asset_id="not-an-int",
)
self.assertEqual(chosen, self.asset1) # first NOT_STARTED by ordering
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
class TranscribableCampaignInternalsTests(CreateTestUsers, TestCase):
def setUp(self):
self.anon = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(sequence=1, slug="tc-a1")
self.asset2 = create_asset(item=self.asset1.item, sequence=2, slug="tc-a2")
self.campaign = self.asset1.campaign
def test_new_transcribable_excludes_reserved_and_cached(self):
reserved_asset = create_asset(item=self.asset1.item, sequence=3, slug="tc-res")
cached_asset = create_asset(item=self.asset1.item, sequence=4, slug="tc-cached")
# Make both potentially transcribable
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
# Reserve one and cache the other
AssetTranscriptionReservation.objects.create(
asset=reserved_asset, reservation_token="tc-rv" # nosec
)
NextTranscribableCampaignAsset.objects.create(
asset=cached_asset,
campaign=self.campaign,
item=cached_asset.item,
item_item_id=cached_asset.item.item_id,
project=cached_asset.item.project,
project_slug=cached_asset.item.project.slug,
sequence=cached_asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
queryset = find_new_transcribable_campaign_assets(self.campaign)
self.assertNotIn(reserved_asset, queryset)
self.assertNotIn(cached_asset, queryset)
def test_order_potential_transcribable_pref(self):
"""
Cached ordering should favor next id, then same project, then same item.
"""
base_item = self.asset1.item
same_item_next = create_asset(item=base_item, sequence=10, slug="tc-ci-next")
same_project = create_asset(
item=create_item(project=base_item.project, item_id="tc-it-2"),
sequence=5,
slug="tc-p-next",
)
other_project_asset = create_asset(
item=create_item(
project=create_project(
campaign=self.campaign, slug="tc-op-proj", title="tc-op-proj"
),
item_id="tc-op-item",
),
sequence=1,
slug="tc-op",
)
for asset in (same_item_next, same_project, other_project_asset):
NextTranscribableCampaignAsset.objects.create(
asset=asset,
campaign=self.campaign,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
sequence=asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
ordered = find_and_order_potential_transcribable_campaign_assets(
self.campaign,
project_slug=base_item.project.slug,
item_id=base_item.item_id,
asset_pk=self.asset1.id,
).values_list("asset_id", flat=True)
ordered = list(ordered)
self.assertEqual(ordered[0], same_item_next.id)
self.assertEqual(ordered[1], same_project.id)
self.assertIn(other_project_asset.id, ordered[2:])
def test_order_potential_transcribable_no_after(self):
"""
With no 'after', prefer same item, then same project.
"""
base_item = self.asset1.item
same_item = create_asset(item=base_item, sequence=9, slug="tc-ci-none")
same_project = create_asset(
item=create_item(project=base_item.project, item_id="tc-it-np"),
sequence=2,
slug="tc-p-none",
)
other_project_asset = create_asset(
item=create_item(
project=create_project(
campaign=self.campaign, slug="tc-proj-none", title="tc-proj-none"
),
item_id="tc-it-op-none",
),
sequence=1,
slug="tc-op-none",
)
for asset in (same_item, same_project, other_project_asset):
NextTranscribableCampaignAsset.objects.create(
asset=asset,
campaign=self.campaign,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
sequence=asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
ordered = find_and_order_potential_transcribable_campaign_assets(
self.campaign,
project_slug=base_item.project.slug,
item_id=base_item.item_id,
asset_pk=None,
).values_list("asset_id", flat=True)
ordered = list(ordered)
self.assertEqual(ordered[0], same_item.id)
self.assertEqual(ordered[1], same_project.id)
self.assertIn(other_project_asset.id, ordered[2:])
def test_invalid_next_transcribable_reserved_and_submitted(self):
"""
Invalid cache rows include reserved or SUBMITTED assets.
"""
reserved_asset = create_asset(
item=self.asset1.item, sequence=30, slug="tc-inv-res"
)
AssetTranscriptionReservation.objects.create(
asset=reserved_asset, reservation_token="tc-rv-2" # nosec
)
NextTranscribableCampaignAsset.objects.create(
asset=reserved_asset,
campaign=self.campaign,
item=reserved_asset.item,
item_item_id=reserved_asset.item.item_id,
project=reserved_asset.item.project,
project_slug=reserved_asset.item.project.slug,
sequence=reserved_asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
wrong_status_asset = create_asset(
item=self.asset1.item, sequence=31, slug="tc-inv-wrong"
)
create_transcription(asset=wrong_status_asset, user=self.anon, submitted=now())
NextTranscribableCampaignAsset.objects.create(
asset=wrong_status_asset,
campaign=self.campaign,
item=wrong_status_asset.item,
item_item_id=wrong_status_asset.item.item_id,
project=wrong_status_asset.item.project,
project_slug=wrong_status_asset.item.project.slug,
sequence=wrong_status_asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
bad = list(
find_invalid_next_transcribable_campaign_assets(
self.campaign.id
).values_list("asset_id", flat=True)
)
self.assertIn(reserved_asset.id, bad)
self.assertIn(wrong_status_asset.id, bad)
class TranscribableCampaignMoreInternalsTests(CreateTestUsers, TestCase):
def setUp(self):
self.anon = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(sequence=1, slug="tc-more-a1", title="TC More A1")
self.asset2 = create_asset(
item=self.asset1.item, sequence=2, slug="tc-more-a2", title="TC More A2"
)
self.campaign = self.asset1.campaign
def test_tc_reserved_ids_filters_to_campaign(self):
AssetTranscriptionReservation.objects.create(
asset=self.asset1, reservation_token="tc-res-here" # nosec
)
other_campaign = create_campaign(slug="tc-other-c", title="tc-other-c")
other_project = create_project(
campaign=other_campaign, slug="tc-other-p", title="tc-other-p"
)
other_item = create_item(project=other_project, item_id="tc-other-it")
other_asset = create_asset(item=other_item, slug="tc-other-a")
AssetTranscriptionReservation.objects.create(
asset=other_asset, reservation_token="tc-res-there" # nosec
)
id_set = set(
tc_reserved_ids_subq(self.campaign).values_list("asset_id", flat=True)
)
self.assertIn(self.asset1.id, id_set)
self.assertNotIn(other_asset.id, id_set)
def test_tc_next_seq_after_variants(self):
self.assertIsNone(tc_next_seq_after(None))
self.assertIsNone(tc_next_seq_after(999_999_999))
self.assertEqual(tc_next_seq_after(self.asset2.id), self.asset2.sequence)
def test_tc_order_unstarted_first_prefers_not_started(self):
create_transcription(asset=self.asset2, user=self.anon)
queryset = Asset.objects.filter(id__in=[self.asset1.id, self.asset2.id])
ordered = list(tc_order_unstarted_first(queryset).values_list("id", flat=True))
self.assertEqual(ordered[0], self.asset1.id)
self.assertEqual(ordered[1], self.asset2.id)
def test_find_in_item_after_none_returns_first_not_started(self):
item_id = self.asset1.item.item_id
chosen = tc_find_in_item(self.campaign, item_id=item_id, after_asset_pk=None)
self.assertEqual(chosen, self.asset1)
def test_find_in_item_skips_inprog_and_reserved_and_advances(self):
create_transcription(asset=self.asset1, user=self.anon)
asset3 = create_asset(item=self.asset1.item, sequence=3, slug="tc-more-a3")
AssetTranscriptionReservation.objects.create(
asset=asset3, reservation_token="tc-res-a3" # nosec
)
chosen = tc_find_in_item(
self.campaign,
item_id=self.asset1.item.item_id,
after_asset_pk=self.asset1.id,
)
self.assertEqual(chosen, self.asset2)
def test_find_in_item_after_missing_excludes_id_only(self):
missing_pk = 987654321
chosen = tc_find_in_item(
self.campaign, item_id=self.asset1.item.item_id, after_asset_pk=missing_pk
)
self.assertEqual(chosen, self.asset1)
def test_find_ns_in_proj_excludes_item_and_reserved(self):
project = self.asset1.item.project
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
item2 = create_item(project=project, item_id="tc-more-it-2")
not_started1 = create_asset(item=item2, sequence=1, slug="tc-more-ns1")
not_started2 = create_asset(item=item2, sequence=2, slug="tc-more-ns2")
AssetTranscriptionReservation.objects.create(
asset=not_started1, reservation_token="tc-res-ns1" # nosec
)
chosen = tc_find_ns_in_proj(
self.campaign,
project_slug=project.slug,
exclude_item_id=self.asset1.item.item_id,
)
self.assertEqual(chosen, not_started2)
def test_find_ns_in_proj_blank_slug_none(self):
self.assertIsNone(tc_find_ns_in_proj(self.campaign, project_slug=""))
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_cache_same_item_is_ignored_then_manual_selects(self, mock_get_task):
"""
Same-item cache entries should be ignored; manual should return other item.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
NextTranscribableCampaignAsset.objects.create(
asset=self.asset2,
campaign=self.campaign,
item=self.asset2.item,
item_item_id=self.asset2.item.item_id,
project=self.asset2.item.project,
project_slug=self.asset2.item.project.slug,
sequence=self.asset2.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
item2 = create_item(project=self.asset1.item.project, item_id="tc-more-it-man")
picked_asset = create_asset(item=item2, sequence=10, slug="tc-more-pick")
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug="", # skip project-level short-circuit
item_id=self.asset1.item.item_id, # forces same-item short-circuit first
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, picked_asset)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.campaign.get_registered_task")
def test_manual_excludes_original_pk_and_same_item(self, mock_get_task):
"""
Manual ranking must exclude the original asset and the current item.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
item2 = create_item(project=self.asset1.item.project, item_id="tc-more-it-3")
keep = create_asset(item=item2, sequence=1, slug="tc-more-keep")
toss = create_asset(item=item2, sequence=2, slug="tc-more-toss")
chosen = find_next_transcribable_campaign_asset(
self.campaign,
project_slug="",
item_id=self.asset1.item.item_id,
original_asset_id=toss.id,
)
self.assertEqual(chosen, keep)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
def test_same_item_inprog_after_when_no_not_started(self):
"""
If no NOT_STARTED anywhere qualifies, select IN_PROGRESS in same item.
"""
create_transcription(asset=self.asset2, user=self.anon) # IN_PROGRESS
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
got = find_next_transcribable_campaign_asset(
self.campaign,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(got, self.asset2)
def test_eligible_base_qs_filters_status_and_published(self):
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
not_started_asset = create_asset(
item=self.asset1.item, sequence=3, slug="tc-more-ns-ok"
)
in_progress_asset = create_asset(
item=self.asset1.item, sequence=4, slug="tc-more-ip-ok"
)
create_transcription(asset=in_progress_asset, user=self.anon) # IN_PROGRESS
other_campaign = create_campaign(slug="tc-ebq-c", title="tc-ebq-c")
other_project = create_project(
campaign=other_campaign, slug="tc-ebq-p", title="tc-ebq-p"
)
other_item = create_item(project=other_project, item_id="tc-ebq-i")
other_asset = create_asset(item=other_item, slug="tc-ebq-a")
queryset = tc_eligible_base_qs(self.campaign)
id_set = set(queryset.values_list("id", flat=True))
self.assertIn(not_started_asset.id, id_set)
self.assertIn(in_progress_asset.id, id_set)
self.assertNotIn(self.asset1.id, id_set)
self.assertNotIn(other_asset.id, id_set)
def test_cached_transcribable_accessor_returns_rows(self):
row = NextTranscribableCampaignAsset.objects.create(
asset=self.asset1,
campaign=self.campaign,
item=self.asset1.item,
item_item_id=self.asset1.item.item_id,
project=self.asset1.item.project,
project_slug=self.asset1.item.project.slug,
sequence=self.asset1.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
queryset = find_cached_transcribable_assets(self.campaign)
self.assertIn(row.id, queryset.values_list("id", flat=True))
def test_find_in_item_blank_item_id_none(self):
chosen = tc_find_in_item(self.campaign, item_id="", after_asset_pk=None)
self.assertIsNone(chosen)
def test_find_ns_in_proj_without_exclude_includes_same_item(self):
"""
exclude_item_id is falsy, so branch where no exclusion is applied.
Should pick the first NOT_STARTED asset, even if it's in the same item.
"""
project = self.asset1.item.project
chosen = tc_find_ns_in_proj(self.campaign, project_slug=project.slug)
self.assertEqual(chosen, self.asset1)
================================================
FILE: concordia/tests/test_utils_next_asset_transcribable_topic.py
================================================
from unittest.mock import MagicMock, patch
from django.test import TestCase
from django.utils.timezone import now
from concordia.models import (
Asset,
AssetTranscriptionReservation,
NextTranscribableTopicAsset,
TranscriptionStatus,
)
from concordia.utils import get_anonymous_user
from concordia.utils.next_asset import (
find_new_transcribable_topic_assets,
find_next_transcribable_topic_asset,
find_transcribable_topic_asset,
)
from concordia.utils.next_asset.transcribable.topic import (
_eligible_transcribable_base_qs as topic_transcribable_eligible_base_qs,
)
from concordia.utils.next_asset.transcribable.topic import (
_find_transcribable_in_item_for_topic as topic_find_in_item_for_topic,
)
from concordia.utils.next_asset.transcribable.topic import (
_find_transcribable_not_started_in_project_for_topic,
find_and_order_potential_transcribable_topic_assets,
find_invalid_next_transcribable_topic_assets,
)
from concordia.utils.next_asset.transcribable.topic import (
_next_seq_after as topic_next_seq_after_for_transcribable,
)
from concordia.utils.next_asset.transcribable.topic import (
_order_unstarted_first as topic_order_unstarted_first,
)
from concordia.utils.next_asset.transcribable.topic import (
_reserved_asset_ids_subq as topic_transcribable_reserved_ids_subq,
)
from .utils import (
CreateTestUsers,
create_asset,
create_campaign,
create_item,
create_project,
create_topic,
create_transcription,
)
topic_find_not_started_in_project_for_topic = (
_find_transcribable_not_started_in_project_for_topic
)
find_invalid_next_transcribable_topic_assets_fn = (
find_invalid_next_transcribable_topic_assets
)
class NextTranscribableTopicAssetTests(CreateTestUsers, TestCase):
def setUp(self):
self.anon = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(
slug="topic-asset-1", sequence=1, title="Topic Asset 1"
)
self.asset2 = create_asset(
item=self.asset1.item,
sequence=2,
slug="topic-asset-2",
title="Topic Asset 2",
)
self.topic = create_topic(project=self.asset1.item.project)
def test_find_new_transcribable_topic_assets_filters_correctly(self):
create_transcription(
asset=self.asset1,
user=self.anon,
submitted=now(),
)
queryset = find_new_transcribable_topic_assets(self.topic)
self.assertNotIn(self.asset1, queryset)
assert_in = self.assertIn
assert_in(self.asset2, queryset)
def test_find_transcribable_topic_asset_from_next_table(self):
NextTranscribableTopicAsset.objects.create(
asset=self.asset1,
topic=self.topic,
item=self.asset1.item,
item_item_id=self.asset1.item.item_id,
project=self.asset1.item.project,
project_slug=self.asset1.item.project.slug,
sequence=self.asset1.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
asset = find_transcribable_topic_asset(self.topic)
self.assertEqual(asset, self.asset1)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_find_transcribable_topic_asset_falls_back_and_spawns_task(
self, mock_get_task
):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset = find_transcribable_topic_asset(self.topic)
self.assertEqual(asset, self.asset1)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_find_next_transcribable_topic_asset_orders_and_falls_back(
self, mock_get_task
):
"""
With short-circuiting: item-level returns the next asset
and we do not spawn a task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
asset = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(asset, self.asset2)
# Short-circuit satisfied -> no cache fallback -> no task spawned
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_find_next_transcribable_topic_asset_when_next_asset_exists(
self, mock_get_task
):
create_transcription(asset=self.asset2, user=self.anon)
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
NextTranscribableTopicAsset.objects.create(
asset=self.asset2,
topic=self.topic,
item=self.asset2.item,
item_item_id=self.asset2.item.item_id,
project=self.asset2.item.project,
project_slug=self.asset2.item.project.slug,
sequence=self.asset2.sequence,
transcription_status=TranscriptionStatus.IN_PROGRESS,
)
asset = find_next_transcribable_topic_asset(
self.topic,
project_slug="",
item_id="",
original_asset_id=None,
)
self.assertEqual(asset, self.asset2)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
def test_short_circuit_same_item_topic_respects_after_sequence_and_reservations(
self,
):
third = create_asset(
item=self.asset1.item,
sequence=3,
slug="topic-asset-3",
title="Topic Asset 3",
)
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, self.asset2)
AssetTranscriptionReservation.objects.create(
asset=self.asset2, reservation_token="tkn" # nosec
)
chosen2 = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen2, third)
def test_project_short_circuit_topic_prefers_not_started_over_in_progress(self):
# Exhaust item
create_transcription(asset=self.asset1, user=self.anon, submitted=now())
create_transcription(asset=self.asset2, user=self.anon, submitted=now())
other_item = create_item(project=self.asset1.item.project, item_id="tproj-item")
in_progress = create_asset(
item=other_item, sequence=1, slug="tproj-inprog", title="TProj InProg"
)
create_transcription(asset=in_progress, user=self.anon)
not_started_asset = create_asset(
item=other_item,
sequence=2,
slug="tproj-notstarted",
title="TProj NotStarted",
)
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, not_started_asset)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_project_short_circuit_topic_without_item_id_allows_same_item(
self, mock_get_task
):
"""
With item_id not set, the project-level short-circuit
should return the first NOT_STARTED in the project,
ordered by (item__item_id, sequence, id).
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id="", # falsy -> do not exclude same item
original_asset_id=None, # no exclusion of original
)
# Both assets are NOT_STARTED in the same item; ordering picks asset1.
self.assertEqual(chosen, self.asset1)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
class TranscribableTopicInternalsTests(CreateTestUsers, TestCase):
def setUp(self):
self.anonymous = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(sequence=1, slug="tt-int-a1")
self.asset2 = create_asset(item=self.asset1.item, sequence=2, slug="tt-int-a2")
self.topic = create_topic(project=self.asset1.item.project)
def test_topic_transcribable_reserved_ids_is_unfiltered(self):
AssetTranscriptionReservation.objects.create(
asset=self.asset1, reservation_token="tt-res-here" # nosec
)
other_campaign = create_campaign(slug="tt-oc", title="tt-oc")
other_project = create_project(
campaign=other_campaign, slug="tt-op", title="tt-op"
)
other_item = create_item(project=other_project, item_id="tt-oi")
other_asset = create_asset(item=other_item, slug="tt-oa")
AssetTranscriptionReservation.objects.create(
asset=other_asset, reservation_token="tt-res-there" # nosec
)
ids = set(
topic_transcribable_reserved_ids_subq().values_list("asset_id", flat=True)
)
self.assertIn(self.asset1.id, ids)
self.assertIn(other_asset.id, ids)
def test_topic_transcribable_eligible_base_qs_filters_correctly(self):
# Submitted, so excluded
create_transcription(asset=self.asset2, user=self.anonymous, submitted=now())
# Not started (included)
asset_not_started = self.asset1
# In progress (included)
asset_in_progress = create_asset(
item=self.asset1.item, sequence=3, slug="tt-int-ip"
)
create_transcription(asset=asset_in_progress, user=self.anonymous)
# Other campaign (excluded)
other_campaign = create_campaign(slug="tt-ebq-c", title="tt-ebq-c")
other_project = create_project(
campaign=other_campaign, slug="tt-ebq-p", title="tt-ebq-p"
)
other_item = create_item(project=other_project, item_id="tt-ebq-i")
other_asset = create_asset(item=other_item, slug="tt-ebq-a")
queryset = topic_transcribable_eligible_base_qs(self.topic)
ids = set(queryset.values_list("id", flat=True))
self.assertIn(asset_not_started.id, ids)
self.assertIn(asset_in_progress.id, ids)
self.assertNotIn(self.asset2.id, ids)
self.assertNotIn(other_asset.id, ids)
def test_topic_next_seq_after_variants_for_transcribable(self):
self.assertIsNone(topic_next_seq_after_for_transcribable(None))
self.assertIsNone(topic_next_seq_after_for_transcribable(987654321))
self.assertEqual(
topic_next_seq_after_for_transcribable(self.asset2.id),
self.asset2.sequence,
)
def test_topic_find_in_item_for_topic_after_none_returns_first_not_started(self):
chosen = topic_find_in_item_for_topic(
self.topic, item_id=self.asset1.item.item_id, after_asset_pk=None
)
self.assertEqual(chosen, self.asset1)
def test_topic_find_in_item_for_topic_skips_reserved_and_advances(self):
third = create_asset(item=self.asset1.item, sequence=3, slug="tt-int-a3")
AssetTranscriptionReservation.objects.create(
asset=self.asset2, reservation_token="tt-int-a2" # nosec
)
chosen = topic_find_in_item_for_topic(
self.topic,
item_id=self.asset1.item.item_id,
after_asset_pk=self.asset1.id,
)
self.assertEqual(chosen, third)
def test_topic_find_in_item_for_topic_after_missing_excludes_only_id(self):
chosen = topic_find_in_item_for_topic(
self.topic,
item_id=self.asset1.item.item_id,
after_asset_pk=987654321,
)
self.assertEqual(chosen, self.asset1)
def test_topic_find_in_item_for_topic_blank_item_id_returns_none(self):
chosen = topic_find_in_item_for_topic(
self.topic, item_id="", after_asset_pk=None
)
self.assertIsNone(chosen)
def test_topic_find_not_started_in_project_excludes_item_and_reserved(self):
other_item = create_item(project=self.asset1.item.project, item_id="tt-int-ex")
not_started_1 = create_asset(item=other_item, sequence=1, slug="tt-int-ns1")
not_started_2 = create_asset(item=other_item, sequence=2, slug="tt-int-ns2")
AssetTranscriptionReservation.objects.create(
asset=not_started_1, reservation_token="tt-int-res" # nosec
)
chosen = topic_find_not_started_in_project_for_topic(
self.topic,
project_slug=self.asset1.item.project.slug,
exclude_item_id=self.asset1.item.item_id,
)
self.assertEqual(chosen, not_started_2)
def test_topic_find_not_started_in_project_blank_slug_none(self):
self.assertIsNone(
topic_find_not_started_in_project_for_topic(self.topic, project_slug="")
)
def test_topic_order_unstarted_first_prefers_not_started(self):
in_progress = create_asset(
item=self.asset1.item, sequence=4, slug="tt-int-ip-2"
)
create_transcription(asset=in_progress, user=self.anonymous)
queryset = Asset.objects.filter(id__in=[self.asset1.id, in_progress.id])
ordered = list(
topic_order_unstarted_first(queryset).values_list("id", flat=True)
)
self.assertEqual(ordered[0], self.asset1.id)
self.assertEqual(ordered[1], in_progress.id)
def test_topic_find_not_started_in_project_without_exclude_includes_same_item(
self,
):
"""
With exclude_item_id falsy, the helper should consider the same item and
pick the first NOT_STARTED asset there (covers the else branch of L154->157).
"""
project = self.asset1.item.project
chosen = topic_find_not_started_in_project_for_topic(
self.topic, project_slug=project.slug, exclude_item_id=""
)
self.assertEqual(chosen, self.asset1)
class NextTranscribableTopicMoreTests(CreateTestUsers, TestCase):
def setUp(self):
self.anonymous = get_anonymous_user()
self.user = self.create_test_user()
self.asset1 = create_asset(slug="tt-more-a1", sequence=1, title="TT More A1")
self.asset2 = create_asset(
item=self.asset1.item, slug="tt-more-a2", sequence=2, title="TT More A2"
)
self.topic = create_topic(project=self.asset1.item.project)
def test_new_transcribable_topic_excludes_reserved_and_cached(self):
reserved_asset = create_asset(
item=self.asset1.item, sequence=3, slug="tt-more-res"
)
cached_asset = create_asset(
item=self.asset1.item, sequence=4, slug="tt-more-cached"
)
NextTranscribableTopicAsset.objects.create(
asset=cached_asset,
topic=self.topic,
item=cached_asset.item,
item_item_id=cached_asset.item.item_id,
project=cached_asset.item.project,
project_slug=cached_asset.item.project.slug,
sequence=cached_asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
AssetTranscriptionReservation.objects.create(
asset=reserved_asset, reservation_token="tt-more-rv" # nosec
)
queryset = find_new_transcribable_topic_assets(self.topic)
self.assertNotIn(reserved_asset, queryset)
self.assertNotIn(cached_asset, queryset)
def test_find_and_order_potential_transcribable_topic_assets_ordering(self):
base_item = self.asset1.item
same_item_next = create_asset(
item=base_item, sequence=10, slug="tt-pot-ci-next"
)
same_project_other_item = create_asset(
item=create_item(project=base_item.project, item_id="tt-pot-it-2"),
sequence=5,
slug="tt-pot-proj",
)
other_project = create_project(
campaign=self.asset1.campaign,
slug="tt-pot-proj-oth",
title="tt-pot-proj-oth",
)
other_item = create_item(project=other_project, item_id="tt-pot-it-3")
other_project_asset = create_asset(
item=other_item, sequence=1, slug="tt-pot-op"
)
for asset in (same_item_next, same_project_other_item, other_project_asset):
NextTranscribableTopicAsset.objects.create(
asset=asset,
topic=self.topic,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
sequence=asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
ordered = find_and_order_potential_transcribable_topic_assets(
self.topic,
project_slug=base_item.project.slug,
item_id=base_item.item_id,
asset_pk=self.asset1.id,
).values_list("asset_id", flat=True)
ordered = list(ordered)
# Prefer same item, then same project, then others
self.assertEqual(ordered[0], same_item_next.id)
self.assertEqual(ordered[1], same_project_other_item.id)
self.assertIn(other_project_asset.id, ordered[2:])
def test_find_invalid_next_transcribable_topic_assets_reserved_and_status(self):
reserved_asset = create_asset(
item=self.asset1.item, sequence=30, slug="tt-inv-res"
)
create_transcription(asset=reserved_asset, user=self.anonymous)
AssetTranscriptionReservation.objects.create(
asset=reserved_asset, reservation_token="tt-inv-rv" # nosec
)
NextTranscribableTopicAsset.objects.create(
asset=reserved_asset,
topic=self.topic,
item=reserved_asset.item,
item_item_id=reserved_asset.item.item_id,
project=reserved_asset.item.project,
project_slug=reserved_asset.item.project.slug,
sequence=reserved_asset.sequence,
transcription_status=TranscriptionStatus.IN_PROGRESS,
)
wrong_status_asset = create_asset(
item=self.asset1.item, sequence=31, slug="tt-inv-wrong"
)
create_transcription(
asset=wrong_status_asset, user=self.anonymous, submitted=now()
)
NextTranscribableTopicAsset.objects.create(
asset=wrong_status_asset,
topic=self.topic,
item=wrong_status_asset.item,
item_item_id=wrong_status_asset.item.item_id,
project=wrong_status_asset.item.project,
project_slug=wrong_status_asset.item.project.slug,
sequence=wrong_status_asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
bad = list(
find_invalid_next_transcribable_topic_assets_fn(self.topic.id).values_list(
"asset_id", flat=True
)
)
self.assertIn(reserved_asset.id, bad)
self.assertIn(wrong_status_asset.id, bad)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_cache_same_item_is_ignored_then_manual_selects_topic(self, mock_get_task):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Same-item cached row should be excluded, forcing manual fallback.
NextTranscribableTopicAsset.objects.create(
asset=self.asset2,
topic=self.topic,
item=self.asset2.item,
item_item_id=self.asset2.item.item_id,
project=self.asset2.item.project,
project_slug=self.asset2.item.project.slug,
sequence=self.asset2.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
# Make same-item short-circuit fail by reserving the only candidate.
AssetTranscriptionReservation.objects.create(
asset=self.asset2, reservation_token="tt-cache-same" # nosec
)
# Provide a valid choice elsewhere to be picked by manual fallback.
other_item = create_item(
project=self.asset1.item.project, item_id="tt-cache-oth"
)
picked = create_asset(item=other_item, sequence=5, slug="tt-cache-pick")
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug="",
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, picked)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_cache_excludes_original_pk_and_chooses_next_topic(self, mock_get_task):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
other_item = create_item(
project=self.asset1.item.project, item_id="tt-cache-exc"
)
first = create_asset(item=other_item, sequence=1, slug="tt-cache-first")
second = create_asset(item=other_item, sequence=2, slug="tt-cache-second")
for asset in (first, second):
NextTranscribableTopicAsset.objects.create(
asset=asset,
topic=self.topic,
item=asset.item,
item_item_id=asset.item.item_id,
project=asset.item.project,
project_slug=asset.item.project.slug,
sequence=asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
chosen = find_next_transcribable_topic_asset(
self.topic, project_slug="", item_id="", original_asset_id=first.id
)
self.assertEqual(chosen, second)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
def test_same_item_inprogress_selected_when_no_not_started_topic(self):
create_transcription(asset=self.asset2, user=self.anonymous)
create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())
got = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=None,
)
self.assertEqual(got, self.asset2)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_next_transcribable_topic_none_anywhere_returns_none_no_spawn(
self, mock_get_task
):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Use a brand-new topic with no eligible assets anywhere.
empty_campaign = create_campaign(slug="tt-none-c", title="tt-none-c")
empty_project = create_project(
campaign=empty_campaign, slug="tt-none-p", title="tt-none-p"
)
empty_topic = create_topic(project=empty_project)
chosen = find_next_transcribable_topic_asset(
topic=empty_topic, project_slug="", item_id="", original_asset_id=None
)
self.assertIsNone(chosen)
self.assertFalse(mock_get_task.called)
self.assertFalse(mock_task.delay.called)
def test_item_gate_ignored_when_original_is_other_item_topic(self):
"""
original_asset_id exists but belongs to a different item; item gate is ignored
and we return the first NOT_STARTED in the requested item
"""
other_item = create_item(
project=self.asset1.item.project, item_id="tt-oth-item"
)
other_asset = create_asset(item=other_item, slug="tt-oth-a")
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=other_asset.id,
)
self.assertEqual(chosen, self.asset1)
def test_item_digit_string_missing_treats_as_no_after_topic(self):
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id="987654321", # valid digits, missing PK
)
self.assertEqual(chosen, self.asset1)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_manual_same_item_ip_when_no_ns_anywhere_topic(self, mock_get_task):
"""
Manual fallback path with item_id present: when there are
no NOT_STARTED anywhere, choose same-item IN_PROGRESS.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# No NOT_STARTED anywhere in this topic's project; only same-item IN_PROGRESS
create_transcription(asset=self.asset2, user=self.anonymous) # IN_PROGRESS
create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())
got = find_next_transcribable_topic_asset(
self.topic,
project_slug="", # bypass short-circuit so we hit the manual path
item_id=self.asset1.item.item_id,
original_asset_id=None,
)
self.assertEqual(got, self.asset2)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
def test_item_invalid_after_str_valueerror_branch_topic(self):
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id="not-an-int",
)
self.assertEqual(chosen, self.asset1)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_manual_valid_after_excludes_original_and_picks_next_topic(
self, mock_get_task
):
"""
Manual fallback with a valid original_asset_id: use after_seq to exclude the
original and return the next NOT_STARTED
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
other_item = create_item(
project=self.asset1.item.project, item_id="tt-man-item"
)
first = create_asset(item=other_item, sequence=1, slug="tt-man-first")
second = create_asset(item=other_item, sequence=2, slug="tt-man-second")
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug="",
item_id="",
original_asset_id=first.id,
)
self.assertEqual(chosen, second)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_inprogress_fallback_spawns_and_uses_after_gate_topic(self, mock_get_task):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Make original (seq=1) SUBMITTED so it can't be chosen; keep it as "original".
create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())
# Only candidate anywhere: same-item IN_PROGRESS (seq=2).
create_transcription(asset=self.asset2, user=self.anonymous) # IN_PROGRESS
# Ensure there are no other items/assets in the topic to be found
# by manual path (manual path excludes same item when item_id is provided).
# No cached rows either.
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, self.asset2)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_inprogress_fallback_with_digit_string_original_id(self, mock_get_task):
"""
Same as above, but pass original_asset_id as a DIGIT STRING to
exercise the int(original_asset_id) path inside the after-seq filter.
Also ensures exclude(pk=original_asset_id) runs without ValueError.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Original: SUBMITTED, seq=1
create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())
# Only candidate: same-item IN_PROGRESS, seq=2
create_transcription(asset=self.asset2, user=self.anonymous)
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=str(self.asset1.id),
)
self.assertEqual(chosen, self.asset2)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_inprogress_fallback_spawns_and_returns_asset_topic(self, mock_get_task):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Use a brand-new topic/project so no cached rows can interfere.
campaign = create_campaign(slug="tt-ip-c1", title="tt-ip-c1")
project = create_project(
campaign=campaign,
slug="tt-ip-p1",
title="tt-ip-p1",
)
topic = create_topic(project=project)
item = create_item(project=project, item_id="tt-ip-i1")
asset1 = create_asset(item=item, sequence=1, slug="tt-ip-a1")
asset2 = create_asset(item=item, sequence=2, slug="tt-ip-a2")
create_transcription(asset=asset1, user=get_anonymous_user(), submitted=now())
create_transcription(asset=asset2, user=get_anonymous_user())
chosen = find_next_transcribable_topic_asset(
topic=topic,
project_slug=project.slug,
item_id=item.item_id,
original_asset_id=asset1.id,
)
self.assertEqual(chosen, asset2)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_inprogress_fallback_with_digit_str_original_id_topic(self, mock_get_task):
"""
Same scenario as above, but pass original_asset_id as a DIGIT STRING to
run the int(...) path inside the after-seq filter and still spawn the task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
campaign = create_campaign(slug="tt-ip-c2", title="tt-ip-c2")
project = create_project(
campaign=campaign,
slug="tt-ip-p2",
title="tt-ip-p2",
)
topic = create_topic(project=project)
item = create_item(project=project, item_id="tt-ip-i2")
asset1 = create_asset(item=item, sequence=1, slug="tt-ip2-a1")
asset2 = create_asset(item=item, sequence=2, slug="tt-ip2-a2")
create_transcription(asset=asset1, user=get_anonymous_user(), submitted=now())
create_transcription(asset=asset2, user=get_anonymous_user()) # IN_PROGRESS
chosen = find_next_transcribable_topic_asset(
topic=topic,
project_slug=project.slug,
item_id=item.item_id,
original_asset_id=str(asset1.id), # digit-string pk
)
self.assertEqual(chosen, asset2)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
def test_project_short_circuit_topic_excludes_current_item_via_item_filter(self):
"""
project-level short-circuit executes with item_id truthy,
so the code runs candidate = candidate.exclude(item__item_id=item_id).
Item-level has no NOT_STARTED, so we land in the project block.
"""
# Exhaust current item (no NOT_STARTED left there)
create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())
create_transcription(asset=self.asset2, user=self.anonymous, submitted=now())
# Create a NOT_STARTED candidate in the same project but a different item
other_item = create_item(
project=self.asset1.item.project, item_id="tt-proj-ex-branch"
)
pick = create_asset(item=other_item, sequence=1, slug="tt-proj-ex-pick")
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, pick)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_inprogress_fallback_spawns_task_with_item_id_topic(self, mock_get_task):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Same item: one IN_PROGRESS, one SUBMITTED -> no NOT_STARTED anywhere.
create_transcription(asset=self.asset2, user=self.anonymous)
create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())
# With item_id set, manual fallback excludes same-item candidates, so
# it returns nothing (spawn_task=True). Then the IN_PROGRESS fallback
# must return asset2 and trigger the task.
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=None,
)
self.assertEqual(chosen, self.asset2)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_manual_same_item_inprogress_triggers_spawn_task_topic(self, mock_get_task):
"""
When there are no NOT_STARTED candidates anywhere (after excluding current
item/original in the manual fallback), the same-item IN_PROGRESS fallback
should return an asset AND spawn the cache population task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Same item only:
# - original (NOT_STARTED) will be excluded by manual fallback,
# - next is IN_PROGRESS (eligible for final fallback).
create_transcription(asset=self.asset2, user=self.anonymous)
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug="",
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, self.asset2)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_project_short_circuit_excludes_current_item_topic(self, mock_get_task):
create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())
create_transcription(asset=self.asset2, user=self.anonymous, submitted=now())
other_item = create_item(
project=self.asset1.item.project, item_id="topic-proj-exclude-item"
)
pick = create_asset(item=other_item, sequence=5, slug="topic-proj-exclude-pick")
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=self.asset1.id,
)
self.assertEqual(chosen, pick)
self.assertFalse(mock_get_task.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_manual_inprogress_fallback_triggers_spawn_task_topic(self, mock_get_task):
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())
create_transcription(asset=self.asset2, user=self.anonymous)
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=None,
)
self.assertEqual(chosen, self.asset2)
self.assertTrue(mock_get_task.called)
self.assertTrue(mock_task.delay.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_inprogress_fallback_item_id_returns_none_when_no_candidates_topic(
self, mock_get_task
):
"""
With item_id truthy but no same-item IN_PROGRESS (and no NOT_STARTED anywhere),
the IN_PROGRESS fallback should yield no asset and the function returns None.
Ensures the path where `asset` is falsy in the IN_PROGRESS block is covered.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Fresh topic/project with a single item and no transcribable assets at all.
campaign = create_campaign(slug="tt-ip-none-c", title="tt-ip-none-c")
project = create_project(
campaign=campaign, slug="tt-ip-none-p", title="tt-ip-none-p"
)
topic = create_topic(project=project)
item = create_item(project=project, item_id="tt-ip-none-i")
a1 = create_asset(item=item, sequence=1, slug="tt-ip-none-a1")
a2 = create_asset(item=item, sequence=2, slug="tt-ip-none-a2")
# Make both SUBMITTED so neither NOT_STARTED nor IN_PROGRESS exists.
create_transcription(asset=a1, user=get_anonymous_user(), submitted=now())
create_transcription(asset=a2, user=get_anonymous_user(), submitted=now())
got = find_next_transcribable_topic_asset(
topic=topic,
project_slug=project.slug,
item_id=item.item_id, # truthy, so we enter the IN_PROGRESS fallback
original_asset_id=None,
)
self.assertIsNone(got)
# No task should be spawned since the IN_PROGRESS block found nothing.
self.assertFalse(mock_get_task.called)
@patch("concordia.utils.next_asset.transcribable.topic.get_registered_task")
def test_inprogress_fallback_returns_asset_without_spawning(self, mock_get_task):
"""
Force a path where the cache stage is exercised (so spawn_task stays False),
but yields no usable asset (simulated by making the cached asset retrieval
return None). The function should then fall through to the IN_PROGRESS
fallback, return that asset and NOT spawn the population task.
"""
mock_task = mock_get_task.return_value
mock_task.delay = MagicMock()
# Current item: seq1 SUBMITTED, seq2 IN_PROGRESS -> same-item NOT_STARTED fails.
create_transcription(asset=self.asset1, user=self.anonymous, submitted=now())
create_transcription(asset=self.asset2, user=self.anonymous) # IN_PROGRESS
# Prepare a cached candidate in a DIFFERENT project so project short-circuit
# doesn't grab it (only checks current project_slug).
other_campaign = create_campaign(slug="tt-ip-cache-c", title="tt-ip-cache-c")
other_project = create_project(
campaign=other_campaign, slug="tt-ip-cache-p", title="tt-ip-cache-p"
)
other_item = create_item(project=other_project, item_id="tt-ip-cache-i")
cached_asset = create_asset(item=other_item, sequence=1, slug="tt-ip-cache-a")
NextTranscribableTopicAsset.objects.create(
asset=cached_asset,
topic=self.topic,
item=cached_asset.item,
item_item_id=cached_asset.item.item_id,
project=cached_asset.item.project,
project_slug=cached_asset.item.project.slug,
sequence=cached_asset.sequence,
transcription_status=TranscriptionStatus.NOT_STARTED,
)
# Patch Asset.objects.filter only for id=cached_asset.id so the cache lookup
# produces an asset_id (spawn_task remains False) but then returns no row,
# forcing the function into the IN_PROGRESS fallback.
real_filter = Asset.objects.filter
def filter_side_effect(*args, **kwargs):
if kwargs == {"id": cached_asset.id}:
qs_mock = MagicMock()
qs_mock.select_for_update.return_value = qs_mock
qs_mock.select_related.return_value = qs_mock
qs_mock.first.return_value = None # simulate skip-locked/missing row
return qs_mock
return real_filter(*args, **kwargs)
with patch.object(Asset.objects, "filter", side_effect=filter_side_effect):
chosen = find_next_transcribable_topic_asset(
self.topic,
project_slug=self.asset1.item.project.slug,
item_id=self.asset1.item.item_id,
original_asset_id=None,
)
# Should fall back to same-item IN_PROGRESS and NOT spawn the task.
self.assertEqual(chosen, self.asset2)
self.assertFalse(mock_get_task.called)
================================================
FILE: concordia/tests/test_validators.py
================================================
import string
from django.core.exceptions import ValidationError
from django.test import TestCase
from concordia.passwords.validators import ComplexityValidator
from concordia.validators import DjangoPasswordsValidator
class TestValidators(TestCase):
def test_DjangoPasswordsValidator(self):
validator = DjangoPasswordsValidator()
expected_error = "Must be more complex (%s)"
self.assertIsNone(validator.validate("Ab1!"))
expected_suberror = "must contain 1 or more unique lowercase characters"
with self.assertRaises(ValidationError) as cm:
validator.validate("AB1!")
self.assertEqual(cm.exception.messages, [expected_error % expected_suberror])
self.assertEqual(cm.exception.error_list[0].code, "complexity")
expected_suberror = "must contain 1 or more unique uppercase characters"
with self.assertRaises(ValidationError) as cm:
validator.validate("ab1!")
self.assertEqual(cm.exception.messages, [expected_error % expected_suberror])
self.assertEqual(cm.exception.error_list[0].code, "complexity")
expected_suberror = "must contain 1 or more unique digits"
with self.assertRaises(ValidationError) as cm:
validator.validate("Ab!")
self.assertEqual(cm.exception.messages, [expected_error % expected_suberror])
self.assertEqual(cm.exception.error_list[0].code, "complexity")
expected_suberror = "must contain 1 or more non unique special characters"
with self.assertRaises(ValidationError) as cm:
validator.validate("Ab1")
self.assertEqual(cm.exception.messages, [expected_error % expected_suberror])
self.assertEqual(cm.exception.error_list[0].code, "complexity")
self.assertEqual(
validator.get_help_text(),
"Your password fails to meet our complexity requirements.",
)
class ComplexityValidatorTests(TestCase):
def assertValid(self, validator, string):
try:
validator(string)
except ValidationError:
self.fail(f"String {string} failed validation unexpectedly")
def assertInvalid(self, validator, string):
self.assertRaises(ValidationError, validator, string)
def make_validator(self, **complexities):
return ComplexityValidator(complexities=complexities)
def test_empty_validator(self):
validator = ComplexityValidator(complexities=None)
self.assertValid(validator, "")
def test_minimum_uppercase_count(self):
validator = self.make_validator(UPPER=0)
self.assertValid(validator, "no uppercase")
self.assertValid(validator, "Some UpperCase")
self.assertValid(validator, "ALL UPPERCASE")
validator = self.make_validator(UPPER=1)
self.assertInvalid(validator, "no uppercase")
self.assertValid(validator, "Some UpperCase")
self.assertValid(validator, "ALL UPPERCASE")
validator = self.make_validator(UPPER=100)
self.assertInvalid(validator, "no uppercase")
self.assertInvalid(validator, "Some UpperCase")
self.assertInvalid(validator, "ALL UPPERCASE")
def test_minimum_lowercase_count(self):
validator = self.make_validator(LOWER=0)
self.assertValid(validator, "NO LOWERCASE")
self.assertValid(validator, "sOME lOWERCASE")
self.assertValid(validator, "all lowercase")
validator = self.make_validator(LOWER=1)
self.assertInvalid(validator, "NO LOWERCASE")
self.assertValid(validator, "sOME lOWERCASE")
self.assertValid(validator, "all lowercase")
validator = self.make_validator(LOWER=100)
self.assertInvalid(validator, "NO LOWERCASE")
self.assertInvalid(validator, "sOME lOWERCASE")
self.assertInvalid(validator, "all lowercase")
def test_minimum_letter_count(self):
validator = self.make_validator(LETTERS=0)
self.assertValid(validator, "1234. ?")
self.assertValid(validator, "soME 123")
self.assertValid(validator, "allletters")
validator = self.make_validator(LETTERS=1)
self.assertInvalid(validator, "1234. ?")
self.assertValid(validator, "soME 123")
self.assertValid(validator, "allletters")
validator = self.make_validator(LETTERS=100)
self.assertInvalid(validator, "1234. ?")
self.assertInvalid(validator, "soME 123")
self.assertInvalid(validator, "allletters")
def test_minimum_digit_count(self):
validator = self.make_validator(DIGITS=0)
self.assertValid(validator, "")
self.assertValid(validator, "0")
self.assertValid(validator, "1")
self.assertValid(validator, "11")
self.assertValid(validator, "one 1")
validator = self.make_validator(DIGITS=1)
self.assertInvalid(validator, "")
self.assertValid(validator, "0")
self.assertValid(validator, "1")
self.assertValid(validator, "11")
self.assertValid(validator, "one 1")
def test_minimum_punctuation_count(self):
none = "no punctuation"
one = "ffs!"
mixed = r"w@oo%lo(om!ol~oo&"
allpunc = string.punctuation
validator = self.make_validator(SPECIAL=0)
self.assertValid(validator, none)
self.assertValid(validator, one)
self.assertValid(validator, mixed)
self.assertValid(validator, allpunc)
validator = self.make_validator(SPECIAL=1)
self.assertInvalid(validator, none)
self.assertValid(validator, one)
self.assertValid(validator, mixed)
self.assertValid(validator, allpunc)
validator = self.make_validator(SPECIAL=100)
self.assertInvalid(validator, none)
self.assertInvalid(validator, one)
self.assertInvalid(validator, mixed)
self.assertInvalid(validator, allpunc)
def test_minimum_nonascii_count(self):
none = "regularchars and numbers 100"
one = "\x00" # null
many = "\x00\x01\x02\x03\x04\x05\t\n\r"
validator = self.make_validator(SPECIAL=0)
self.assertValid(validator, none)
self.assertValid(validator, one)
self.assertValid(validator, many)
validator = self.make_validator(SPECIAL=1)
self.assertInvalid(validator, none)
self.assertValid(validator, one)
self.assertValid(validator, many)
validator = self.make_validator(SPECIAL=100)
self.assertInvalid(validator, none)
self.assertInvalid(validator, one)
self.assertInvalid(validator, many)
def test_minimum_words_count(self):
none = ""
one = "oneword"
some = "one or two words"
many = "a b c d e f g h i 1 2 3 4 5 6 7 8 9 { $ # ! )}"
validator = self.make_validator(WORDS=0)
self.assertValid(validator, none)
self.assertValid(validator, one)
self.assertValid(validator, some)
self.assertValid(validator, many)
validator = self.make_validator(WORDS=1)
self.assertInvalid(validator, none)
self.assertValid(validator, one)
self.assertValid(validator, some)
self.assertValid(validator, many)
validator = self.make_validator(WORDS=10)
self.assertInvalid(validator, none)
self.assertInvalid(validator, one)
self.assertInvalid(validator, some)
self.assertValid(validator, many)
validator = self.make_validator(WORDS=100)
self.assertInvalid(validator, none)
self.assertInvalid(validator, one)
self.assertInvalid(validator, some)
self.assertInvalid(validator, many)
================================================
FILE: concordia/tests/test_view_decorators.py
================================================
from unittest.mock import MagicMock, patch
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.http import HttpRequest
from django.test import TestCase
from concordia.views.decorators import next_asset_rate
class TestNextAssetRate(TestCase):
def setUp(self):
self.request = HttpRequest()
def test_authenticated_user_returns_none(self):
self.request.user = MagicMock(is_authenticated=True)
result = next_asset_rate("any.group", self.request)
self.assertIsNone(result)
@patch("concordia.views.decorators.configuration_value")
@patch("concordia.views.decorators.validate_rate")
def test_anonymous_user_valid_rate(self, mock_validate_rate, mock_config_value):
self.request.user = MagicMock(is_authenticated=False)
mock_config_value.return_value = "10/m"
mock_validate_rate.return_value = "10/m"
result = next_asset_rate("next_asset", self.request)
self.assertEqual(result, "10/m")
@patch("concordia.views.decorators.configuration_value")
@patch("concordia.views.decorators.validate_rate")
def test_anonymous_user_invalid_rate_falls_back(
self, mock_validate_rate, mock_config_value
):
self.request.user = MagicMock(is_authenticated=False)
mock_config_value.return_value = "invalid"
mock_validate_rate.side_effect = ValidationError("bad")
result = next_asset_rate("next_asset", self.request)
self.assertEqual(result, "4/m")
@patch("concordia.views.decorators.configuration_value")
def test_anonymous_user_missing_value_falls_back(self, mock_config_value):
self.request.user = MagicMock(is_authenticated=False)
mock_config_value.side_effect = ObjectDoesNotExist()
result = next_asset_rate("next_asset", self.request)
self.assertEqual(result, "4/m")
================================================
FILE: concordia/tests/test_views.py
================================================
import json
from datetime import date, timedelta
from unittest.mock import patch
from django import forms
from django.contrib.auth.models import AnonymousUser
from django.core.cache import caches
from django.db.models.signals import post_save
from django.http import HttpResponse, JsonResponse
from django.test import (
Client,
RequestFactory,
TestCase,
override_settings,
)
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.utils.timezone import now
from concordia.models import (
Asset,
Campaign,
Transcription,
)
from concordia.signals.handlers import on_transcription_save
from concordia.tasks.reports.sitereport import campaign_report
from concordia.utils import get_anonymous_user
from concordia.views.accounts import AccountProfileView, registration_rate
from concordia.views.campaigns import CompletedCampaignListView
from concordia.views.decorators import reserve_rate, user_cache_control
from concordia.views.items import FilteredItemDetailView
from concordia.views.projects import FilteredProjectDetailView
from concordia.views.rate_limit import ratelimit_view
from concordia.views.visualizations import VisualizationDataView
from .utils import (
CreateTestUsers,
JSONAssertMixin,
create_asset,
create_campaign,
create_card_family,
create_guide,
create_item,
create_project,
create_research_center,
create_tag_collection,
create_topic,
create_transcription,
)
def setup_view(view, request, user=None, *args, **kwargs):
"""
https://stackoverflow.com/a/33647251/10320488
"""
if user:
request.user = user
view.request = request
view.args = args
view.kwargs = kwargs
return view
class AccountProfileViewTests(CreateTestUsers, TestCase):
"""
This class contains the unit tests for the AccountProfileView.
"""
def test_get_queryset(self):
"""
Test the get_queryset method
"""
self.login_user()
v = setup_view(
AccountProfileView(),
RequestFactory().get("account/password_reset/"),
user=self.user,
)
qs = v.get_queryset()
self.assertEqual(qs.count(), 0)
class CompletedCampaignListViewTests(TestCase):
"""
This class contains the unit tests for the CompletedCampaignListView
"""
def setUp(self):
today = date.today()
yesterday = today - timedelta(days=1)
self.research_center = create_research_center()
self.campaign2 = create_campaign(
published=True,
status=Campaign.Status.COMPLETED,
slug="test-campaign-2",
completed_date=yesterday,
)
self.campaign2.research_centers.add(self.research_center)
self.campaign3 = create_campaign(
published=True,
status=Campaign.Status.RETIRED,
slug="test-campaign-3",
completed_date=yesterday,
)
def test_get_all_campaigns(self):
active = create_campaign(
published=True,
slug="test-campaign-4",
completed_date=self.campaign2.completed_date,
)
view = CompletedCampaignListView()
view.request = RequestFactory().get("/campaigns/completed/")
completed_and_retired = view._get_all_campaigns()
self.assertNotIn(active, completed_and_retired)
self.assertIn(self.campaign2, completed_and_retired)
self.assertIn(self.campaign3, completed_and_retired)
view.request = RequestFactory().get("/campaigns/completed/?type=completed")
completed_campaigns = view._get_all_campaigns()
self.assertNotIn(active, completed_campaigns)
self.assertIn(self.campaign2, completed_campaigns)
self.assertNotIn(self.campaign3, completed_campaigns)
view.request = RequestFactory().get("/campaigns/completed/?type=retired")
retired_campaigns = view._get_all_campaigns()
self.assertNotIn(active, retired_campaigns)
self.assertNotIn(self.campaign2, retired_campaigns)
self.assertIn(self.campaign3, retired_campaigns)
def test_queryset(self):
today = date.today()
create_campaign(
published=True, status=Campaign.Status.COMPLETED, completed_date=today
)
view = CompletedCampaignListView()
# Test default
view.request = RequestFactory().get("/campaigns/completed/")
queryset = view.get_queryset()
self.assertGreater(
queryset.first().completed_date, queryset.last().completed_date
)
# Test retired
view.request = RequestFactory().get("/campaigns/completed/?type=retired")
queryset = view.get_queryset()
self.assertEqual(queryset.count(), 1)
def test_context_data(self):
request = RequestFactory().get("/campaigns/completed/")
response = CompletedCampaignListView.as_view()(request)
self.assertIsInstance(response.context_data, dict)
self.assertEqual(response.context_data["result_count"], 2)
request = RequestFactory().get("/campaigns/completed/?type=completed")
response = CompletedCampaignListView.as_view()(request)
self.assertIsInstance(response.context_data, dict)
self.assertEqual(response.context_data["result_count"], 1)
request = RequestFactory().get("/campaigns/completed/?type=completed")
response = CompletedCampaignListView.as_view()(request)
self.assertIsInstance(response.context_data, dict)
self.assertEqual(response.context_data["result_count"], 1)
request = RequestFactory().get(
f"/campaigns/completed/?research_center={self.research_center.id}"
)
response = CompletedCampaignListView.as_view()(request)
self.assertIsInstance(response.context_data, dict)
self.assertEqual(response.context_data["result_count"], 1)
def test_research_centers(self):
today = date.today()
create_campaign(
published=True, status=Campaign.Status.COMPLETED, completed_date=today
)
url = f"/campaigns/completed/?research_center={self.research_center.id}"
# Test queryset directly
view = CompletedCampaignListView()
view.request = RequestFactory().get(url)
queryset = view.get_queryset()
self.assertEqual(queryset.count(), 1)
# Test get_context_data through a get
response = self.client.get(url)
self.assertIn("research_centers", response.context)
self.assertEqual(response.context["research_centers"][0], self.research_center)
@override_settings(
RATELIMIT_ENABLE=False, SESSION_ENGINE="django.contrib.sessions.backends.cache"
)
class ConcordiaViewTests(CreateTestUsers, JSONAssertMixin, TestCase):
"""
This class contains the unit tests for the view in the concordia app.
"""
def setUp(self):
for cache in caches.all():
cache.clear()
def tearDown(self):
for cache in caches.all():
cache.clear()
def test_ratelimit_view(self):
c = Client()
response = c.get("/error/429/")
self.assertIsInstance(response, HttpResponse)
self.assertEqual(response.status_code, 429)
headers = {"HTTP_X_REQUESTED_WITH": "XMLHttpRequest"}
response = c.get("/error/429/", **headers)
self.assertIsInstance(response, JsonResponse)
self.assertEqual(response.status_code, 429)
def test_campaign_topic_list_view(self):
"""
Test the GET method for route /campaigns-topics
"""
campaign = create_campaign(title="Hello Everyone")
topic_project = create_project(campaign=campaign)
campaign_item = create_item(project=topic_project)
create_asset(item=campaign_item)
unlisted_campaign = create_campaign(
title="Hello to only certain people", unlisted=True
)
unlisted_topic_project = create_project(campaign=unlisted_campaign)
unlisted_campaign_item = create_item(project=unlisted_topic_project)
create_asset(item=unlisted_campaign_item)
topic = create_topic(title="A Listed Topic", project=topic_project)
unlisted_topic = create_topic(
title="An Unlisted Topic", unlisted=True, project=unlisted_topic_project
)
response = self.client.get(reverse("campaign-topic-list"))
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/campaign_topic_list.html"
)
self.assertContains(response, topic.title)
self.assertNotContains(response, unlisted_topic.title)
self.assertContains(response, campaign.title)
self.assertNotContains(response, unlisted_campaign.title)
def test_campaign_list_view(self):
"""
Test the GET method for route /campaigns
"""
campaign = create_campaign(title="Hello Everyone 2")
unlisted_campaign = create_campaign(
title="Hello to only certain people 2", unlisted=True
)
response = self.client.get(reverse("transcriptions:campaign-list"))
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/campaign_list.html"
)
self.assertContains(response, campaign.title)
self.assertNotContains(response, unlisted_campaign.title)
def test_campaign_detail_view(self):
"""
Test GET on route /campaigns/ (campaign)
"""
campaign = create_campaign(title="GET Campaign", slug="get-campaign")
response = self.client.get(
reverse("transcriptions:campaign-detail", args=(campaign.slug,))
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/campaign_detail.html"
)
self.assertContains(response, campaign.title)
# Filter by reviewable parameter check
response = self.client.get(
reverse("transcriptions:campaign-detail", args=(campaign.slug,)),
{"filter_by_reviewable": True},
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/campaign_detail.html"
)
self.assertContains(response, campaign.title)
# Bad status parameter check
response = self.client.get(
reverse("transcriptions:campaign-detail", args=(campaign.slug,)),
{"transcription_status": "bad_parameter"},
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/campaign_detail.html"
)
self.assertContains(response, campaign.title)
# Unlisted
campaign = create_campaign(
title="GET Unlisted Campaign", unlisted=True, slug="get-unlisted-campaign"
)
response = self.client.get(
reverse("transcriptions:campaign-detail", args=(campaign.slug,))
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/campaign_detail.html"
)
self.assertContains(response, campaign.title)
# Completed
campaign = create_campaign(
title="GET Completed Campaign",
slug="get-completed-campaign",
status=Campaign.Status.COMPLETED,
)
response = self.client.get(
reverse("transcriptions:campaign-detail", args=(campaign.slug,))
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/campaign_detail_completed.html"
)
self.assertContains(response, campaign.title)
# Retired
campaign = create_campaign(
title="GET Retired Campaign",
slug="get-retired-campaign",
status=Campaign.Status.RETIRED,
)
# We need a site report for a retired campaign because
# that's where the view pulls data from
campaign_report(campaign=campaign)
response = self.client.get(
reverse("transcriptions:campaign-detail", args=(campaign.slug,))
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/campaign_detail_retired.html"
)
self.assertContains(response, campaign.title)
def test_campaign_unicode_slug(self):
"""Confirm that Unicode characters are usable in Campaign URLs"""
campaign = create_campaign(title="你好 World")
self.assertEqual(campaign.slug, "你好-world")
response = self.client.get(campaign.get_absolute_url())
self.assertEqual(response.status_code, 200)
def test_concordiaCampaignView_get_page2(self):
"""
Test GET on route /campaigns// (campaign) on page 2
"""
c = create_campaign()
response = self.client.get(
reverse("transcriptions:campaign-detail", args=(c.slug,)), {"page": 2}
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/campaign_detail.html"
)
def test_empty_item_detail_view(self):
"""
Test item detail display with no assets
"""
item = create_item()
response = self.client.get(
reverse(
"transcriptions:item-detail",
args=(item.project.campaign.slug, item.project.slug, item.item_id),
)
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/item_detail.html"
)
self.assertContains(response, item.title)
self.assertEqual(0, response.context["not_started_percent"])
self.assertEqual(0, response.context["in_progress_percent"])
self.assertEqual(0, response.context["submitted_percent"])
self.assertEqual(0, response.context["completed_percent"])
def test_item_detail_view(self):
"""
Test item detail display with assets
"""
self.login_user() # Implicitly create the test account
anon = get_anonymous_user()
item = create_item()
# We'll create 10 assets and transcriptions for some of them so we can
# confirm that the math is working correctly:
for i in range(1, 11):
asset = create_asset(item=item, sequence=i, slug=f"test-{i}")
if i > 9:
t = asset.transcription_set.create(asset=asset, user=anon)
t.submitted = now()
t.accepted = now()
t.reviewed_by = self.user
elif i > 7:
t = asset.transcription_set.create(asset=asset, user=anon)
t.submitted = now()
elif i > 4:
t = asset.transcription_set.create(asset=asset, user=anon)
else:
continue
t.full_clean()
t.save()
response = self.client.get(
reverse(
"transcriptions:item-detail",
args=(item.project.campaign.slug, item.project.slug, item.item_id),
)
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/item_detail.html"
)
self.assertContains(response, item.title)
# We have 10 total, 6 of which have transcription records and of those
# 6, 3 have been submitted and one of those was accepted:
self.assertEqual(40, response.context["not_started_percent"])
self.assertEqual(30, response.context["in_progress_percent"])
self.assertEqual(20, response.context["submitted_percent"])
self.assertEqual(10, response.context["completed_percent"])
# Filter by reviewable parameter check
response = self.client.get(
reverse(
"transcriptions:item-detail",
args=(item.project.campaign.slug, item.project.slug, item.item_id),
),
{"filter_by_reviewable": True},
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/item_detail.html"
)
# Bad status parameter check
response = self.client.get(
reverse(
"transcriptions:item-detail",
args=(item.project.campaign.slug, item.project.slug, item.item_id),
),
{"transcription_status": "bad_parameter"},
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/item_detail.html"
)
# Non-existent item in an existing campaign
response = self.client.get(
reverse(
"transcriptions:item-detail",
args=(item.project.campaign.slug, item.project.slug, "bad-id"),
)
)
self.assertRedirects(
response,
reverse(
"transcriptions:campaign-detail", args=(item.project.campaign.slug,)
),
)
def test_asset_unicode_slug(self):
"""Confirm that Unicode characters are usable in Asset URLs"""
asset = create_asset(title="你好 World")
self.assertEqual(asset.slug, "你好-world")
response = self.client.get(asset.get_absolute_url())
self.assertEqual(response.status_code, 200)
def test_asset_detail_view(self):
"""
This unit test test the GET route /campaigns//asset//
with already in use.
"""
self.login_user()
asset = create_asset(sequence=100)
self.transcription = asset.transcription_set.create(
user_id=self.user.id, text="Test transcription 1"
)
self.transcription.save()
asset.item.project.campaign.card_family = create_card_family()
asset.item.project.campaign.save()
title = "Transcription: Basic Rules"
create_guide(title=title)
tag_collection = create_tag_collection(asset=asset)
response = self.client.get(
reverse(
"transcriptions:asset-detail",
kwargs={
"campaign_slug": asset.item.project.campaign.slug,
"project_slug": asset.item.project.slug,
"item_id": asset.item.item_id,
"slug": asset.slug,
},
)
)
self.assertEqual(response.status_code, 200)
self.assertIn("cards", response.context)
self.assertIn("guides", response.context)
self.assertEqual(title, response.context["guides"][0]["title"])
self.assertIn("tags", response.context)
self.assertEqual([tag_collection.tags.all()[0].value], response.context["tags"])
# Next and previous asset checks
previous_asset = create_asset(
item=asset.item, slug="previous-asset", sequence=1
)
next_asset = create_asset(item=asset.item, slug="next-asset", sequence=1000)
response = self.client.get(
reverse(
"transcriptions:asset-detail",
kwargs={
"campaign_slug": asset.item.project.campaign.slug,
"project_slug": asset.item.project.slug,
"item_id": asset.item.item_id,
"slug": asset.slug,
},
)
)
self.assertEqual(response.status_code, 200)
self.assertIn("previous_asset_url", response.context)
self.assertEqual(
previous_asset.get_absolute_url(), response.context["previous_asset_url"]
)
self.assertIn("next_asset_url", response.context)
self.assertEqual(
next_asset.get_absolute_url(), response.context["next_asset_url"]
)
# Download URL iiif check
asset.download_url = "http://tile.loc.gov/image-services/iiif/service:music:mussuffrage:mussuffrage-100183:mussuffrage-100183.0001/full/pct:100/0/default.jpg"
asset.save()
response = self.client.get(
reverse(
"transcriptions:asset-detail",
kwargs={
"campaign_slug": asset.item.project.campaign.slug,
"project_slug": asset.item.project.slug,
"item_id": asset.item.item_id,
"slug": asset.slug,
},
)
)
self.assertEqual(response.status_code, 200)
self.assertIn("thumbnail_url", response.context)
self.assertEqual(
"https://tile.loc.gov/image-services/iiif/service:music:mussuffrage:mussuffrage-100183:mussuffrage-100183.0001/full/!512,512/0/default.jpg",
response.context["thumbnail_url"],
)
# Non-existent asset in an existing campaign
response = self.client.get(
reverse(
"transcriptions:asset-detail",
kwargs={
"campaign_slug": asset.item.project.campaign.slug,
"project_slug": asset.item.project.slug,
"item_id": asset.item.item_id,
"slug": "bad-slug",
},
)
)
self.assertRedirects(
response,
reverse(
"transcriptions:campaign-detail",
args=(asset.item.project.campaign.slug,),
),
)
@patch.object(Asset, "get_ocr_transcript")
def test_generate_ocr_transcription(self, mock):
asset1 = create_asset(storage_image="tests/test-european.jpg")
url = reverse("generate-ocr-transcription", kwargs={"asset_pk": asset1.pk})
# Anonymous user test; should redirect
response = self.client.post(url)
self.assertEqual(302, response.status_code)
self.assertFalse(mock.called)
mock.reset_mock()
self.login_user()
response = self.client.post(url)
self.assertEqual(201, response.status_code)
self.assertTrue(mock.called)
mock.reset_mock()
asset2 = create_asset(
item=asset1.item,
slug="test-asset-2",
storage_image="tests/test-european.jpg",
)
url = reverse("generate-ocr-transcription", kwargs={"asset_pk": asset2.pk})
response = self.client.post(url, data={"language": "spa"})
self.assertEqual(201, response.status_code)
mock.assert_called_with("spa")
mock.reset_mock()
with patch(
"concordia.views.ajax.get_transcription_superseded"
) as superseded_mock:
# Test case if the trancription being replaced has already been superseded
superseded_mock.return_value = HttpResponse(status=409)
url = reverse("generate-ocr-transcription", kwargs={"asset_pk": asset2.pk})
response = self.client.post(url)
self.assertEqual(409, response.status_code)
self.assertTrue(superseded_mock.called)
self.assertFalse(mock.called)
# Test case if the transcription being replaced hasn't been superseded
superseded_mock.reset_mock()
superseded_mock.return_value = create_transcription(
asset=asset2, user=get_anonymous_user(), submitted=now()
)
url = reverse("generate-ocr-transcription", kwargs={"asset_pk": asset2.pk})
response = self.client.post(url)
self.assertEqual(201, response.status_code)
self.assertTrue(superseded_mock.called)
self.assertTrue(mock.called)
def test_project_detail_view(self):
"""
Test GET on route /campaigns/ (campaign)
"""
project = create_project()
response = self.client.get(
reverse(
"transcriptions:project-detail",
args=(project.campaign.slug, project.slug),
)
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/project_detail.html"
)
# Filter by reviewable parameter check
response = self.client.get(
reverse(
"transcriptions:project-detail",
args=(project.campaign.slug, project.slug),
),
{"filter_by_reviewable": True},
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/project_detail.html"
)
# Bad status parameter check
response = self.client.get(
reverse(
"transcriptions:project-detail",
args=(project.campaign.slug, project.slug),
),
{"transcription_status": "bad_parameter"},
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/project_detail.html"
)
# Non-existent project in an existing campaign
response = self.client.get(
reverse(
"transcriptions:project-detail",
args=(project.campaign.slug, "bad-slug"),
)
)
self.assertRedirects(
response,
reverse("transcriptions:campaign-detail", args=(project.campaign.slug,)),
)
def test_project_unicode_slug(self):
"""Confirm that Unicode characters are usable in Project URLs"""
project = create_project(title="你好 World")
self.assertEqual(project.slug, "你好-world")
response = self.client.get(project.get_absolute_url())
self.assertEqual(response.status_code, 200)
def test_campaign_report(self):
"""
Test campaign reporting
"""
item = create_item()
# We'll create 10 assets and transcriptions for some of them so we can
# confirm that the math is working correctly:
for i in range(1, 11):
create_asset(item=item, sequence=i, slug=f"test-{i}")
response = self.client.get(
reverse(
"transcriptions:campaign-report",
kwargs={"campaign_slug": item.project.campaign.slug},
)
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(response, "transcriptions/campaign_report.html")
ctx = response.context
self.assertEqual(ctx["title"], item.project.campaign.title)
self.assertEqual(ctx["total_asset_count"], 10)
response = self.client.get(
reverse(
"transcriptions:campaign-report",
kwargs={"campaign_slug": item.project.campaign.slug},
),
{"page": "not-an-int"},
)
ctx = response.context
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(response, "transcriptions/campaign_report.html")
self.assertEqual(ctx["projects"].number, 1)
response = self.client.get(
reverse(
"transcriptions:campaign-report",
kwargs={"campaign_slug": item.project.campaign.slug},
),
{"page": 10000},
)
ctx = response.context
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(response, "transcriptions/campaign_report.html")
self.assertEqual(ctx["projects"].number, 1)
class UserCacheControlTest(CreateTestUsers, TestCase):
"""
Tests for the user_cache_control decorator
"""
def setUp(self):
self.factory = RequestFactory()
self.user = self.create_user("testuser")
def test_vary_on_cookie(self):
@method_decorator(user_cache_control, name="dispatch")
def a_view(request):
return HttpResponse()
request = self.factory.get("/rand")
request.user = self.user
resp = a_view(None, request)
self.assertEqual(resp.status_code, 200)
class FilteredCampaignDetailViewTests(CreateTestUsers, TestCase):
def test_get_context_data(self):
campaign = create_campaign()
kwargs = {"slug": campaign.slug}
url = reverse("transcriptions:filtered-campaign-detail", kwargs=kwargs)
self.login_user(is_staff=False)
response = self.client.get(url, kwargs)
self.assertFalse(response.context.get("filter_by_reviewable", False))
self.logout_user()
self.user = self.create_staff_user()
self.login_user()
response = self.client.get(url, kwargs)
self.assertTrue(response.context.get("filter_by_reviewable"))
class FilteredProjectDetailViewTests(CreateTestUsers, TestCase):
def setUp(self):
self.project = create_project()
self.kwargs = {
"campaign_slug": self.project.campaign.slug,
"slug": self.project.slug,
}
self.url = reverse("transcriptions:filtered-project-detail", kwargs=self.kwargs)
self.login_user()
def test_get_queryset(self):
item1 = create_item(project=self.project, item_id="testitem.012345679")
asset1 = create_asset(item=item1)
create_transcription(asset=asset1, user=get_anonymous_user(), submitted=now())
item2 = create_item(
project=create_project(slug="project-two", campaign=self.project.campaign)
)
asset2 = create_asset(item=item2)
create_transcription(asset=asset2, user=self.user, submitted=now())
view = FilteredProjectDetailView()
view.kwargs = self.kwargs
view.request = RequestFactory().get(self.url, self.kwargs)
view.request.user = self.user
qs = view.get_queryset()
self.assertIn(item1, qs)
self.assertNotIn(item2, qs)
def test_get_context_data(self):
response = self.client.get(self.url, self.kwargs)
self.assertTrue(response.context.get("filter_by_reviewable"))
def tearDown(self):
post_save.connect(on_transcription_save, sender=Transcription)
class FilteredItemDetailViewTests(CreateTestUsers, TestCase):
def setUp(self):
self.item = create_item()
self.kwargs = {
"campaign_slug": self.item.project.campaign.slug,
"project_slug": self.item.project.slug,
"item_id": self.item.item_id,
}
self.url = reverse("transcriptions:filtered-item-detail", kwargs=self.kwargs)
self.login_user()
def test_get_queryset(self):
asset1 = create_asset(item=self.item)
create_transcription(asset=asset1, user=get_anonymous_user(), submitted=now())
asset2 = create_asset(item=self.item, slug="asset-two")
create_transcription(asset=asset2, user=self.user, submitted=now())
view = FilteredItemDetailView()
view.kwargs = self.kwargs
view.request = RequestFactory().get(self.url, self.kwargs)
view.request.user = self.user
qs = view.get_queryset()
self.assertIn(asset1, qs)
self.assertNotIn(asset2, qs)
def test_get_context_data(self):
response = self.client.get(self.url, self.kwargs)
self.assertTrue(response.context.get("filter_by_reviewable"))
def tearDown(self):
post_save.connect(on_transcription_save, sender=Transcription)
class RateLimitTests(CreateTestUsers, TestCase):
def setUp(self):
self.request_factory = RequestFactory()
self.user = self.create_user("test-user")
def test_registration_rate(self):
request = self.request_factory.get("/")
self.assertEqual(registration_rate(None, request), "10/h")
with patch("concordia.views.accounts.UserRegistrationForm", autospec=True):
# This causes the form to test as valid even though there's no data
self.assertIsNone(registration_rate(None, request))
def test_ratelimit_view(self):
request = self.request_factory.post("/")
exception = Exception()
response = ratelimit_view(request, exception)
self.assertEqual(response.status_code, 429)
self.assertNotEqual(response["Retry-After"], 0)
def test_reserve_rate(self):
request = self.request_factory.post("/")
request.user = AnonymousUser()
self.assertEqual("100/m", reserve_rate("test.group", request))
request.user = self.user
self.assertEqual(None, reserve_rate("test.group", request))
class LoginTests(TestCase, CreateTestUsers):
def setUp(self):
self.user = self.create_user("test-user")
def test_ConcordiaLoginView(self):
with patch("concordia.turnstile.fields.TurnstileField.validate") as mock:
mock.side_effect = forms.ValidationError(
"Testing error", code="invalid_turnstile"
)
response = self.client.post(
reverse("registration_login"),
data={"username": self.user.username, "password": self.user._password},
)
self.assertIn("user", response.context)
self.assertFalse(response.context["user"].is_authenticated)
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
response = self.client.post(
reverse("registration_login"),
data={"username": self.user.username, "password": self.user._password},
follow=True,
)
self.assertRedirects(
response,
expected_url=reverse("homepage"),
target_status_code=200,
)
self.assertIn("user", response.context)
self.assertTrue(response.context["user"].is_authenticated)
class TranscriptionViewTests(CreateTestUsers, TestCase):
def setUp(self):
self.asset = create_asset()
def test_rollback_transcription(self):
path = reverse("rollback-transcription", args=[self.asset.id])
self.login_user()
# Test rollback when there are no transcriptions
response = self.client.post(path)
self.assertEqual(400, response.status_code)
self.assertIn("error", response.json())
transcription1 = create_transcription(
asset=self.asset, text="Test transcription 1"
)
user = transcription1.user
# Test rollback when there are no transcriptions to rollback to
response = self.client.post(path)
self.assertEqual(400, response.status_code)
self.assertIn("error", response.json())
# Test successful rollback
create_transcription(asset=self.asset, user=user, text="Test transcription 2")
response = self.client.post(path)
self.assertEqual(201, response.status_code)
response_json = response.json()
self.assertIn("id", response_json)
self.assertIn("text", response_json)
self.assertEqual(response_json["text"], transcription1.text)
self.assertIn("undo_available", response_json)
self.assertEqual(response_json["undo_available"], False)
self.assertIn("redo_available", response_json)
self.assertEqual(response_json["redo_available"], True)
# Test after a rollforward
self.asset.rollforward_transcription(user)
response = self.client.post(path)
self.assertEqual(201, response.status_code)
response_json = response.json()
self.assertIn("id", response_json)
self.assertIn("text", response_json)
self.assertEqual(response_json["text"], transcription1.text)
self.assertIn("undo_available", response_json)
self.assertEqual(response_json["undo_available"], False)
self.assertIn("redo_available", response_json)
self.assertEqual(response_json["redo_available"], True)
# Test anonymous user
self.client.logout()
create_transcription(asset=self.asset, user=user, text="Test transcription 3")
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
response = self.client.post(path)
self.assertEqual(201, response.status_code)
response_json = response.json()
self.assertIn("id", response_json)
self.assertIn("text", response_json)
self.assertEqual(response_json["text"], transcription1.text)
self.assertIn("undo_available", response_json)
self.assertEqual(response_json["undo_available"], False)
self.assertIn("redo_available", response_json)
self.assertEqual(response_json["redo_available"], True)
def test_rollforward_transcription(self):
path = reverse("rollforward-transcription", args=[self.asset.id])
self.login_user()
# Test rollforward when there are no transcriptions
response = self.client.post(path)
self.assertEqual(400, response.status_code)
self.assertIn("error", response.json())
transcription1 = create_transcription(
asset=self.asset, text="Test transcription 1"
)
user = transcription1.user
# Test rollback when there are no transcriptions to rollforward to
response = self.client.post(path)
self.assertEqual(400, response.status_code)
self.assertIn("error", response.json())
# Test successful rollforward, which requires a rollback first
transcription2 = create_transcription(
asset=self.asset, user=user, text="Test transcription 2"
)
self.asset.rollback_transcription(user)
response = self.client.post(path)
self.assertEqual(201, response.status_code)
response_json = response.json()
self.assertIn("id", response_json)
self.assertIn("text", response_json)
self.assertEqual(response_json["text"], transcription2.text)
self.assertIn("undo_available", response_json)
self.assertEqual(response_json["undo_available"], True)
self.assertIn("redo_available", response_json)
self.assertEqual(response_json["redo_available"], False)
# Test aftering rolling back then creating a new transcription
self.asset.rollback_transcription(user)
create_transcription(asset=self.asset, user=user, text="Test transcription 3")
response = self.client.post(path)
self.assertEqual(400, response.status_code)
self.assertIn("error", response.json())
# Test anonymous user after a rollback
self.client.logout()
transcription3 = create_transcription(
asset=self.asset, user=user, text="Test transcription 3"
)
self.asset.rollback_transcription(user)
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
response = self.client.post(path)
response_json = response.json()
self.assertEqual(201, response.status_code)
self.assertIn("id", response_json)
self.assertIn("text", response_json)
self.assertEqual(response_json["text"], transcription3.text)
self.assertIn("undo_available", response_json)
self.assertEqual(response_json["undo_available"], True)
self.assertIn("redo_available", response_json)
self.assertEqual(response_json["redo_available"], False)
def tearDown(self):
post_save.connect(on_transcription_save, sender=Transcription)
@override_settings(
CACHES={
"visualization_cache": {
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
}
}
)
class VisualizationDataViewTests(TestCase):
def setUp(self):
self.factory = RequestFactory()
self.cache = caches["visualization_cache"]
VisualizationDataView.cache = self.cache
self.cache.clear()
self.view = VisualizationDataView.as_view()
def test_get_missing_data_returns_404(self):
# If no entry exists in the cache under the given name,
# the view should return a 404 with a JSON error message.
request = self.factory.get("/visualizations/data/missing-key/")
response = self.view(request, name="missing-key")
self.assertEqual(response.status_code, 404)
self.assertEqual(response["Content-Type"], "application/json")
data = json.loads(response.content)
self.assertEqual(
data, {"error": "No visualization data found for 'missing-key'"}
)
def test_get_existing_data_returns_200_and_json(self):
# When the cache contains data for the given name,
# the view should return it as JSON with status 200.
sample_data = {"foo": "bar", "numbers": [1, 2, 3]}
self.cache.set("sample-key", sample_data)
request = self.factory.get("/visualizations/data/sample-key/")
response = self.view(request, name="sample-key")
self.assertEqual(response.status_code, 200)
self.assertEqual(response["Content-Type"], "application/json")
data = json.loads(response.content)
self.assertEqual(data, sample_data)
================================================
FILE: concordia/tests/test_views_asset_reservation.py
================================================
from datetime import timedelta
from django.conf import settings
from django.contrib.auth.models import User
from django.db.models.signals import post_save
from django.test import (
RequestFactory,
TransactionTestCase,
override_settings,
)
from django.urls import reverse
from django.utils.timezone import now
from concordia.models import (
AssetTranscriptionReservation,
Transcription,
)
from concordia.signals.handlers import on_transcription_save
from concordia.tasks.reservations import (
delete_old_tombstoned_reservations,
expire_inactive_asset_reservations,
tombstone_old_active_asset_reservations,
)
from concordia.utils import get_anonymous_user, get_or_create_reservation_token
from .utils import (
CreateTestUsers,
JSONAssertMixin,
create_asset,
)
@override_settings(
RATELIMIT_ENABLE=False, SESSION_ENGINE="django.contrib.sessions.backends.cache"
)
class AssetReservationViewTests(CreateTestUsers, JSONAssertMixin, TransactionTestCase):
def test_asset_reservation(self):
"""
Test the basic Asset reservation process
"""
self.login_user()
self._asset_reservation_test_payload(self.user.pk)
def test_asset_reservation_anonymously(self):
"""
Test the basic Asset reservation process as an anonymous user
"""
anon_user = get_anonymous_user()
self._asset_reservation_test_payload(anon_user.pk, anonymous=True)
def _asset_reservation_test_payload(self, user_id, anonymous=False):
asset = create_asset()
# Acquire the reservation: 1 acquire
# + 1 reservation check
# + 1 logging if not anonymous
# + 1 session if not anonymous and using a database session engine:
expected_update_queries = 2
if not anonymous:
expected_update_queries += 1 # Added by django-structlog middleware
if settings.SESSION_ENGINE.endswith("db"):
expected_update_queries += 1 # Added by database session engine
# We don't need to add an extra query for accessing request.user
# because the django-structlog middleware will do that for non-anonymous
expected_acquire_queries = expected_update_queries
else:
expected_acquire_queries = expected_update_queries + 1
with self.assertNumQueries(expected_acquire_queries):
resp = self.client.post(reverse("reserve-asset", args=(asset.pk,)))
data = self.assertValidJSON(resp, expected_status=200)
reservation = AssetTranscriptionReservation.objects.get()
self.assertEqual(reservation.reservation_token, data["reservation_token"])
self.assertEqual(reservation.asset, asset)
# Confirm that an update did not change the pk when it updated the timestamp:
with self.assertNumQueries(expected_update_queries):
resp = self.client.post(reverse("reserve-asset", args=(asset.pk,)))
data = self.assertValidJSON(resp, expected_status=200)
self.assertEqual(1, AssetTranscriptionReservation.objects.count())
updated_reservation = AssetTranscriptionReservation.objects.get()
self.assertEqual(
updated_reservation.reservation_token, data["reservation_token"]
)
self.assertEqual(updated_reservation.asset, asset)
self.assertEqual(reservation.created_on, updated_reservation.created_on)
self.assertLess(reservation.created_on, updated_reservation.updated_on)
# Release the reservation now that we're done:
# 1 release
# + 1 logging if not anonymous
# + 1 session if not anonymous and using a database
expected_release_queries = 1
if not anonymous:
expected_release_queries += 1 # Added by django-structlog middleware
if settings.SESSION_ENGINE.endswith("db"):
expected_release_queries += 1
with self.assertNumQueries(expected_release_queries):
resp = self.client.post(
reverse("reserve-asset", args=(asset.pk,)), data={"release": True}
)
data = self.assertValidJSON(resp, expected_status=200)
self.assertEqual(
updated_reservation.reservation_token, data["reservation_token"]
)
self.assertEqual(0, AssetTranscriptionReservation.objects.count())
def test_asset_reservation_competition(self):
"""
Confirm that two users cannot reserve the same asset at the same time
"""
asset = create_asset()
# We'll reserve the test asset as the anonymous user and then attempt
# to edit it after logging in
# 4 queries =
# 1 expiry + 1 acquire + 2 get user ID + 2 get user profile from request
with self.assertNumQueries(6):
resp = self.client.post(reverse("reserve-asset", args=(asset.pk,)))
self.assertEqual(200, resp.status_code)
self.assertEqual(1, AssetTranscriptionReservation.objects.count())
# Clear the login session so the reservation_token will be regenerated:
self.client.logout()
self.login_user()
# 1 session check + 1 acquire + get user ID from request
with self.assertNumQueries(3 if settings.SESSION_ENGINE.endswith("db") else 2):
resp = self.client.post(reverse("reserve-asset", args=(asset.pk,)))
self.assertEqual(409, resp.status_code)
self.assertEqual(1, AssetTranscriptionReservation.objects.count())
def test_asset_reservation_expiration(self):
"""
Simulate an expired reservation which should not cause the request to fail
"""
asset = create_asset()
stale_reservation = AssetTranscriptionReservation( # nosec
asset=asset, reservation_token="stale"
)
stale_reservation.full_clean()
stale_reservation.save()
# Backdate the object as if it happened 31 minutes ago:
old_timestamp = now() - timedelta(minutes=31)
AssetTranscriptionReservation.objects.update(
created_on=old_timestamp, updated_on=old_timestamp
)
expire_inactive_asset_reservations()
self.login_user()
# 1 reservation check + 1 acquire + 1 get user ID from request
expected_queries = 3
if settings.SESSION_ENGINE.endswith("db"):
# 1 session check
expected_queries += 1
with self.assertNumQueries(expected_queries):
resp = self.client.post(reverse("reserve-asset", args=(asset.pk,)))
data = self.assertValidJSON(resp, expected_status=200)
self.assertEqual(1, AssetTranscriptionReservation.objects.count())
reservation = AssetTranscriptionReservation.objects.get()
self.assertEqual(reservation.reservation_token, data["reservation_token"])
def test_asset_reservation_tombstone(self):
"""
Simulate a tombstoned reservation which should:
- return 408 during the tombstone period
- during the tombstone period, another user may
obtain the reservation but the original user may not
"""
asset = create_asset()
self.login_user()
request_factory = RequestFactory()
request = request_factory.get("/")
request.session = {}
reservation_token = get_or_create_reservation_token(request)
session = self.client.session
session["reservation_token"] = reservation_token
session.save()
tombstone_reservation = AssetTranscriptionReservation( # nosec
asset=asset, reservation_token=reservation_token
)
tombstone_reservation.full_clean()
tombstone_reservation.save()
# Backdate the object as if it was created hours ago,
# even if it was recently updated
old_timestamp = now() - timedelta(
hours=settings.TRANSCRIPTION_RESERVATION_TOMBSTONE_HOURS + 1
)
current_timestamp = now()
AssetTranscriptionReservation.objects.update(
created_on=old_timestamp, updated_on=current_timestamp
)
tombstone_old_active_asset_reservations()
self.assertEqual(1, AssetTranscriptionReservation.objects.count())
reservation = AssetTranscriptionReservation.objects.get()
self.assertEqual(reservation.tombstoned, True)
# 1 session check + 1 reservation check + 1 logging
if settings.SESSION_ENGINE.endswith("db"):
expected_queries = 3
else:
expected_queries = 2
with self.assertNumQueries(expected_queries):
resp = self.client.post(reverse("reserve-asset", args=(asset.pk,)))
self.assertEqual(resp.status_code, 408)
self.assertEqual(1, AssetTranscriptionReservation.objects.count())
reservation = AssetTranscriptionReservation.objects.get()
self.assertEqual(reservation.reservation_token, reservation_token)
self.client.logout()
# 1 reservation check + 1 acquire + 1 get user ID
expected_queries = 3
if settings.SESSION_ENGINE.endswith("db"):
# + 1 session check
expected_queries += 1
User.objects.create_user(username="anonymous")
with self.assertNumQueries(expected_queries):
resp = self.client.post(reverse("reserve-asset", args=(asset.pk,)))
self.assertValidJSON(resp, expected_status=200)
self.assertEqual(2, AssetTranscriptionReservation.objects.count())
def test_asset_reservation_tombstone_expiration(self):
"""
Simulate a tombstoned reservation which should expire after
the configured period of time, allowing the original user
to reserve the asset again
"""
asset = create_asset()
self.login_user()
request_factory = RequestFactory()
request = request_factory.get("/")
request.session = {}
reservation_token = get_or_create_reservation_token(request)
session = self.client.session
session["reservation_token"] = reservation_token
session.save()
tombstone_reservation = AssetTranscriptionReservation( # nosec
asset=asset, reservation_token=reservation_token
)
tombstone_reservation.full_clean()
tombstone_reservation.save()
# Backdate the object as if it was created hours ago
# and tombstoned hours ago
old_timestamp = now() - timedelta(
hours=settings.TRANSCRIPTION_RESERVATION_TOMBSTONE_HOURS
+ settings.TRANSCRIPTION_RESERVATION_TOMBSTONE_LENGTH_HOURS
+ 1
)
not_as_old_timestamp = now() - timedelta(
hours=settings.TRANSCRIPTION_RESERVATION_TOMBSTONE_LENGTH_HOURS + 1
)
AssetTranscriptionReservation.objects.update(
created_on=old_timestamp, updated_on=not_as_old_timestamp, tombstoned=True
)
delete_old_tombstoned_reservations()
self.assertEqual(0, AssetTranscriptionReservation.objects.count())
# 1 session check + 1 reservation check + 1 acquire + 1logging
if settings.SESSION_ENGINE.endswith("db"):
expected_queries = 4
else:
expected_queries = 3
with self.assertNumQueries(expected_queries):
resp = self.client.post(reverse("reserve-asset", args=(asset.pk,)))
data = self.assertValidJSON(resp, expected_status=200)
self.assertEqual(1, AssetTranscriptionReservation.objects.count())
reservation = AssetTranscriptionReservation.objects.get()
self.assertEqual(reservation.reservation_token, data["reservation_token"])
self.assertEqual(reservation.tombstoned, False)
def tearDown(self):
# We'll test the signal handler separately
post_save.connect(on_transcription_save, sender=Transcription)
================================================
FILE: concordia/tests/test_views_redirect_next_reviewable.py
================================================
from unittest.mock import patch
from django.db.models.signals import post_save
from django.test import (
TransactionTestCase,
override_settings,
)
from django.urls import reverse
from django.utils.timezone import now
from concordia.models import (
Transcription,
)
from concordia.signals.handlers import on_transcription_save
from concordia.utils import get_anonymous_user
from .utils import (
CreateTestUsers,
JSONAssertMixin,
create_asset,
create_campaign,
create_item,
create_project,
create_topic,
)
@override_settings(
RATELIMIT_ENABLE=False, SESSION_ENGINE="django.contrib.sessions.backends.cache"
)
class NextReviewableRedirectViewTests(
CreateTestUsers, JSONAssertMixin, TransactionTestCase
):
def test_find_next_reviewable_no_campaign(self):
user = self.create_user("test-user")
anon = get_anonymous_user()
# Test case where there are no reviewable assets
response = self.client.get(reverse("redirect-to-next-reviewable-asset"))
self.assertRedirects(response, expected_url="/")
asset1 = create_asset(slug="test-asset-1", title="Test Asset 1")
asset2 = create_asset(
item=asset1.item, slug="test-asset-2", title="Test Asset 2"
)
asset3 = create_asset(
item=asset1.item, slug="test-asset-3", title="Test Asset 3"
)
campaign = asset1.item.project.campaign
t1 = Transcription(asset=asset1, user=user, text="test", submitted=now())
t1.full_clean()
t1.save()
t2 = Transcription(asset=asset2, user=anon, text="test", submitted=now())
t2.full_clean()
t2.save()
t3 = Transcription(asset=asset3, user=anon, text="test", submitted=now())
t3.full_clean()
t3.save()
response = self.client.get(reverse("redirect-to-next-reviewable-asset"))
self.assertRedirects(response, expected_url=asset1.get_absolute_url())
# Test logged in user (this creates a new user)
# asset1 is no longer available due to the request above reserving it
self.login_user()
response = self.client.get(reverse("redirect-to-next-reviewable-asset"))
self.assertRedirects(response, expected_url=asset2.get_absolute_url())
# Configure campaign to be next review cmpaign for tests below
campaign.next_review_campaign = True
campaign.save()
# Test when next reviewable campaign doesn't exist and there
# are no other campaigns/assets
with patch("concordia.models.Campaign.objects.get") as mock:
mock.side_effect = IndexError
response = self.client.get(reverse("redirect-to-next-reviewable-asset"))
self.assertRedirects(response, expected_url="/")
# Test case when a campaign is configured to be default next reviewable
response = self.client.get(reverse("redirect-to-next-reviewable-asset"))
self.assertRedirects(response, expected_url=asset3.get_absolute_url())
# Test when next reviewable campaign has no reviewable assets
asset1.delete()
asset2.delete()
response = self.client.get(reverse("redirect-to-next-reviewable-asset"))
self.assertRedirects(response, expected_url="/")
# Test when next reviewable campaign has no reviewable assets
# and other campaigns exist and have no reviewable assets
create_campaign(slug="test-campaign-2")
response = self.client.get(reverse("redirect-to-next-reviewable-asset"))
self.assertRedirects(response, expected_url="/")
def test_find_next_reviewable_campaign(self):
anon = get_anonymous_user()
asset1 = create_asset(slug="test-review-asset-1", title="Test Asset 1")
asset2 = create_asset(
item=asset1.item, slug="test-review-asset-2", title="Test Asset 2"
)
t1 = Transcription(asset=asset1, user=anon, text="test", submitted=now())
t1.full_clean()
t1.save()
t2 = Transcription(asset=asset2, user=anon, text="test", submitted=now())
t2.full_clean()
t2.save()
campaign = asset1.item.project.campaign
# Anonymous user test
response = self.client.get(
reverse(
"transcriptions:redirect-to-next-reviewable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
)
)
self.assertRedirects(response, expected_url=asset1.get_absolute_url())
# Authenticated user test
# asset1 is no longer available since the previous request reserved it
self.login_user()
response = self.client.get(
reverse(
"transcriptions:redirect-to-next-reviewable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
)
)
self.assertRedirects(response, expected_url=asset2.get_absolute_url())
def test_find_next_reviewable_topic(self):
anon = get_anonymous_user()
asset1 = create_asset(slug="test-review-asset-1")
asset2 = create_asset(item=asset1.item, slug="test-review-asset-2")
project = asset1.item.project
topic = create_topic(project=project)
t1 = Transcription(asset=asset1, user=anon, text="test", submitted=now())
t1.full_clean()
t1.save()
t2 = Transcription(asset=asset2, user=anon, text="test", submitted=now())
t2.full_clean()
t2.save()
# Anonymous user test
response = self.client.get(
reverse(
"redirect-to-next-reviewable-topic-asset",
kwargs={"topic_slug": topic.slug},
)
)
self.assertRedirects(response, expected_url=asset1.get_absolute_url())
# Authenticated user test
# We expect that asset1 is no longer available. Even though
# anonymous users can't reserve assets for review, we still will
# have removed the asset from the NextReviewableTopicAsset table
# to ensure two users don't receive the same asset
self.login_user()
response = self.client.get(
reverse(
"redirect-to-next-reviewable-topic-asset",
kwargs={"topic_slug": topic.slug},
)
)
self.assertRedirects(response, expected_url=asset2.get_absolute_url())
def test_find_next_reviewable_unlisted_campaign(self):
anon = get_anonymous_user()
unlisted_campaign = create_campaign(
slug="campaign-transcribe-redirect-unlisted",
title="Test Unlisted Review Redirect Campaign",
unlisted=True,
)
unlisted_project = create_project(
title="Unlisted Project",
slug="unlisted-project",
campaign=unlisted_campaign,
)
unlisted_item = create_item(
title="Unlisted Item",
item_id="unlisted-item",
item_url="https://blah.com/unlisted-item",
project=unlisted_project,
)
asset1 = create_asset(slug="test-asset-1", item=unlisted_item)
asset2 = create_asset(item=asset1.item, slug="test-asset-2")
t1 = Transcription(asset=asset1, user=anon, text="test", submitted=now())
t1.full_clean()
t1.save()
t2 = Transcription(asset=asset2, user=anon, text="test", submitted=now())
t2.full_clean()
t2.save()
response = self.client.get(
reverse(
"transcriptions:redirect-to-next-reviewable-campaign-asset",
kwargs={"campaign_slug": unlisted_campaign.slug},
)
)
self.assertRedirects(response, expected_url=asset1.get_absolute_url())
def tearDown(self):
# We'll test the signal handler separately
post_save.connect(on_transcription_save, sender=Transcription)
================================================
FILE: concordia/tests/test_views_redirect_next_transcribable.py
================================================
from unittest.mock import patch
from django.db.models.signals import post_save
from django.test import (
TransactionTestCase,
override_settings,
)
from django.urls import reverse
from concordia.models import (
AssetTranscriptionReservation,
Transcription,
TranscriptionStatus,
)
from concordia.signals.handlers import on_transcription_save
from .utils import (
CreateTestUsers,
JSONAssertMixin,
create_asset,
create_campaign,
create_item,
create_project,
create_topic,
)
@override_settings(
RATELIMIT_ENABLE=False, SESSION_ENGINE="django.contrib.sessions.backends.cache"
)
class NextTranscribableRedirectViewTests(
CreateTestUsers, JSONAssertMixin, TransactionTestCase
):
def test_find_next_transcribable_no_campaign(self):
# Test case where there are no transcribable assets
resp = self.client.get(reverse("redirect-to-next-transcribable-asset"))
self.assertRedirects(resp, expected_url="/")
asset1 = create_asset(slug="test-asset-1")
asset2 = create_asset(item=asset1.item, slug="test-asset-2")
campaign = asset1.item.project.campaign
resp = self.client.get(reverse("redirect-to-next-transcribable-asset"))
self.assertRedirects(resp, expected_url=asset1.get_absolute_url())
# Configure next transcription campaign for tests below
campaign.next_transcription_campaign = True
campaign.save()
# Test when next transcribable campaign doesn't exist and there
# are no other campaigns/assets
with patch("concordia.models.Campaign.objects.get") as mock:
mock.side_effect = IndexError
response = self.client.get(reverse("redirect-to-next-transcribable-asset"))
self.assertRedirects(response, expected_url="/")
# Test case when a campaign is configured to be default next transcribable
response = self.client.get(reverse("redirect-to-next-transcribable-asset"))
self.assertRedirects(response, expected_url=asset2.get_absolute_url())
# Test when next transcribable campaign has not transcribable assets
asset1.delete()
asset2.delete()
response = self.client.get(reverse("redirect-to-next-transcribable-asset"))
self.assertRedirects(response, expected_url="/")
# Test when next transcription campaign has no transcribable assets
# and other campaigns exist and have no transcribable assets
create_campaign(slug="test-campaign-2")
response = self.client.get(reverse("redirect-to-next-transcribable-asset"))
self.assertRedirects(response, expected_url="/")
def test_find_next_transcribable_campaign(self):
asset1 = create_asset(slug="test-asset-1")
asset2 = create_asset(item=asset1.item, slug="test-asset-2")
campaign = asset1.item.project.campaign
# Anonymous user test
resp = self.client.get(
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
)
)
self.assertRedirects(resp, expected_url=asset1.get_absolute_url())
# Authenticated user test
self.login_user()
resp = self.client.get(
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
)
)
self.assertRedirects(resp, expected_url=asset2.get_absolute_url())
def test_find_next_transcribable_topic(self):
asset1 = create_asset(slug="test-asset-1")
asset2 = create_asset(item=asset1.item, slug="test-asset-2")
project = asset1.item.project
topic = create_topic(project=project)
# Anonymous user test
resp = self.client.get(
reverse(
"redirect-to-next-transcribable-topic-asset",
kwargs={"topic_slug": topic.slug},
)
)
self.assertRedirects(resp, expected_url=asset1.get_absolute_url())
# Authenticated user test
self.login_user()
resp = self.client.get(
reverse(
"redirect-to-next-transcribable-topic-asset",
kwargs={"topic_slug": topic.slug},
)
)
self.assertRedirects(resp, expected_url=asset2.get_absolute_url())
def test_find_next_transcribable_unlisted_campaign(self):
unlisted_campaign = create_campaign(
slug="campaign-transcribe-redirect-unlisted",
title="Test Unlisted Transcribe Redirect Campaign",
unlisted=True,
)
unlisted_project = create_project(
title="Unlisted Project",
slug="unlisted-project",
campaign=unlisted_campaign,
)
unlisted_item = create_item(
title="Unlisted Item",
item_id="unlisted-item",
item_url="https://blah.com/unlisted-item",
project=unlisted_project,
)
asset1 = create_asset(slug="test-asset-1", item=unlisted_item)
create_asset(item=asset1.item, slug="test-asset-2")
response = self.client.get(
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": unlisted_campaign.slug},
)
)
self.assertRedirects(response, expected_url=asset1.get_absolute_url())
def test_find_next_transcribable_single_asset(self):
asset = create_asset()
campaign = asset.item.project.campaign
resp = self.client.get(
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
)
)
self.assertRedirects(resp, expected_url=asset.get_absolute_url())
def test_find_next_transcribable_in_singleton_campaign(self):
asset = create_asset(transcription_status=TranscriptionStatus.SUBMITTED)
campaign = asset.item.project.campaign
resp = self.client.get(
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
)
)
self.assertRedirects(resp, expected_url=reverse("homepage"))
def test_find_next_transcribable_project_redirect(self):
asset = create_asset(transcription_status=TranscriptionStatus.SUBMITTED)
project = asset.item.project
campaign = project.campaign
resp = self.client.get(
"%s?project=%s"
% (
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
),
project.slug,
)
)
self.assertRedirects(resp, expected_url=reverse("homepage"))
def test_find_next_transcribable_hierarchy(self):
"""Confirm that find-next-page selects assets in the expected order"""
asset = create_asset()
item = asset.item
project = item.project
campaign = project.campaign
asset_in_item = create_asset(item=item, slug="test-asset-in-same-item")
in_progress_asset_in_item = create_asset(
item=item,
slug="inprogress-asset-in-same-item",
transcription_status=TranscriptionStatus.IN_PROGRESS,
)
asset_in_project = create_asset(
item=create_item(project=project, item_id="other-item-in-same-project"),
title="test-asset-in-same-project",
)
asset_in_campaign = create_asset(
item=create_item(
project=create_project(campaign=campaign, title="other project"),
item_id="another-item-in-different-project",
title="item in other project",
),
slug="test-asset-in-same-campaign",
)
# Now that we have test assets we'll see what find-next-page gives us as
# successive test records are marked as submitted and thus ineligible.
# The expected ordering is that it will favor moving forward (i.e. not
# landing you on the same asset unless that's the only one available),
# and will keep you closer to the asset you started from (i.e. within
# the same item or project in that order).
self.assertRedirects(
self.client.get(
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
),
{"project": project.slug, "item": item.item_id, "asset": asset.pk},
),
asset_in_item.get_absolute_url(),
)
asset_in_item.transcription_status = TranscriptionStatus.SUBMITTED
asset_in_item.save()
AssetTranscriptionReservation.objects.all().delete()
self.assertRedirects(
self.client.get(
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
),
{"project": project.slug, "item": item.item_id, "asset": asset.pk},
),
asset_in_project.get_absolute_url(),
)
asset_in_project.transcription_status = TranscriptionStatus.SUBMITTED
asset_in_project.save()
AssetTranscriptionReservation.objects.all().delete()
self.assertRedirects(
self.client.get(
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
),
{"project": project.slug, "item": item.item_id, "asset": asset.pk},
),
asset_in_campaign.get_absolute_url(),
)
asset_in_campaign.transcription_status = TranscriptionStatus.SUBMITTED
asset_in_campaign.save()
AssetTranscriptionReservation.objects.all().delete()
self.assertRedirects(
self.client.get(
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": campaign.slug},
),
{"project": project.slug, "item": item.item_id, "asset": asset.pk},
),
in_progress_asset_in_item.get_absolute_url(),
)
def tearDown(self):
# We'll test the signal handler separately
post_save.connect(on_transcription_save, sender=Transcription)
================================================
FILE: concordia/tests/test_views_tags.py
================================================
from unittest.mock import patch
from django import forms
from django.contrib.auth.models import User
from django.db.models.signals import post_save
from django.test import (
TransactionTestCase,
override_settings,
)
from django.urls import reverse
from concordia.models import (
Transcription,
)
from concordia.signals.handlers import on_transcription_save
from .utils import (
CreateTestUsers,
JSONAssertMixin,
create_asset,
)
@override_settings(
RATELIMIT_ENABLE=False, SESSION_ENGINE="django.contrib.sessions.backends.cache"
)
class TagSubmissionViewTests(CreateTestUsers, JSONAssertMixin, TransactionTestCase):
def test_anonymous_tag_submission(self):
"""Confirm that anonymous users cannot submit tags"""
asset = create_asset()
submit_url = reverse("submit-tags", kwargs={"asset_pk": asset.pk})
resp = self.client.post(submit_url, data={"tags": ["foo", "bar"]})
self.assertRedirects(resp, "%s?next=%s" % (reverse("login"), submit_url))
def test_tag_submission(self):
asset = create_asset()
self.login_user()
test_tags = ["foo", "bar"]
resp = self.client.post(
reverse("submit-tags", kwargs={"asset_pk": asset.pk}),
data={"tags": test_tags},
)
data = self.assertValidJSON(resp, expected_status=200)
self.assertIn("user_tags", data)
self.assertIn("all_tags", data)
self.assertEqual(sorted(test_tags), data["user_tags"])
self.assertEqual(sorted(test_tags), data["all_tags"])
def test_invalid_tag_submission(self):
asset = create_asset()
self.login_user()
test_tags = ["foo", "bar"]
with patch("concordia.models.Tag.full_clean") as mock:
mock.side_effect = forms.ValidationError("Testing error")
resp = self.client.post(
reverse("submit-tags", kwargs={"asset_pk": asset.pk}),
data={"tags": test_tags},
)
data = self.assertValidJSON(resp, expected_status=400)
self.assertIn("error", data)
def test_tag_submission_with_diacritics(self):
asset = create_asset()
self.login_user()
test_tags = ["Café", "château", "señor", "façade"]
resp = self.client.post(
reverse("submit-tags", kwargs={"asset_pk": asset.pk}),
data={"tags": test_tags},
)
data = self.assertValidJSON(resp, expected_status=200)
self.assertIn("user_tags", data)
self.assertIn("all_tags", data)
self.assertEqual(sorted(test_tags), data["user_tags"])
self.assertEqual(sorted(test_tags), data["all_tags"])
def test_tag_submission_with_multiple_users(self):
asset = create_asset()
self.login_user()
test_tags = ["foo", "bar"]
resp = self.client.post(
reverse("submit-tags", kwargs={"asset_pk": asset.pk}),
data={"tags": test_tags},
)
data = self.assertValidJSON(resp, expected_status=200)
self.assertIn("user_tags", data)
self.assertIn("all_tags", data)
self.assertEqual(sorted(test_tags), data["user_tags"])
self.assertEqual(sorted(test_tags), data["all_tags"])
def test_duplicate_tag_submission(self):
"""Confirm that tag values cannot be duplicated"""
asset = create_asset()
self.login_user()
resp = self.client.post(
reverse("submit-tags", kwargs={"asset_pk": asset.pk}),
data={"tags": ["foo", "bar", "baaz"]},
)
data = self.assertValidJSON(resp, expected_status=200)
second_user = self.create_test_user(
username="second_tester", email="second_tester@example.com"
)
self.client.login(username=second_user.username, password=second_user._password)
resp = self.client.post(
reverse("submit-tags", kwargs={"asset_pk": asset.pk}),
data={"tags": ["foo", "bar", "quux"]},
)
data = self.assertValidJSON(resp, expected_status=200)
# Even though the user submitted (through some horrible bug) duplicate
# values, they should not be stored:
self.assertEqual(["bar", "foo", "quux"], data["user_tags"])
# Users are allowed to delete other users' tags, so since the second
# user didn't send the "baaz" tag, it was removed
self.assertEqual(["bar", "foo", "quux"], data["all_tags"])
def test_tag_deletion(self):
asset = create_asset()
self.login_user()
initial_tags = ["foo", "bar"]
self.client.post(
reverse("submit-tags", kwargs={"asset_pk": asset.pk}),
data={"tags": initial_tags},
)
updated_tags = [
"foo",
]
resp = self.client.post(
reverse("submit-tags", kwargs={"asset_pk": asset.pk}),
data={"tags": updated_tags},
)
data = self.assertValidJSON(resp, expected_status=200)
self.assertIn("user_tags", data)
self.assertIn("all_tags", data)
self.assertCountEqual(updated_tags, data["user_tags"])
self.assertCountEqual(updated_tags, data["all_tags"])
def test_tag_deletion_with_multiple_users(self):
asset = create_asset()
self.login_user("first_user")
initial_tags = ["foo", "bar"]
resp = self.client.post(
reverse("submit-tags", kwargs={"asset_pk": asset.pk}),
data={"tags": initial_tags},
)
self.assertIn(
"first_user",
asset.userassettagcollection_set.values().values_list(
"user__username", flat=True
),
)
data = self.assertValidJSON(resp, expected_status=200)
self.assertIn("user_tags", data)
self.assertIn("all_tags", data)
self.assertCountEqual(initial_tags, data["user_tags"])
self.assertCountEqual(initial_tags, data["all_tags"])
self.client.logout()
second_user = self.create_test_user("second_user")
self.client.login(username=second_user.username, password=second_user._password)
updated_tags = [
"foo",
]
resp = self.client.post(
reverse("submit-tags", kwargs={"asset_pk": asset.pk}),
data={"tags": updated_tags},
)
data = self.assertValidJSON(resp, expected_status=200)
self.assertIn(
"second_user",
asset.userassettagcollection_set.values().values_list(
"user__username", flat=True
),
)
self.assertEqual(asset.userassettagcollection_set.count(), 2)
self.assertEqual(
User.objects.filter(userassettagcollection__asset=asset).count(), 2
)
self.assertIn("user_tags", data)
self.assertIn("all_tags", data)
self.assertCountEqual(updated_tags, data["user_tags"])
def tearDown(self):
# We'll test the signal handler separately
post_save.connect(on_transcription_save, sender=Transcription)
================================================
FILE: concordia/tests/test_views_topics.py
================================================
from django.core.cache import caches
from django.test import TestCase, override_settings
from django.urls import reverse
from concordia.models import ProjectTopic, TranscriptionStatus
from .utils import (
CreateTestUsers,
JSONAssertMixin,
create_asset,
create_campaign,
create_item,
create_project,
create_topic,
)
@override_settings(
RATELIMIT_ENABLE=False,
SESSION_ENGINE="django.contrib.sessions.backends.cache",
CACHES={
"default": {"BACKEND": "django.core.cache.backends.dummy.DummyCache"},
"view_cache": {"BACKEND": "django.core.cache.backends.dummy.DummyCache"},
},
)
class TopicDetailViewTests(CreateTestUsers, JSONAssertMixin, TestCase):
"""
Focused tests for the Topic detail view.
"""
def setUp(self):
for cache in caches.all():
cache.clear()
def tearDown(self):
for cache in caches.all():
cache.clear()
def test_topic_detail_basic(self):
topic = create_topic(title="GET Topic", slug="get-topic")
response = self.client.get(reverse("topic-detail", args=(topic.slug,)))
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/topic_detail.html"
)
self.assertContains(response, topic.title)
def test_unlisted_topic_detail_view(self):
c2 = create_topic(
title="GET Unlisted Topic", unlisted=True, slug="get-unlisted-topic"
)
response2 = self.client.get(reverse("topic-detail", args=(c2.slug,)))
self.assertEqual(response2.status_code, 200)
self.assertTemplateUsed(
response2, template_name="transcriptions/topic_detail.html"
)
self.assertContains(response2, c2.title)
def test_topic_detail_with_status_sets_querystring(self):
"""
When a valid transcription_status is supplied, sublevel_querystring
contains only that param.
"""
topic = create_topic(title="GET Topic", slug="get-topic")
response = self.client.get(
reverse("topic-detail", args=(topic.slug,)),
{"transcription_status": "not_started"},
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(
response, template_name="transcriptions/topic_detail.html"
)
self.assertContains(response, topic.title)
self.assertIn("sublevel_querystring", response.context)
self.assertEqual(
response.context["sublevel_querystring"], "transcription_status=not_started"
)
def test_url_filter_links_without_sublevel_querystring(self):
"""
With a project-level url_filter and no sublevel filter, links for that
project include transcription_status=, while projects without
a url_filter do not include a transcription_status param.
"""
topic = create_topic(title="Filter Topic", slug="filter-topic")
campaign = create_campaign(title="Filter Test Campaign", slug="filter-test")
project_with_filter = create_project(
campaign=campaign, title="Project With Filter", slug="with-filter"
)
project_without_filter = create_project(
campaign=campaign, title="Project Without Filter", slug="without-filter"
)
ProjectTopic.objects.create(
project=project_with_filter,
topic=topic,
url_filter=TranscriptionStatus.SUBMITTED,
)
ProjectTopic.objects.create(
project=project_without_filter,
topic=topic,
url_filter=None,
)
response = self.client.get(reverse("topic-detail", args=(topic.slug,)))
self.assertEqual(response.status_code, 200)
# project_with_filter has ?transcription_status=submitted
# (appears twice: image+title)
self.assertContains(
response,
f"/campaigns/{campaign.slug}/{project_with_filter.slug}/?transcription_status=submitted",
2,
)
# project_without_filter should not include any transcription_status param
self.assertNotContains(
response,
f"/campaigns/{campaign.slug}/{project_without_filter.slug}/?transcription_status=",
)
def test_sublevel_querystring_only_keeps_transcription_status(self):
"""
If extra params are provided along with transcription_status, only
transcription_status is retained in sublevel_querystring.
"""
topic = create_topic(title="GET Topic", slug="get-topic")
response = self.client.get(
reverse("topic-detail", args=(topic.slug,)),
{"transcription_status": "not_started", "another_param": "some_value"},
)
self.assertEqual(response.status_code, 200)
self.assertIn("sublevel_querystring", response.context)
self.assertEqual(
response.context["sublevel_querystring"], "transcription_status=not_started"
)
def test_with_status_and_no_assets_excludes_projects(self):
"""
When a transcription_status is present and projects have no assets,
those projects are excluded (no links rendered).
"""
topic = create_topic(title="Filter Topic", slug="filter-topic")
campaign = create_campaign(title="Filter Test Campaign", slug="filter-test")
project_with_filter = create_project(
campaign=campaign, title="Project With Filter", slug="with-filter"
)
project_without_filter = create_project(
campaign=campaign, title="Project Without Filter", slug="without-filter"
)
ProjectTopic.objects.create(
project=project_with_filter,
topic=topic,
url_filter=TranscriptionStatus.SUBMITTED,
)
ProjectTopic.objects.create(
project=project_without_filter,
topic=topic,
url_filter=None,
)
response = self.client.get(
reverse("topic-detail", args=(topic.slug,)),
{"transcription_status": "not_started", "another_param": "some_value"},
)
self.assertEqual(response.status_code, 200)
self.assertIn("sublevel_querystring", response.context)
self.assertEqual(
response.context["sublevel_querystring"], "transcription_status=not_started"
)
# No assets exist, so neither project should appear with the filter applied
self.assertContains(
response,
f"/campaigns/{campaign.slug}/{project_with_filter.slug}/?transcription_status=not_started",
0,
)
self.assertContains(
response,
f"/campaigns/{campaign.slug}/{project_without_filter.slug}/?transcription_status=not_started",
0,
)
def test_with_status_and_assets_uses_sublevel_and_overrides_url_filter(self):
"""
When assets exist and a transcription_status is supplied, projects with no
url_filter are shown using the sublevel filter. Projects with a url_filter
that does not match the sublevel filter are excluded.
"""
topic = create_topic(title="Filter Topic", slug="filter-topic")
campaign = create_campaign(title="Filter Test Campaign", slug="filter-test")
project_with_filter = create_project(
campaign=campaign, title="Project With Filter", slug="with-filter"
)
project_without_filter = create_project(
campaign=campaign, title="Project Without Filter", slug="without-filter"
)
ProjectTopic.objects.create(
project=project_with_filter,
topic=topic,
url_filter=TranscriptionStatus.SUBMITTED,
)
ProjectTopic.objects.create(
project=project_without_filter,
topic=topic,
url_filter=None,
)
# Add assets so eligible projects will display
item_with_filter = create_item(project=project_with_filter)
create_asset(item=item_with_filter)
item_without_filter = create_item(project=project_without_filter)
create_asset(item=item_without_filter)
response = self.client.get(
reverse("topic-detail", args=(topic.slug,)),
{"transcription_status": "not_started", "another_param": "some_value"},
)
self.assertEqual(response.status_code, 200)
self.assertIn("sublevel_querystring", response.context)
self.assertEqual(
response.context["sublevel_querystring"], "transcription_status=not_started"
)
# Project WITH a mismatching url_filter should be excluded
self.assertContains(
response,
f"/campaigns/{campaign.slug}/{project_with_filter.slug}/?transcription_status=not_started",
0,
)
# Project WITHOUT a url_filter should use the sublevel filter
# (appears twice: image + title)
self.assertContains(
response,
f"/campaigns/{campaign.slug}/{project_without_filter.slug}/?transcription_status=not_started",
2,
)
def test_with_status_and_assets_includes_matching_url_filter(self):
"""
When assets exist and a transcription_status is supplied, projects with a
matching url_filter should be included, and links should use that status.
"""
topic = create_topic(title="Filter Topic", slug="filter-topic")
campaign = create_campaign(title="Filter Test Campaign", slug="filter-test")
project_with_filter = create_project(
campaign=campaign, title="Project With Filter", slug="with-filter"
)
project_without_filter = create_project(
campaign=campaign, title="Project Without Filter", slug="without-filter"
)
ProjectTopic.objects.create(
project=project_with_filter,
topic=topic,
url_filter=TranscriptionStatus.SUBMITTED,
)
ProjectTopic.objects.create(
project=project_without_filter,
topic=topic,
url_filter=None,
)
# Ensure both projects have at least one asset counted as "submitted"
item_with_filter = create_item(project=project_with_filter)
a1 = create_asset(item=item_with_filter)
a1.transcription_status = TranscriptionStatus.SUBMITTED
a1.save(update_fields=["transcription_status"])
item_without_filter = create_item(project=project_without_filter)
a2 = create_asset(item=item_without_filter)
a2.transcription_status = TranscriptionStatus.SUBMITTED
a2.save(update_fields=["transcription_status"])
response = self.client.get(
reverse("topic-detail", args=(topic.slug,)),
{"transcription_status": "submitted"},
)
self.assertEqual(response.status_code, 200)
self.assertIn("sublevel_querystring", response.context)
self.assertEqual(
response.context["sublevel_querystring"], "transcription_status=submitted"
)
# Project WITH a matching url_filter should be included (2 links: image + title)
self.assertContains(
response,
f"/campaigns/{campaign.slug}/{project_with_filter.slug}/?transcription_status=submitted",
2,
)
# Project WITHOUT a url_filter should also be included (also 2 links)
self.assertContains(
response,
f"/campaigns/{campaign.slug}/{project_without_filter.slug}/?transcription_status=submitted",
2,
)
def test_topic_detail_with_invalid_status_ignores_filter(self):
"""
If transcription_status is present but invalid, the view should treat it
as absent: no filtering by status and no sublevel_querystring.
"""
topic = create_topic(title="Filter Topic", slug="filter-topic")
campaign = create_campaign(title="Filter Test Campaign", slug="filter-test")
project_with_filter = create_project(
campaign=campaign, title="Project With Filter", slug="with-filter"
)
project_without_filter = create_project(
campaign=campaign, title="Project Without Filter", slug="without-filter"
)
ProjectTopic.objects.create(
project=project_with_filter,
topic=topic,
url_filter=TranscriptionStatus.SUBMITTED,
)
ProjectTopic.objects.create(
project=project_without_filter,
topic=topic,
url_filter=None,
)
# Make both projects eligible to display
create_asset(item=create_item(project=project_with_filter))
create_asset(item=create_item(project=project_without_filter))
# Supply an invalid status
response = self.client.get(
reverse("topic-detail", args=(topic.slug,)),
{"transcription_status": "not-a-real-status", "another_param": "x"},
)
self.assertEqual(response.status_code, 200)
# sublevel_querystring should be empty (invalid status ignored)
self.assertIn("sublevel_querystring", response.context)
self.assertEqual(response.context["sublevel_querystring"], "")
# Project WITH a url_filter should use its own filter in links
self.assertContains(
response,
f"/campaigns/{campaign.slug}/{project_with_filter.slug}/?transcription_status=submitted",
2,
)
# Project WITHOUT a url_filter should not include a transcription_status param
self.assertNotContains(
response,
f"/campaigns/{campaign.slug}/{project_without_filter.slug}/?transcription_status=",
)
def test_url_filter_empty_string_treated_as_missing(self):
topic = create_topic(title="Filter Topic", slug="filter-topic")
campaign = create_campaign(title="Filter Test Campaign", slug="filter-test")
project_empty_filter = create_project(
campaign=campaign, title="Project Empty Filter", slug="empty-filter"
)
project_none_filter = create_project(
campaign=campaign, title="Project None Filter", slug="none-filter"
)
ProjectTopic.objects.create(
project=project_empty_filter, topic=topic, url_filter=""
)
ProjectTopic.objects.create(
project=project_none_filter, topic=topic, url_filter=None
)
# Make both eligible
item_empty = create_item(project=project_empty_filter)
asset_empty = create_asset(item=item_empty)
item_none = create_item(project=project_none_filter)
asset_none = create_asset(item=item_none)
# no sublevel filter, so neither link should have transcription_status
resp1 = self.client.get(reverse("topic-detail", args=(topic.slug,)))
self.assertEqual(resp1.status_code, 200)
self.assertNotContains(
resp1,
f"/campaigns/{campaign.slug}/{project_empty_filter.slug}/?transcription_status=",
)
self.assertNotContains(
resp1,
f"/campaigns/{campaign.slug}/{project_none_filter.slug}/?transcription_status=",
)
# Set at least one asset to SUBMITTED for each project (so they’re not excluded)
asset_empty.transcription_status = TranscriptionStatus.SUBMITTED
asset_empty.save(update_fields=["transcription_status"])
asset_none.transcription_status = TranscriptionStatus.SUBMITTED
asset_none.save(update_fields=["transcription_status"])
# valid sublevel filter, so both included and use that status in links
resp2 = self.client.get(
reverse("topic-detail", args=(topic.slug,)),
{"transcription_status": "submitted"},
)
self.assertEqual(resp2.status_code, 200)
self.assertContains(
resp2,
f"/campaigns/{campaign.slug}/{project_empty_filter.slug}/?transcription_status=submitted",
2,
)
self.assertContains(
resp2,
f"/campaigns/{campaign.slug}/{project_none_filter.slug}/?transcription_status=submitted",
2,
)
================================================
FILE: concordia/tests/test_views_transcription_review.py
================================================
from django.core.cache import caches
from django.db.models.signals import post_save
from django.test import (
TransactionTestCase,
override_settings,
)
from django.urls import reverse
from django.utils.timezone import now
from concordia.models import (
Asset,
Transcription,
TranscriptionStatus,
)
from concordia.signals.handlers import on_transcription_save
from concordia.utils import get_anonymous_user
from configuration.models import Configuration
from .utils import (
CreateTestUsers,
JSONAssertMixin,
create_asset,
create_transcription,
)
@override_settings(
RATELIMIT_ENABLE=False, SESSION_ENGINE="django.contrib.sessions.backends.cache"
)
class ReviewTranscriptionViewTests(
CreateTestUsers, JSONAssertMixin, TransactionTestCase
):
def test_transcription_review(self):
asset = create_asset()
anon = get_anonymous_user()
t1 = Transcription(asset=asset, user=anon, text="test", submitted=now())
t1.full_clean()
t1.save()
self.login_user()
resp = self.client.post(
reverse("review-transcription", args=(t1.pk,)), data={"action": "foobar"}
)
data = self.assertValidJSON(resp, expected_status=400)
self.assertIn("error", data)
self.assertEqual(
1, Transcription.objects.filter(pk=t1.pk, accepted__isnull=True).count()
)
resp = self.client.post(
reverse("review-transcription", args=(t1.pk,)), data={"action": "accept"}
)
data = self.assertValidJSON(resp, expected_status=200)
self.assertEqual(
1, Transcription.objects.filter(pk=t1.pk, accepted__isnull=False).count()
)
def test_transcription_review_rate_limit(self):
for cache in caches.all():
cache.clear()
anon = get_anonymous_user()
self.login_user()
try:
config = Configuration.objects.get(key="review_rate_limit")
config.value = "4"
config.data_type = Configuration.DataType.NUMBER
config.save()
except Configuration.DoesNotExist:
Configuration.objects.create(
key="review_rate_limit",
value="4",
data_type=Configuration.DataType.NUMBER,
)
Configuration.objects.get_or_create(
key="review_rate_limit_popup_message",
defaults={
"value": "Test message",
"data_type": Configuration.DataType.HTML,
},
)
Configuration.objects.get_or_create(
key="review_rate_limit_popup_title",
defaults={
"value": "Test message",
"data_type": Configuration.DataType.HTML,
},
)
Configuration.objects.get_or_create(
key="review_rate_limit_banner_message",
defaults={
"value": "Test message",
"data_type": Configuration.DataType.HTML,
},
)
asset = create_asset()
t1 = create_transcription(user=anon, asset=asset)
t2 = create_transcription(
user=anon, asset=create_asset(item=asset.item, slug="test-asset-2")
)
t3 = create_transcription(
user=anon, asset=create_asset(item=asset.item, slug="test-asset-3")
)
t4 = create_transcription(
user=anon, asset=create_asset(item=asset.item, slug="test-asset-4")
)
t5 = create_transcription(
user=anon, asset=create_asset(item=asset.item, slug="test-asset-5")
)
resp = self.client.post(
reverse("review-transcription", args=(t1.pk,)), data={"action": "accept"}
)
self.assertValidJSON(resp, expected_status=200)
resp = self.client.post(
reverse("review-transcription", args=(t2.pk,)), data={"action": "accept"}
)
self.assertValidJSON(resp, expected_status=200)
resp = self.client.post(
reverse("review-transcription", args=(t3.pk,)), data={"action": "accept"}
)
self.assertValidJSON(resp, expected_status=200)
resp = self.client.post(
reverse("review-transcription", args=(t4.pk,)), data={"action": "accept"}
)
self.assertValidJSON(resp, expected_status=200)
resp = self.client.post(
reverse("review-transcription", args=(t5.pk,)), data={"action": "accept"}
)
data = self.assertValidJSON(resp, expected_status=429)
self.assertIn("error", data)
def test_transcription_review_rate_limit_superuser(self):
for cache in caches.all():
cache.clear()
anon = get_anonymous_user()
self.user = self.create_super_user()
self.login_user()
try:
config = Configuration.objects.get(key="review_rate_limit")
config.value = "4"
config.data_type = Configuration.DataType.NUMBER
config.save()
except Configuration.DoesNotExist:
Configuration.objects.create(
key="review_rate_limit",
value="4",
data_type=Configuration.DataType.NUMBER,
)
Configuration.objects.get_or_create(
key="review_rate_limit_popup_message",
defaults={
"value": "Test message",
"data_type": Configuration.DataType.HTML,
},
)
Configuration.objects.get_or_create(
key="review_rate_limit_popup_title",
defaults={
"value": "Test message",
"data_type": Configuration.DataType.HTML,
},
)
Configuration.objects.get_or_create(
key="review_rate_limit_banner_message",
defaults={
"value": "Test message",
"data_type": Configuration.DataType.HTML,
},
)
asset = create_asset()
t1 = create_transcription(user=anon, asset=asset)
t2 = create_transcription(
user=anon, asset=create_asset(item=asset.item, slug="test-asset-2")
)
t3 = create_transcription(
user=anon, asset=create_asset(item=asset.item, slug="test-asset-3")
)
t4 = create_transcription(
user=anon, asset=create_asset(item=asset.item, slug="test-asset-4")
)
t5 = create_transcription(
user=anon, asset=create_asset(item=asset.item, slug="test-asset-5")
)
resp = self.client.post(
reverse("review-transcription", args=(t1.pk,)), data={"action": "accept"}
)
self.assertValidJSON(resp, expected_status=200)
resp = self.client.post(
reverse("review-transcription", args=(t2.pk,)), data={"action": "accept"}
)
self.assertValidJSON(resp, expected_status=200)
resp = self.client.post(
reverse("review-transcription", args=(t3.pk,)), data={"action": "accept"}
)
self.assertValidJSON(resp, expected_status=200)
resp = self.client.post(
reverse("review-transcription", args=(t4.pk,)), data={"action": "accept"}
)
self.assertValidJSON(resp, expected_status=200)
resp = self.client.post(
reverse("review-transcription", args=(t5.pk,)), data={"action": "accept"}
)
self.assertValidJSON(resp, expected_status=200)
def test_transcription_review_asset_status_updates(self):
"""
Confirm that the Asset.transcription_status field is correctly updated
throughout the review process
"""
asset = create_asset()
anon = get_anonymous_user()
# We should see NOT_STARTED only when no transcription records exist:
self.assertEqual(asset.transcription_set.count(), 0)
self.assertEqual(
Asset.objects.get(pk=asset.pk).transcription_status,
TranscriptionStatus.NOT_STARTED,
)
t1 = Transcription(asset=asset, user=anon, text="test", submitted=now())
t1.full_clean()
t1.save()
self.assertEqual(
Asset.objects.get(pk=asset.pk).transcription_status,
TranscriptionStatus.SUBMITTED,
)
# “Login” so we can review the anonymous transcription:
self.login_user()
self.assertEqual(
1, Transcription.objects.filter(pk=t1.pk, accepted__isnull=True).count()
)
resp = self.client.post(
reverse("review-transcription", args=(t1.pk,)), data={"action": "reject"}
)
self.assertValidJSON(resp, expected_status=200)
# After rejecting a transcription, the asset status should be reset to
# in-progress:
self.assertEqual(
1,
Transcription.objects.filter(
pk=t1.pk, accepted__isnull=True, rejected__isnull=False
).count(),
)
self.assertEqual(
Asset.objects.get(pk=asset.pk).transcription_status,
TranscriptionStatus.IN_PROGRESS,
)
# We'll simulate a second attempt:
t2 = Transcription(
asset=asset, user=anon, text="test", submitted=now(), supersedes=t1
)
t2.full_clean()
t2.save()
self.assertEqual(
Asset.objects.get(pk=asset.pk).transcription_status,
TranscriptionStatus.SUBMITTED,
)
resp = self.client.post(
reverse("review-transcription", args=(t2.pk,)), data={"action": "accept"}
)
self.assertValidJSON(resp, expected_status=200)
self.assertEqual(
1, Transcription.objects.filter(pk=t2.pk, accepted__isnull=False).count()
)
self.assertEqual(
Asset.objects.get(pk=asset.pk).transcription_status,
TranscriptionStatus.COMPLETED,
)
def test_transcription_disallow_self_review(self):
asset = create_asset()
self.login_user()
t1 = Transcription(asset=asset, user=self.user, text="test", submitted=now())
t1.full_clean()
t1.save()
resp = self.client.post(
reverse("review-transcription", args=(t1.pk,)), data={"action": "accept"}
)
data = self.assertValidJSON(resp, expected_status=400)
self.assertIn("error", data)
self.assertEqual("You cannot accept your own transcription", data["error"])
def test_transcription_allow_self_reject(self):
asset = create_asset()
self.login_user()
t1 = Transcription(asset=asset, user=self.user, text="test", submitted=now())
t1.full_clean()
t1.save()
resp = self.client.post(
reverse("review-transcription", args=(t1.pk,)), data={"action": "reject"}
)
self.assertValidJSON(resp, expected_status=200)
self.assertEqual(
Asset.objects.get(pk=asset.pk).transcription_status,
TranscriptionStatus.IN_PROGRESS,
)
self.assertEqual(Transcription.objects.get(pk=t1.pk).reviewed_by, self.user)
def test_transcription_double_review(self):
asset = create_asset()
anon = get_anonymous_user()
t1 = Transcription(asset=asset, user=anon, text="test", submitted=now())
t1.full_clean()
t1.save()
self.login_user()
resp = self.client.post(
reverse("review-transcription", args=(t1.pk,)), data={"action": "accept"}
)
data = self.assertValidJSON(resp, expected_status=200)
resp = self.client.post(
reverse("review-transcription", args=(t1.pk,)), data={"action": "reject"}
)
data = self.assertValidJSON(resp, expected_status=400)
self.assertIn("error", data)
self.assertEqual("This transcription has already been reviewed", data["error"])
def tearDown(self):
# We'll test the signal handler separately
post_save.connect(on_transcription_save, sender=Transcription)
================================================
FILE: concordia/tests/test_views_transcription_save.py
================================================
import sys
from unittest.mock import patch
from django import forms
from django.db.models.signals import post_save
from django.test import (
TransactionTestCase,
override_settings,
)
from django.urls import reverse
from concordia.models import (
Transcription,
)
from concordia.signals.handlers import on_transcription_save
from .utils import (
CreateTestUsers,
JSONAssertMixin,
create_asset,
)
@override_settings(
RATELIMIT_ENABLE=False, SESSION_ENGINE="django.contrib.sessions.backends.cache"
)
class SaveTranscriptionViewTests(CreateTestUsers, JSONAssertMixin, TransactionTestCase):
def setUp(self):
self.asset = create_asset()
def test_turnstile_validation_fails(self):
# Test when Turnstile validation failes
with patch("concordia.turnstile.fields.TurnstileField.validate") as mock:
mock.side_effect = forms.ValidationError(
"Testing error", code="invalid_turnstile"
)
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "test"},
)
data = self.assertValidJSON(resp, expected_status=401)
self.assertIn("error", data)
def test_initial_save_success(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "test"},
)
data = self.assertValidJSON(resp, expected_status=201)
self.assertIn("submissionUrl", data)
def test_duplicate_without_supersedes_conflict(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "test"},
)
# Test attempts to create a second transcription without marking that it
# supersedes the previous one:
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "test"},
)
data = self.assertValidJSON(resp, expected_status=409)
self.assertIn("error", data)
def test_save_with_url_error(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "test"},
)
# If a transcription contains a URL, it should return an error
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={
"text": "http://example.com",
"supersedes": self.asset.transcription_set.get().pk,
},
)
data = self.assertValidJSON(resp, expected_status=400)
self.assertIn("error", data)
def test_unacceptable_characters_are_removed_on_save(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
bad_text = "He\u200bllo\tWorld\xa0\u3000\u2003!\nBad\x00Char\x1fHere\u200b"
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": bad_text},
)
data = self.assertValidJSON(resp, expected_status=201)
self.assertIn("submissionUrl", data)
t = self.asset.transcription_set.get()
self.assertEqual(t.text, "Hello\tWorld\xa0\u3000\u2003!\nBadCharHere")
def test_unacceptable_characters_are_removed_when_superseding(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "first"},
)
bad_text = "b\u200bad\x00"
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={
"text": bad_text,
"supersedes": self.asset.transcription_set.get().pk,
},
)
data = self.assertValidJSON(resp, expected_status=201)
self.assertIn("submissionUrl", data)
new_t = self.asset.transcription_set.order_by("pk").last()
self.assertEqual(new_t.text, "bad")
def test_save_with_supersedes_success(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "test"},
)
# Test that it correctly works when supersedes is set
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={
"text": "test",
"supersedes": self.asset.transcription_set.get().pk,
},
)
data = self.assertValidJSON(resp, expected_status=201)
self.assertIn("submissionUrl", data)
def test_supersedes_sets_ocr_originated_when_previous_was_ocr_originated(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "test"},
)
# Test that it correctly works when supersedes is set and confirm
# ocr_originaed is properly set
transcription = self.asset.transcription_set.order_by("pk").last()
transcription.ocr_originated = True
transcription.save()
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={
"text": "test",
"supersedes": self.asset.transcription_set.order_by("pk").last().pk,
},
)
data = self.assertValidJSON(resp, expected_status=201)
self.assertIn("submissionUrl", data)
new_transcription = self.asset.transcription_set.order_by("pk").last()
self.assertTrue(new_transcription.ocr_originated)
def test_supersede_already_superseded_conflict(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
first_resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "test"},
)
self.assertValidJSON(first_resp, expected_status=201)
first_pk = self.asset.transcription_set.order_by("pk").first().pk
self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "test 2", "supersedes": first_pk},
)
# We should see an error if you attempt to supersede a transcription
# which has already been superseded:
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={
"text": "test",
"supersedes": self.asset.transcription_set.order_by("pk")
.first()
.pk,
},
)
data = self.assertValidJSON(resp, expected_status=409)
self.assertIn("error", data)
def test_supersede_nonexistent_returns_400(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
# We should get an error if you attempt to supersede a transcription
# that doesn't exist
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={
"text": "test",
"supersedes": sys.maxsize,
},
)
data = self.assertValidJSON(resp, expected_status=400)
self.assertIn("error", data)
def test_supersede_invalid_pk_returns_400(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
# We should get an error if you attempt to supersede with
# with a pk that is invalid (i.e., a string instead of int)
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={
"text": "test",
"supersedes": "bad-pk",
},
)
data = self.assertValidJSON(resp, expected_status=400)
self.assertIn("error", data)
def test_logged_in_user_can_take_over_from_anonymous(self):
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
anon_resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={"text": "test"},
)
self.assertValidJSON(anon_resp, expected_status=201)
# A logged in user can take over from an anonymous user:
self.login_user()
resp = self.client.post(
reverse("save-transcription", args=(self.asset.pk,)),
data={
"text": "test",
"supersedes": self.asset.transcription_set.order_by("pk").last().pk,
},
)
data = self.assertValidJSON(resp, expected_status=201)
self.assertIn("submissionUrl", data)
def tearDown(self):
# We'll test the signal handler separately
post_save.connect(on_transcription_save, sender=Transcription)
================================================
FILE: concordia/tests/test_views_transcription_submit.py
================================================
from unittest.mock import patch
from django import forms
from django.db.models.signals import post_save
from django.test import (
TransactionTestCase,
override_settings,
)
from django.urls import reverse
from concordia.models import (
Transcription,
)
from concordia.signals.handlers import on_transcription_save
from concordia.utils import get_anonymous_user
from .utils import (
CreateTestUsers,
JSONAssertMixin,
create_asset,
)
@override_settings(
RATELIMIT_ENABLE=False, SESSION_ENGINE="django.contrib.sessions.backends.cache"
)
class SubmitTranscriptionViewTests(
CreateTestUsers, JSONAssertMixin, TransactionTestCase
):
def test_anonymous_transcription_submission(self):
asset = create_asset()
anon = get_anonymous_user()
transcription = Transcription(asset=asset, user=anon, text="previous entry")
transcription.full_clean()
transcription.save()
with patch("concordia.turnstile.fields.TurnstileField.validate") as mock:
mock.side_effect = forms.ValidationError(
"Testing error", code="invalid_turnstile"
)
resp = self.client.post(
reverse("submit-transcription", args=(transcription.pk,))
)
data = self.assertValidJSON(resp, expected_status=401)
self.assertIn("error", data)
self.assertFalse(Transcription.objects.filter(submitted__isnull=False).exists())
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
self.client.post(
reverse("submit-transcription", args=(transcription.pk,)),
)
self.assertTrue(
Transcription.objects.filter(submitted__isnull=False).exists()
)
def test_transcription_submission(self):
asset = create_asset()
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
resp = self.client.post(
reverse("save-transcription", args=(asset.pk,)), data={"text": "test"}
)
data = self.assertValidJSON(resp, expected_status=201)
transcription = Transcription.objects.get()
self.assertIsNone(transcription.submitted)
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
resp = self.client.post(
reverse("submit-transcription", args=(transcription.pk,))
)
data = self.assertValidJSON(resp, expected_status=200)
self.assertIn("id", data)
self.assertEqual(data["id"], transcription.pk)
transcription = Transcription.objects.get()
self.assertTrue(transcription.submitted)
def test_stale_transcription_submission(self):
asset = create_asset()
anon = get_anonymous_user()
t1 = Transcription(asset=asset, user=anon, text="test")
t1.full_clean()
t1.save()
t2 = Transcription(asset=asset, user=anon, text="test", supersedes=t1)
t2.full_clean()
t2.save()
with patch(
"concordia.turnstile.fields.TurnstileField.validate", return_value=True
):
resp = self.client.post(reverse("submit-transcription", args=(t1.pk,)))
data = self.assertValidJSON(resp, expected_status=400)
self.assertIn("error", data)
def tearDown(self):
# We'll test the signal handler separately
post_save.connect(on_transcription_save, sender=Transcription)
================================================
FILE: concordia/tests/test_views_utils.py
================================================
import datetime
from time import time
from django.contrib.auth.models import AnonymousUser
from django.test import RequestFactory, TestCase, override_settings
from django.utils.timezone import make_aware, now
from concordia.models import (
Asset,
Transcription,
TranscriptionStatus,
)
from concordia.views.utils import (
AnonymousUserValidationCheckMixin,
_get_pages,
annotate_children_with_progress_stats,
calculate_asset_stats,
)
from .utils import (
CreateTestUsers,
create_asset,
create_campaign,
create_item,
create_project,
create_transcription,
)
class GetPagesTests(CreateTestUsers, TestCase):
def setUp(self):
self.factory = RequestFactory()
self.user = self.create_test_user()
# Base campaign, project, item setup
self.campaign = create_campaign(slug="gp-camp", title="gp-camp")
self.project = create_project(
campaign=self.campaign, slug="gp-proj", title="gp-proj"
)
self.item = create_item(project=self.project, item_id="gp-item")
# Two assets in the same item
self.asset1 = create_asset(item=self.item, slug="gp-a1", sequence=1)
self.asset2 = create_asset(item=self.item, slug="gp-a2", sequence=2)
# Another campaign and project for campaign filtering tests
self.campaign2 = create_campaign(slug="gp-camp-2", title="gp-camp-2")
self.project2 = create_project(
campaign=self.campaign2, slug="gp-proj-2", title="gp-proj-2"
)
self.item2 = create_item(project=self.project2, item_id="gp-item-2")
self.asset3_other_campaign = create_asset(
item=self.item2, slug="gp-a3", sequence=1
)
def _request(self, params: dict[str, str]):
request = self.factory.get("/dummy", data=params)
request.user = self.user
return request
def _touch_transcription_times(
self,
transcription: Transcription,
*,
created_on=None,
updated_on=None,
reviewer=None,
):
if reviewer is not None:
transcription.reviewed_by = reviewer
if created_on is not None:
transcription.created_on = created_on
if updated_on is not None:
transcription.updated_on = updated_on
transcription.save(update_fields=["reviewed_by", "created_on", "updated_on"])
def test_activity_filters_transcribed_vs_reviewed_vs_default(self):
now_reference = now()
# asset1: transcribed by self.user
transcription1 = create_transcription(asset=self.asset1, user=self.user)
self._touch_transcription_times(
transcription1, created_on=now_reference - datetime.timedelta(hours=2)
)
# asset2: reviewed by self.user
transcription2 = create_transcription(
asset=self.asset2, user=self.create_test_user("transcriber")
)
self._touch_transcription_times(
transcription2,
reviewer=self.user,
updated_on=now_reference - datetime.timedelta(hours=1),
)
# Default behavior includes both
queryset_default = _get_pages(self._request({}))
self.assertCountEqual(
list(queryset_default.values_list("id", flat=True)),
[self.asset1.id, self.asset2.id],
)
# Transcribed only
queryset_transcribed = _get_pages(self._request({"activity": "transcribed"}))
self.assertListEqual(
list(queryset_transcribed.values_list("id", flat=True)),
[self.asset1.id],
)
# Reviewed only
queryset_reviewed = _get_pages(self._request({"activity": "reviewed"}))
self.assertListEqual(
list(queryset_reviewed.values_list("id", flat=True)),
[self.asset2.id],
)
def test_status_filter_exclusions(self):
# Ensure the user is associated with each asset via transcriptions
create_transcription(asset=self.asset1, user=self.user)
create_transcription(asset=self.asset2, user=self.user, submitted=now())
# Mark asset1 as IN_PROGRESS explicitly
Asset.objects.filter(pk=self.asset1.pk).update(
transcription_status=TranscriptionStatus.IN_PROGRESS
)
# Also add an asset in COMPLETED with a user transcription
completed_asset = create_asset(item=self.item, slug="gp-a4", sequence=3)
create_transcription(asset=completed_asset, user=self.user)
Asset.objects.filter(pk=completed_asset.pk).update(
transcription_status=TranscriptionStatus.COMPLETED
)
# Only "submitted" requested, so exclude IN_PROGRESS and COMPLETED
queryset = _get_pages(self._request({"status": "submitted"}))
self.assertListEqual(
list(queryset.values_list("id", flat=True)), [self.asset2.id]
)
def test_date_range_and_single_day_filters_and_ordering(self):
# Transcriptions (associate user) with distinct activity dates
today = now()
day_minus_3 = make_aware(
datetime.datetime.combine(
(today - datetime.timedelta(days=3)).date(), datetime.time(12)
)
)
day_minus_1 = make_aware(
datetime.datetime.combine(
(today - datetime.timedelta(days=1)).date(), datetime.time(12)
)
)
transcription1 = create_transcription(asset=self.asset1, user=self.user)
self._touch_transcription_times(
transcription1, created_on=day_minus_3, updated_on=day_minus_3
)
transcription2 = create_transcription(asset=self.asset2, user=self.user)
self._touch_transcription_times(
transcription2, created_on=day_minus_1, updated_on=day_minus_1
)
# The range filter from two days ago through today should include
# asset2 (day minus one) and exclude asset1 (day minus three)
start = (today - datetime.timedelta(days=2)).strftime("%Y-%m-%d")
end = today.strftime("%Y-%m-%d")
queryset_range = _get_pages(self._request({"start": start, "end": end}))
self.assertListEqual(
list(queryset_range.values_list("id", flat=True)), [self.asset2.id]
)
# A single-day filter for day minus three picks asset1
only_day = (today - datetime.timedelta(days=3)).strftime("%Y-%m-%d")
queryset_single = _get_pages(self._request({"start": only_day}))
self.assertListEqual(
list(queryset_single.values_list("id", flat=True)), [self.asset1.id]
)
# Ordering: ascending vs default (descending)
queryset_ascending = _get_pages(self._request({"order_by": "date-ascending"}))
self.assertEqual(
list(queryset_ascending.values_list("id", flat=True)),
[self.asset1.id, self.asset2.id],
)
queryset_descending = _get_pages(self._request({}))
self.assertEqual(
list(queryset_descending.values_list("id", flat=True)),
[self.asset2.id, self.asset1.id],
)
def test_campaign_filter_and_six_month_cutoff(self):
# Link user to assets in both campaigns
recent_timestamp = now() - datetime.timedelta(days=5)
old_timestamp = now() - datetime.timedelta(days=6 * 30 + 10)
# Asset in base campaign (recent)
transcription1 = create_transcription(asset=self.asset1, user=self.user)
self._touch_transcription_times(
transcription1, created_on=recent_timestamp, updated_on=recent_timestamp
)
# Asset in other campaign (recent)
transcription2 = create_transcription(
asset=self.asset3_other_campaign, user=self.user
)
self._touch_transcription_times(
transcription2, created_on=recent_timestamp, updated_on=recent_timestamp
)
# Very old activity on asset2 so it should be filtered out by the
# six months cutoff
transcription_old = create_transcription(asset=self.asset2, user=self.user)
self._touch_transcription_times(
transcription_old, created_on=old_timestamp, updated_on=old_timestamp
)
# Without a campaign filter, both recent assets are present
# and the old one is excluded
queryset = _get_pages(self._request({}))
asset_ids = set(queryset.values_list("id", flat=True))
self.assertSetEqual(asset_ids, {self.asset1.id, self.asset3_other_campaign.id})
# The campaign filter picks only the other campaign's asset
queryset_campaign2 = _get_pages(
self._request({"campaign": str(self.campaign2.pk)})
)
self.assertListEqual(
list(queryset_campaign2.values_list("id", flat=True)),
[self.asset3_other_campaign.id],
)
def test_status_filter_includes_completed_when_requested(self):
"""
When "completed" is requested, completed assets are kept while
submitted and in progress assets are excluded.
"""
# Prepare three assets that all have activity from this user.
completed_asset = create_asset(
item=self.item, slug="gp-a4-completed", sequence=4
)
create_transcription(asset=completed_asset, user=self.user)
Asset.objects.filter(pk=completed_asset.pk).update(
transcription_status=TranscriptionStatus.COMPLETED
)
create_transcription(asset=self.asset1, user=self.user)
Asset.objects.filter(pk=self.asset1.pk).update(
transcription_status=TranscriptionStatus.IN_PROGRESS
)
create_transcription(
asset=self.asset2, user=self.user, submitted=now()
) # submitted
# Request only "completed" status.
queryset = _get_pages(self._request({"status": "completed"}))
self.assertListEqual(
list(queryset.values_list("id", flat=True)), [completed_asset.id]
)
def test_status_filter_includes_in_progress_and_excludes_submitted_not_requested(
self,
):
"""
When "in_progress" is requested, in progress assets are kept and
submitted assets are excluded because "submitted" is not requested.
"""
# Prepare one in progress and one submitted asset with this user's activity.
create_transcription(asset=self.asset1, user=self.user)
Asset.objects.filter(pk=self.asset1.pk).update(
transcription_status=TranscriptionStatus.IN_PROGRESS
)
create_transcription(
asset=self.asset2, user=self.user, submitted=now()
) # submitted
# Request only "in_progress" status.
queryset = _get_pages(self._request({"status": "in_progress"}))
ids = list(queryset.values_list("id", flat=True))
self.assertIn(self.asset1.id, ids)
self.assertNotIn(self.asset2.id, ids)
class CalculateAssetStatsTests(CreateTestUsers, TestCase):
def setUp(self):
self.user = self.create_test_user()
self.campaign = create_campaign(slug="cas-c", title="cas-c")
self.project = create_project(
campaign=self.campaign, slug="cas-p", title="cas-p"
)
self.item = create_item(project=self.project, item_id="cas-i")
def test_counts_percents_and_contributors_remove_none_branch(self):
# Build a small asset set with varied statuses.
asset_not_started = create_asset(item=self.item, slug="cas-ns", sequence=1)
asset_in_progress = create_asset(item=self.item, slug="cas-ip", sequence=2)
asset_submitted = create_asset(item=self.item, slug="cas-sub", sequence=3)
# Set desired statuses directly.
Asset.objects.filter(pk=asset_not_started.pk).update(
transcription_status=TranscriptionStatus.NOT_STARTED
)
Asset.objects.filter(pk=asset_in_progress.pk).update(
transcription_status=TranscriptionStatus.IN_PROGRESS
)
Asset.objects.filter(pk=asset_submitted.pk).update(
transcription_status=TranscriptionStatus.SUBMITTED
)
# Create transcriptions ONLY for the assets that should not remain NOT_STARTED.
# For IN_PROGRESS, a plain transcription moves or keeps the asset in progress.
transcription_in_progress = create_transcription(
asset=asset_in_progress, user=self.user
)
# For SUBMITTED, mark the transcription as submitted so the
# signal preserves SUBMITTED.
transcription_submitted = create_transcription(
asset=asset_submitted, user=self.user, submitted=now()
)
# Ensure there is at least one None in reviewed_by so the remove(None)
# path is exercised.
Transcription.objects.filter(
pk__in=[transcription_in_progress.pk, transcription_submitted.pk]
).update(reviewed_by=None)
context = {}
calculate_asset_stats(
Asset.objects.filter(
pk__in=[asset_not_started.pk, asset_in_progress.pk, asset_submitted.pk]
),
context,
)
# contributor_count counts unique user_ids and reviewed_by values, minus None.
self.assertEqual(context["contributor_count"], 1)
# Counts per status.
self.assertEqual(context["not_started_count"], 1)
self.assertEqual(context["in_progress_count"], 1)
self.assertEqual(context["submitted_count"], 1)
# COMPLETED not present.
self.assertEqual(context.get("completed_count", 0), 0)
# Percentages should round sensibly for 1 out of 3.
self.assertEqual(context["not_started_percent"], round(100 * (1 / 3)))
self.assertEqual(context["in_progress_percent"], round(100 * (1 / 3)))
self.assertEqual(context["submitted_percent"], round(100 * (1 / 3)))
# Labeled list populated and includes "not_started".
self.assertTrue(
any(
status_key == "not_started"
for status_key, _, _ in context["transcription_status_counts"]
)
)
def test_contributors_keyerror_branch_and_cap_99(self):
# Create 100 assets and set 99 to NOT_STARTED and 1 to IN_PROGRESS.
assets = []
for i in range(1, 101):
a = create_asset(item=self.item, slug=f"cas-bulk-{i}", sequence=i)
assets.append(a)
Asset.objects.filter(pk__in=[a.pk for a in assets[:99]]).update(
transcription_status=TranscriptionStatus.NOT_STARTED
)
Asset.objects.filter(pk=assets[-1].pk).update(
transcription_status=TranscriptionStatus.IN_PROGRESS
)
# Create a transcription ONLY for the single IN_PROGRESS asset
# and set a reviewer. This ensures there is no None in reviewed_by,
# which triggers the KeyError branch when calculate_asset_stats
# attempts to remove(None) from the contributor set.
other_user = self.create_test_user(username="cas-reviewer")
transcription = create_transcription(asset=assets[-1], user=self.user)
transcription.reviewed_by = other_user
transcription.save(update_fields=["reviewed_by"])
context = {}
calculate_asset_stats(
Asset.objects.filter(pk__in=[a.pk for a in assets]), context
)
# Two distinct contributors: the creator (self.user) and
# the reviewer (other_user).
self.assertEqual(context["contributor_count"], 2)
# Verify percentages and that the 99 percent capping behavior is applied.
self.assertEqual(context["not_started_percent"], 99)
self.assertEqual(context["in_progress_percent"], 1)
self.assertEqual(context.get("submitted_percent", 0), 0)
self.assertEqual(context.get("completed_percent", 0), 0)
# Also verify counts to ensure the underlying distribution is as intended.
self.assertEqual(context["not_started_count"], 99)
self.assertEqual(context["in_progress_count"], 1)
self.assertEqual(context.get("submitted_count", 0), 0)
self.assertEqual(context.get("completed_count", 0), 0)
class AnnotateChildrenProgressStatsTests(TestCase):
class Obj:
pass
def test_progress_stats_with_capping_and_lowest_status(self):
obj = self.Obj()
# Construct counts such that one bucket yields at least ninety nine
# but less than one hundred percent
obj.not_started_count = 99
obj.in_progress_count = 1
obj.submitted_count = 0
obj.completed_count = 0
annotate_children_with_progress_stats([obj])
# Total
self.assertEqual(obj.total_count, 100)
# Capping at ninety nine
self.assertEqual(obj.not_started_percent, 99)
# Others
self.assertEqual(obj.in_progress_percent, 1)
self.assertEqual(obj.submitted_percent, 0)
self.assertEqual(obj.completed_percent, 0)
# Lowest is the first non-zero by CHOICES order; expect "not_started"
self.assertEqual(obj.lowest_transcription_status, "not_started")
def test_progress_stats_zero_total(self):
obj = self.Obj()
obj.not_started_count = 0
obj.in_progress_count = 0
obj.submitted_count = 0
obj.completed_count = 0
annotate_children_with_progress_stats([obj])
self.assertEqual(obj.total_count, 0)
self.assertEqual(obj.not_started_percent, 0)
self.assertEqual(obj.in_progress_percent, 0)
self.assertEqual(obj.submitted_percent, 0)
self.assertEqual(obj.completed_percent, 0)
self.assertIsNone(obj.lowest_transcription_status)
class _BaseView:
"""
Minimal base class that provides get_context_data so the mixin can call super().
"""
def get_context_data(self, **kwargs):
return {}
class DummyTemplateView(AnonymousUserValidationCheckMixin, _BaseView):
"""
Stand-in view. The mixin is first in the MRO so its get_context_data runs,
then it calls super() which resolves to _BaseView.get_context_data.
"""
pass
class AnonymousUserValidationCheckMixinTests(CreateTestUsers, TestCase):
def setUp(self):
self.factory = RequestFactory()
self.user = self.create_test_user()
def _attach_session(self, request):
# Attach a session dictionary-like attribute without middleware dependency
request.session = {}
return request
@override_settings(ANONYMOUS_USER_VALIDATION_INTERVAL=10)
def test_unauthenticated_requires_validation_when_stale(self):
request = self.factory.get("/dummy")
request.user = AnonymousUser()
self._attach_session(request)
# There is no prior validation so the default timestamp is zero
# and the validation is stale
view = DummyTemplateView()
view.request = request
context = view.get_context_data()
self.assertTrue(context["anonymous_user_validation_required"])
@override_settings(ANONYMOUS_USER_VALIDATION_INTERVAL=10)
def test_unauthenticated_recent_validation_is_not_required(self):
request = self.factory.get("/dummy")
request.user = AnonymousUser()
self._attach_session(request)
request.session["turnstile_last_validated"] = int(time())
view = DummyTemplateView()
view.request = request
context = view.get_context_data()
self.assertFalse(context["anonymous_user_validation_required"])
@override_settings(ANONYMOUS_USER_VALIDATION_INTERVAL=10)
def test_authenticated_never_requires_validation(self):
request = self.factory.get("/dummy")
request.user = self.user
self._attach_session(request)
view = DummyTemplateView()
view.request = request
context = view.get_context_data()
self.assertFalse(context["anonymous_user_validation_required"])
================================================
FILE: concordia/tests/test_widgets.py
================================================
from django.test import TestCase, override_settings
from concordia.turnstile.widgets import TurnstileWidget
from concordia.widgets import EmailWidget
class TestWidgets(TestCase):
def test_EmailWidget(self):
widget = EmailWidget()
output = widget.render("email", None)
self.assertHTMLEqual(
output, '/reviewable/",
views.campaigns.FilteredCampaignDetailView.as_view(),
name="filtered-campaign-detail",
),
path(
"/",
views.campaigns.CampaignDetailView.as_view(),
name="campaign-detail",
),
path(
"/export/csv/",
exporter_views.ExportCampaignToCSV.as_view(),
name="campaign-export-csv",
),
path(
"/export/bagit/",
exporter_views.ExportCampaignToBagIt.as_view(),
name="campaign-export-bagit",
),
path(
"//export/bagit/",
exporter_views.ExportProjectToBagIt.as_view(),
name="project-export-bagit",
),
path(
(
"//"
"/export/bagit/"
),
exporter_views.ExportItemToBagIt.as_view(),
name="item-export-bagit",
),
path(
"/report/",
views.campaigns.ReportCampaignView.as_view(),
name="campaign-report",
),
path(
"///reviewable/",
views.items.FilteredItemDetailView.as_view(),
name="filtered-item-detail",
),
path(
(
"//"
"//"
),
views.assets.AssetDetailView.as_view(),
name="asset-detail",
),
# n.b. this must be above project-detail to avoid being seen as a project slug:
path(
"/next-transcribable-asset/",
views.assets.redirect_to_next_transcribable_campaign_asset,
name="redirect-to-next-transcribable-campaign-asset",
),
path(
"/next-reviewable-asset/",
views.assets.redirect_to_next_reviewable_campaign_asset,
name="redirect-to-next-reviewable-campaign-asset",
),
path(
"//reviewable/",
views.projects.FilteredProjectDetailView.as_view(),
name="filtered-project-detail",
),
path(
"//",
views.projects.ProjectDetailView.as_view(),
name="project-detail",
),
path(
"///",
views.items.ItemDetailView.as_view(),
name="item-detail",
),
],
"transcriptions",
)
urlpatterns = [
path("", views.HomeView.as_view(), name="homepage"),
path("healthz", views.healthz, name="health-check"),
path("letter", views.accounts.account_letter, name="user-letter"),
path("about/", views.simple_pages.about_simple_page, name="about"),
# These patterns are to make sure various links to help-center URLs don't break
# when the URLs are changed to not include help-center and can be removed after
# all links are updated.
path(
"help-center/",
RedirectView.as_view(pattern_name="welcome-guide"),
name="help-center",
),
path(
"help-center/welcome-guide/", RedirectView.as_view(pattern_name="welcome-guide")
),
path(
"help-center/welcome-guide-esp/",
RedirectView.as_view(pattern_name="welcome-guide-spanish"),
),
path(
"help-center/-esp/",
views.simple_pages.HelpCenterSpanishRedirectView.as_view(),
),
path(
"help-center//",
views.simple_pages.HelpCenterRedirectView.as_view(),
),
# End of help-center patterns
path("get-started/", views.simple_pages.simple_page, name="welcome-guide"),
path(
"get-started/how-to-transcribe/",
views.simple_pages.simple_page,
name="transcription-basic-rules",
),
path(
"get-started/how-to-review/",
views.simple_pages.simple_page,
name="how-to-review",
),
path("get-started/how-to-tag/", views.simple_pages.simple_page, name="how-to-tag"),
path(
"get-started//", views.simple_pages.simple_page, name="simple-page"
),
path(
"get-started-esp/",
views.simple_pages.simple_page,
name="welcome-guide-spanish",
),
path(
"get-started-esp/how-to-transcribe-esp/",
views.simple_pages.simple_page,
name="how-to-transcribe-spanish",
),
path(
"get-started-esp/how-to-review-esp/",
views.simple_pages.simple_page,
name="how-to-review-spanish",
),
path(
"get-started-esp/how-to-tag-esp/",
views.simple_pages.simple_page,
name="how-to-tag-spanish",
),
path(
"get-started-esp//",
views.simple_pages.simple_page,
name="simple-page-spanish",
),
path(
"for-educators/",
views.simple_pages.simple_page,
name="for-educators",
),
path(
"for-staff/",
views.simple_pages.simple_page,
name="for-staff",
),
path(
"resources/",
RedirectView.as_view(
pattern_name="guidelines", permanent=True, query_string=True
),
name="resources",
),
path(
"service/",
views.simple_pages.simple_page,
name="service",
),
path(
"guidelines/",
views.simple_pages.simple_page,
name="guidelines",
),
path(
"programs/",
views.simple_pages.simple_page,
name="programs",
),
path(
"latest/",
RedirectView.as_view(pattern_name="about", permanent=True, query_string=True),
),
path("questions/", views.simple_pages.simple_page, name="questions"),
path(
"contact/",
RedirectView.as_view(url="https://ask.loc.gov/crowd"),
name="contact",
),
path(
"help-center/",
RedirectView.as_view(pattern_name="welcome-guide"),
name="help-center",
),
path(
"campaigns-topics/",
views.campaigns.CampaignTopicListView.as_view(),
name="campaign-topic-list",
),
path(
"topics//",
views.topics.TopicDetailView.as_view(),
name="topic-detail",
),
path(
"topics//next-transcribable-asset/",
views.assets.redirect_to_next_transcribable_topic_asset,
name="redirect-to-next-transcribable-topic-asset",
),
path(
"topics//next-reviewable-asset/",
views.assets.redirect_to_next_reviewable_topic_asset,
name="redirect-to-next-reviewable-topic-asset",
),
path(
"next-transcribable-asset/",
views.assets.redirect_to_next_transcribable_asset,
name="redirect-to-next-transcribable-asset",
),
path(
"next-reviewable-asset/",
views.assets.redirect_to_next_reviewable_asset,
name="redirect-to-next-reviewable-asset",
),
path("campaigns/", include(tx_urlpatterns, namespace="transcriptions")),
path(
"reserve-asset//", views.ajax.reserve_asset, name="reserve-asset"
),
path(
"assets//transcriptions/save/",
views.ajax.save_transcription,
name="save-transcription",
),
path(
"transcriptions//submit/",
views.ajax.submit_transcription,
name="submit-transcription",
),
path(
"transcriptions//review/",
views.ajax.review_transcription,
name="review-transcription",
),
path(
"assets//transcriptions/generate-ocr/",
views.ajax.generate_ocr_transcription,
name="generate-ocr-transcription",
),
path(
"assets//transcriptions/rollback/",
views.ajax.rollback_transcription,
name="rollback-transcription",
),
path(
"assets//transcriptions/rollforward/",
views.ajax.rollforward_transcription,
name="rollforward-transcription",
),
path(
"assets//tags/submit/", views.ajax.submit_tags, name="submit-tags"
),
path(
"account/ajax-status/",
views.ajax.ajax_session_status,
name="ajax-session-status",
),
path("account/ajax-messages/", views.ajax.ajax_messages, name="ajax-messages"),
path(
"account/register/",
views.accounts.ConcordiaRegistrationView.as_view(),
name="registration_register",
),
path(
"account/login/",
views.accounts.ConcordiaLoginView.as_view(),
name="registration_login",
),
path("account/get_pages/", views.accounts.get_pages, name="get_pages"),
path(
"account/profile/",
views.accounts.AccountProfileView.as_view(),
name="user-profile",
),
path(
"account/password_reset/",
views.accounts.ConcordiaPasswordResetRequestView.as_view(),
name="password_reset",
),
path(
"account/reset///",
views.accounts.ConcordiaPasswordResetConfirmView.as_view(),
name="password_reset_confirm",
),
path("account/", include("django_registration.backends.activation.urls")),
path("account/", include("django.contrib.auth.urls")),
path(
"account/email_confirmation//",
views.accounts.EmailReconfirmationView.as_view(),
name="email-reconfirmation",
),
path(
"account/delete/",
views.accounts.AccountDeletionView.as_view(),
name="account-deletion",
),
path(
".well-known/change-password", # https://wicg.github.io/change-password-url/
RedirectView.as_view(pattern_name="password_change"),
),
path("admin/", admin.site.urls),
# Internal support assists:
path("error/500/", server_error),
path("error/404/", page_not_found, {"exception": Http404()}),
path("error/429/", views.rate_limit.ratelimit_view),
path("error/403/", permission_denied, {"exception": HttpResponseForbidden()}),
path("tinymce/", include("tinymce.urls")),
path("metrics", MetricsView.as_view(), name="prometheus-django-metrics"),
path("robots.txt", include("robots.urls")),
path(
"maintenance-mode/off/",
views.maintenance_mode.maintenance_mode_off,
name="maintenance_mode_off",
),
path(
"maintenance-mode/on/",
views.maintenance_mode.maintenance_mode_on,
name="maintenance_mode_on",
),
path(
"maintenance-mode/frontend/available",
views.maintenance_mode.maintenance_mode_frontend_available,
name="maintenance_mode_frontend_available",
),
path(
"maintenance-mode/frontend/unavailable",
views.maintenance_mode.maintenance_mode_frontend_unavailable,
name="maintenance_mode_frontend_unavailable",
),
path(
"api/visualization//",
views.visualizations.VisualizationDataView.as_view(),
name="visualization",
),
]
if settings.DEBUG:
import debug_toolbar
from django.conf.urls.static import static
from django.views.generic import TemplateView
from concordia.api import api as concordia_api
urlpatterns = [path("__debug__/", include(debug_toolbar.urls))] + urlpatterns
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
urlpatterns += (
path(
"transcription/",
TemplateView.as_view(template_name="transcriptions/transcription.html"),
name="transcription",
),
path("api/", concordia_api.urls, name="api"),
)
================================================
FILE: concordia/utils/__init__.py
================================================
from secrets import token_hex
from django.contrib.auth.models import User
from concordia.logging import ConcordiaLogger
from concordia.templatetags.concordia_media_tags import asset_media_url
__all__ = [
"get_anonymous_user",
"request_accepts_json",
"get_or_create_reservation_token",
"get_image_urls_from_asset",
]
structured_logger = ConcordiaLogger.get_logger(__name__)
def get_anonymous_user():
"""
Get the user called "anonymous" if it exist. Create the user if it doesn't
exist This is the default concordia user if someone is working on the site
without logging in first.
"""
try:
return User.objects.get(username="anonymous")
except User.DoesNotExist:
return User.objects.create_user(username="anonymous")
def request_accepts_json(request):
accept_header = request.headers.get("Accept", "*/*")
return "application/json" in accept_header
def get_or_create_reservation_token(request):
# Reservation tokens are 44 characters (22 bytes
# converted into 44 hex digits) plus the user's
# database id padded with leading zeroes until it's
# at least 6 characters long
if "reservation_token" not in request.session:
request.session["reservation_token"] = token_hex(22)
user = getattr(request, "user", None)
if user is not None:
uid = user.id
if uid is None:
uid = get_anonymous_user().id
request.session["reservation_token"] += str(uid).zfill(6)
structured_logger.info(
"Reservation token created.",
event_code="reservation_token_created",
reservation_token=request.session["reservation_token"],
user=user,
)
else:
structured_logger.info(
"Reservation token reused.",
event_code="reservation_token_reused",
reservation_token=request.session["reservation_token"],
)
return request.session["reservation_token"]
def get_image_urls_from_asset(asset):
"""
Given an Asset, return a tuple containing the normalized full-size and
thumbnail-size image URLs
"""
image_url = asset_media_url(asset)
if asset.download_url and "iiif" in asset.download_url:
thumbnail_url = asset.download_url.replace(
"http://tile.loc.gov", "https://tile.loc.gov"
)
else:
thumbnail_url = image_url
return image_url, thumbnail_url
================================================
FILE: concordia/utils/celery.py
================================================
from celery import Task
from concordia.celery import app as concordia_celery_app
def get_registered_task(name: str) -> Task:
"""
Retrieve a Celery task by its fully qualified name.
This function looks up a task in the Celery app task registry. It raises a
RuntimeError if the task is not found. The purpose of this function is to
provide a usable interface for safely calling a task without importing it
directly, to avoid issues such as circular imports. This avoids issues with
`app.send_task`, which ignores settings such as `ALWAYS_EAGER`.
Args:
name (str): Fully qualified task name, for example
"myapp.tasks.my_task".
Returns:
Task: The registered Celery task object.
Raises:
RuntimeError: If the task name is not found in the registry.
"""
try:
return concordia_celery_app.tasks[name]
except KeyError as err:
raise RuntimeError(f"Task {name} is not registered. Did you typo it?") from err
================================================
FILE: concordia/utils/constants.py
================================================
from django.contrib import messages
ASSETS_PER_PAGE = 36
PROJECTS_PER_PAGE = 36
ITEMS_PER_PAGE = 36
URL_REGEX = r"http[s]?://"
MESSAGE_LEVEL_NAMES = dict(
zip(
messages.DEFAULT_LEVELS.values(),
map(str.lower, messages.DEFAULT_LEVELS.keys()),
strict=False,
)
)
================================================
FILE: concordia/utils/next_asset/__init__.py
================================================
from concordia.logging import ConcordiaLogger
from concordia.models import (
NextReviewableCampaignAsset,
NextReviewableTopicAsset,
NextTranscribableCampaignAsset,
NextTranscribableTopicAsset,
)
from .reviewable import (
find_and_order_potential_reviewable_campaign_assets,
find_and_order_potential_reviewable_topic_assets,
find_invalid_next_reviewable_campaign_assets,
find_invalid_next_reviewable_topic_assets,
find_new_reviewable_campaign_assets,
find_new_reviewable_topic_assets,
find_next_reviewable_campaign_asset,
find_next_reviewable_campaign_assets,
find_next_reviewable_topic_asset,
find_next_reviewable_topic_assets,
find_reviewable_campaign_asset,
find_reviewable_topic_asset,
)
from .transcribable import (
find_and_order_potential_transcribable_campaign_assets,
find_and_order_potential_transcribable_topic_assets,
find_invalid_next_transcribable_campaign_assets,
find_invalid_next_transcribable_topic_assets,
find_new_transcribable_campaign_assets,
find_new_transcribable_topic_assets,
find_next_transcribable_campaign_asset,
find_next_transcribable_campaign_assets,
find_next_transcribable_topic_asset,
find_next_transcribable_topic_assets,
find_transcribable_campaign_asset,
find_transcribable_topic_asset,
)
__all__ = [
"find_and_order_potential_transcribable_campaign_assets",
"find_and_order_potential_transcribable_topic_assets",
"find_new_transcribable_campaign_assets",
"find_new_transcribable_topic_assets",
"find_next_transcribable_campaign_asset",
"find_next_transcribable_topic_asset",
"find_next_transcribable_campaign_assets",
"find_next_transcribable_topic_assets",
"find_transcribable_campaign_asset",
"find_transcribable_topic_asset",
"find_and_order_potential_reviewable_campaign_assets",
"find_and_order_potential_reviewable_topic_assets",
"find_new_reviewable_campaign_assets",
"find_new_reviewable_topic_assets",
"find_next_reviewable_campaign_assets",
"find_next_reviewable_topic_assets",
"find_next_reviewable_campaign_asset",
"find_next_reviewable_topic_asset",
"find_reviewable_campaign_asset",
"find_reviewable_topic_asset",
"remove_next_asset_objects",
"find_invalid_next_reviewable_campaign_assets",
"find_invalid_next_reviewable_topic_assets",
"find_invalid_next_transcribable_campaign_assets",
"find_invalid_next_transcribable_topic_assets",
]
structured_logger = ConcordiaLogger.get_logger(__name__)
def remove_next_asset_objects(asset_id):
"""
Remove all cached next asset entries associated with the given asset id.
This function deletes entries from the four next asset tables:
- NextTranscribableCampaignAsset
- NextTranscribableTopicAsset
- NextReviewableCampaignAsset
- NextReviewableTopicAsset
It is typically used when an asset is no longer valid for caching
(e.g., after transcription or review status changes, or when a user
reserves the asset).
Args:
asset_id (int): The ID of the asset to remove from next-asset tables.
"""
structured_logger.info(
"Removing next asset objects",
event_code="remove_next_asset_objects",
asset_id=asset_id,
)
NextTranscribableCampaignAsset.objects.filter(asset_id=asset_id).delete()
NextTranscribableTopicAsset.objects.filter(asset_id=asset_id).delete()
NextReviewableCampaignAsset.objects.filter(asset_id=asset_id).delete()
NextReviewableTopicAsset.objects.filter(asset_id=asset_id).delete()
================================================
FILE: concordia/utils/next_asset/reviewable/__init__.py
================================================
from .campaign import (
find_and_order_potential_reviewable_campaign_assets,
find_invalid_next_reviewable_campaign_assets,
find_new_reviewable_campaign_assets,
find_next_reviewable_campaign_asset,
find_next_reviewable_campaign_assets,
find_reviewable_campaign_asset,
)
from .topic import (
find_and_order_potential_reviewable_topic_assets,
find_invalid_next_reviewable_topic_assets,
find_new_reviewable_topic_assets,
find_next_reviewable_topic_asset,
find_next_reviewable_topic_assets,
find_reviewable_topic_asset,
)
__all__ = [
"find_new_reviewable_campaign_assets",
"find_next_reviewable_campaign_assets",
"find_reviewable_campaign_asset",
"find_and_order_potential_reviewable_campaign_assets",
"find_next_reviewable_campaign_asset",
"find_and_order_potential_reviewable_topic_assets",
"find_new_reviewable_topic_assets",
"find_next_reviewable_topic_asset",
"find_next_reviewable_topic_assets",
"find_reviewable_topic_asset",
"find_invalid_next_reviewable_campaign_assets",
"find_invalid_next_reviewable_topic_assets",
]
================================================
FILE: concordia/utils/next_asset/reviewable/campaign.py
================================================
from typing import Dict
from django.contrib.auth.models import User
from django.db import transaction
from django.db.models import Case, IntegerField, Q, QuerySet, Subquery, Value, When
from concordia import models as concordia_models
from concordia.logging import ConcordiaLogger
from concordia.utils.celery import get_registered_task
structured_logger = ConcordiaLogger.get_logger(__name__)
def _reserved_asset_ids_subq(
campaign: concordia_models.Campaign,
) -> "QuerySet[Dict[str, int]]":
"""
Return a subquery of reserved asset identifiers for a campaign.
Behavior:
Produces a subquery suitable for use with `Subquery(...)` and
`exclude(pk__in=...)` clauses to filter out assets that currently have
an active reservation.
Args:
campaign (concordia_models.Campaign): Campaign whose reserved
assets should be excluded.
Returns:
QuerySet[Dict[str, int]]: A queryset of dictionaries with a single key
"asset_id" corresponding to reserved assets.
"""
return concordia_models.AssetTranscriptionReservation.objects.filter(
asset__campaign=campaign
).values("asset_id")
def _eligible_reviewable_base_qs(
campaign: concordia_models.Campaign,
user: User | None = None,
) -> "QuerySet[concordia_models.Asset]":
"""
Build the base queryset of reviewable assets for a campaign.
Behavior:
Restricts to published projects, items, and assets, and to assets whose
transcription status is `SUBMITTED`. Optionally excludes assets
transcribed by the supplied user.
Args:
campaign (concordia_models.Campaign): Campaign scope for filtering.
user (User | None): If provided, exclude assets transcribed by this user.
Returns:
QuerySet[concordia_models.Asset]: Reviewable assets, with `item` and
`item__project` selected via `select_related`.
"""
qs = concordia_models.Asset.objects.filter(
campaign_id=campaign.id,
item__project__published=True,
item__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,
).select_related("item", "item__project")
if user:
qs = qs.exclude(transcription__user=user.id)
return qs
def _next_seq_after(pk: int | None) -> int | None:
"""
Resolve the sequence number for a given asset primary key.
Behavior:
Convenience utility for ordering logic when advancing within a series
of assets.
Args:
pk (int | None): Asset primary key whose sequence to resolve.
Returns:
int | None: The asset's sequence number, or None if `pk` is falsy
or the asset does not exist.
"""
if not pk:
return None
return (
concordia_models.Asset.objects.filter(pk=pk)
.values_list("sequence", flat=True)
.first()
)
@transaction.atomic
def _find_reviewable_in_item(
campaign: concordia_models.Campaign,
user: User,
*,
item_id: str,
after_asset_pk: int | None,
) -> "concordia_models.Asset | None":
"""
Select the next reviewable asset within the same item.
Behavior:
Attempts a short-circuit within the user's current item to provide a
locally contiguous review flow.
Eligibility:
- Asset, Item, and Project are published.
- Asset transcription status is `SUBMITTED`.
- Asset is not reserved.
- Asset was not transcribed by the current user.
Ordering:
- If `after_asset_pk` refers to an asset in the same item and campaign,
select the earliest asset whose (sequence, id) is strictly greater
than the current asset's pair.
- Otherwise, select the earliest eligible by (sequence, id).
Args:
campaign (concordia_models.Campaign): Campaign scope.
user (User): Current user; used to exclude their own work.
item_id (str): Identifier of the item to stay within.
after_asset_pk (int | None): Asset primary key to advance from.
Returns:
concordia_models.Asset | None: A locked eligible asset, or
None if no match is available.
"""
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(
asset__item__item_id=item_id,
asset__item__project__campaign=campaign,
).values("asset_id")
eligible = (
concordia_models.Asset.objects.filter(
item__item_id=item_id,
item__project__campaign=campaign,
item__project__published=True,
item__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,
)
.exclude(pk__in=Subquery(reserved_asset_ids))
.exclude(transcription__user=user.id)
)
seq_gt_filter = None
if after_asset_pk is not None:
try:
current = (
concordia_models.Asset.objects.only("id", "sequence", "item_id", "item")
.select_related("item")
.get(pk=after_asset_pk)
)
if (
current.item.item_id == item_id
and current.item.project.campaign_id == campaign.id
):
seq_gt_filter = Q(sequence__gt=current.sequence) | (
Q(sequence=current.sequence) & Q(id__gt=after_asset_pk)
)
except concordia_models.Asset.DoesNotExist:
pass
if seq_gt_filter is not None:
eligible = eligible.filter(seq_gt_filter)
asset = (
eligible.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.order_by("sequence", "id")
.first()
)
structured_logger.debug(
"Item short-circuit (campaign reviewable) resolved.",
event_code="reviewable_item_short_circuit_campaign",
campaign=campaign,
item_id=item_id,
after_asset_pk=after_asset_pk,
chosen_asset_id=getattr(asset, "id", None),
)
return asset
@transaction.atomic
def _find_reviewable_in_project(
campaign: concordia_models.Campaign,
user: User,
*,
project_slug: str,
after_asset_pk: int | None,
) -> "concordia_models.Asset | None":
"""
Select the first eligible reviewable asset within the same project.
Behavior:
Short-circuit when staying within a project. Sequence is per item,
so this returns the first eligible asset, not strictly "after" a given asset.
Eligibility:
- Same campaign and project.
- Asset, Item, and Project are published.
- Asset transcription status is `SUBMITTED`.
- Asset is not reserved.
- Asset was not transcribed by the current user.
Ordering:
Deterministic by (item__item_id, sequence, id).
Args:
campaign (concordia_models.Campaign): Campaign scope.
user (User): Current user; used to exclude their own work.
project_slug (str): Slug of the project to stay within.
after_asset_pk (int | None): Present for parity with the item
variant; not used for ordering here.
Returns:
concordia_models.Asset | None: A locked eligible asset, or
None if no match is available.
"""
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(
asset__item__project__slug=project_slug,
asset__item__project__campaign=campaign,
).values("asset_id")
eligible = (
concordia_models.Asset.objects.filter(
item__project__campaign=campaign,
item__project__slug=project_slug,
item__project__published=True,
item__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,
)
.exclude(pk__in=Subquery(reserved_asset_ids))
.exclude(transcription__user=user.id)
.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.order_by("item__item_id", "sequence", "id")
.first()
)
structured_logger.debug(
"Project short-circuit (campaign reviewable) resolved.",
event_code="reviewable_project_short_circuit_campaign",
campaign=campaign,
project_slug=project_slug,
after_asset_pk=after_asset_pk,
chosen_asset_id=getattr(eligible, "id", None),
)
return eligible
def find_new_reviewable_campaign_assets(
campaign: concordia_models.Campaign,
user: User | None = None,
) -> "QuerySet[concordia_models.Asset]":
"""
Return assets in a campaign that are eligible to be added to the cache.
Behavior:
Builds the candidate set for the NextReviewableCampaignAsset cache by
excluding assets that are not `SUBMITTED`, assets already reserved, and
assets already present in the cache. Optionally excludes assets
transcribed by the provided user.
Args:
campaign (concordia_models.Campaign): Campaign to filter by.
user (User | None): If provided, exclude assets transcribed by this user.
Returns:
QuerySet[concordia_models.Asset]: Eligible assets ordered by sequence.
"""
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(
asset__campaign=campaign
).values("asset_id")
next_asset_ids = concordia_models.NextReviewableCampaignAsset.objects.filter(
campaign=campaign
).values("asset_id")
queryset = (
concordia_models.Asset.objects.filter(
campaign_id=campaign.id,
item__project__published=True,
item__published=True,
published=True,
)
.filter(transcription_status=concordia_models.TranscriptionStatus.SUBMITTED)
.exclude(pk__in=Subquery(reserved_asset_ids))
.exclude(pk__in=Subquery(next_asset_ids))
.order_by("sequence")
)
if user:
queryset = queryset.exclude(transcription__user=user.id)
return queryset
def find_next_reviewable_campaign_assets(
campaign: concordia_models.Campaign,
user: User,
) -> "QuerySet[concordia_models.NextReviewableCampaignAsset]":
"""
Return cached reviewable assets in a campaign not transcribed by the user.
Behavior:
Reads from the NextReviewableCampaignAsset cache table and filters out
assets where the requesting user appears in transcriber_ids.
Args:
campaign (concordia_models.Campaign): Campaign to retrieve cached assets from.
user (User): Requesting user.
Returns:
QuerySet[concordia_models.NextReviewableCampaignAsset]: Cached candidate rows
for the given user.
"""
return concordia_models.NextReviewableCampaignAsset.objects.filter(
campaign=campaign
).exclude(transcriber_ids__contains=[user.id])
@transaction.atomic
def find_reviewable_campaign_asset(
campaign: concordia_models.Campaign,
user: User,
) -> "concordia_models.Asset | None":
"""
Retrieve a single reviewable asset for a user from a campaign.
Behavior:
First attempts to select a cached asset from NextReviewableCampaignAsset.
If none is available, falls back to a direct query over Asset and
triggers a background task to replenish the cache.
Concurrency:
Uses `select_for_update(skip_locked=True, of=("self",))` so only the
Asset row is locked and concurrent consumers skip locked rows.
Args:
campaign (concordia_models.Campaign): Campaign to search within.
user (User): Requesting user; their own transcriptions are excluded.
Returns:
concordia_models.Asset | None: A locked eligible asset, or None if unavailable.
"""
next_asset = (
find_next_reviewable_campaign_assets(campaign, user)
.select_for_update(skip_locked=True, of=("self",))
.values_list("asset_id", flat=True)
.first()
)
spawn_task = False
if next_asset:
asset_query = concordia_models.Asset.objects.filter(id=next_asset)
else:
# No asset in the NextReviewableCampaignAsset table for this campaign
# and user, so fallback to manually finding one
structured_logger.debug(
"No cached assets available, falling back to manual lookup",
event_code="reviewable_fallback_manual_lookup",
campaign=campaign,
user=user,
)
spawn_task = True
asset_query = find_new_reviewable_campaign_assets(campaign, user)
# select_for_update(of=("self",)) causes the row locking only to
# apply to the Asset table, rather than also locking joined item table
asset = (
asset_query.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if spawn_task:
# Spawn a task to populate the table for this campaign
# We wait to do this until after getting an asset because otherwise there's
# a chance all valid assets get grabbed by the task and our query will return
# nothing
structured_logger.debug(
"Spawned background task to populate cache",
event_code="reviewable_cache_population_triggered",
campaign=campaign,
user=user,
)
populate_task = get_registered_task(
"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_campaign"
)
populate_task.delay(campaign.id)
return asset
def find_and_order_potential_reviewable_campaign_assets(
campaign: concordia_models.Campaign,
user: User,
project_slug: str,
item_id: str,
asset_pk: int | None,
) -> "QuerySet[concordia_models.NextReviewableCampaignAsset]":
"""
Retrieve and prioritize cached reviewable assets for proximity.
Behavior:
Orders cached candidates from NextReviewableCampaignAsset to prefer
continuity with the user's current location.
Annotations added to each row (transient fields):
- next_asset (int): 1 if the candidate's asset_id is greater than
asset_pk, else 0.
- same_project (int): 1 if the candidate shares the given
project_slug, else 0.
- same_item (int): 1 if the candidate shares the given item_id, else 0.
Prioritization (descending on the following keys, then ascending by sequence):
- next_asset
- same_project
- same_item
- sequence
Args:
campaign (concordia_models.Campaign): Campaign to filter by.
user (User): Requesting user.
project_slug (str): Slug of the user's current project.
item_id (str): Identifier of the user's current item.
asset_pk (int | None): Identifier of the current asset, if any.
Returns:
QuerySet[concordia_models.NextReviewableCampaignAsset]: Prioritized
cached candidates.
"""
potential_next_assets = find_next_reviewable_campaign_assets(campaign, user)
# We'll favor assets which are in the same item or project as the original:
next_case = (
Case(
When(asset_id__gt=asset_pk, then=1),
default=0,
output_field=IntegerField(),
)
if asset_pk is not None
else Value(0, output_field=IntegerField())
)
potential_next_assets = potential_next_assets.annotate(
same_project=Case(
When(project_slug=project_slug, then=1),
default=0,
output_field=IntegerField(),
),
same_item=Case(
When(item_item_id=item_id, then=1), default=0, output_field=IntegerField()
),
next_asset=next_case,
).order_by("-next_asset", "-same_project", "-same_item", "sequence")
return potential_next_assets
@transaction.atomic
def find_next_reviewable_campaign_asset(
campaign: concordia_models.Campaign,
user: User,
project_slug: str,
item_id: str,
original_asset_id: int | None,
) -> "concordia_models.Asset | None":
"""
Retrieve the next best reviewable asset for a user within a campaign.
Strategy:
1. If `item_id` is provided, try a same-item short-circuit that advances
by (sequence, id) relative to `original_asset_id`.
2. Else, if `project_slug` is provided, select the first eligible asset
within that project (short-circuit).
3. Else, prioritize cached candidates, and if none are suitable, fall
back to computing from Asset and trigger cache population.
Concurrency:
Uses `select_for_update(skip_locked=True, of=("self",))` to avoid
double-assignments across concurrent consumers.
Args:
campaign (concordia_models.Campaign): Campaign to search within.
user (User): Requesting user.
project_slug (str): Slug of the user's current project.
item_id (str): Identifier of the user's current item.
original_asset_id (int | None): Identifier of the asset just reviewed.
Returns:
concordia_models.Asset | None: A locked eligible asset, or None if unavailable.
"""
try:
after_pk = int(original_asset_id) if original_asset_id else None
except (TypeError, ValueError):
after_pk = None
# Short-circuit: same item
if item_id:
asset = _find_reviewable_in_item(
campaign, user, item_id=item_id, after_asset_pk=after_pk
)
if asset:
return asset
# Short-circuit: same project
if project_slug:
asset = _find_reviewable_in_project(
campaign, user, project_slug=project_slug, after_asset_pk=after_pk
)
if asset:
return asset
# cache-backed selection, then manual fallback
potential_next_assets = find_and_order_potential_reviewable_campaign_assets(
campaign, user, project_slug, item_id, after_pk
)
asset_id = (
potential_next_assets.select_for_update(skip_locked=True, of=("self",))
.values_list("asset_id", flat=True)
.first()
)
spawn_task = False
if asset_id:
asset_query = concordia_models.Asset.objects.filter(id=asset_id)
else:
# Since we had no potential next assets in the caching table, we have to check
# the asset table directly.
structured_logger.debug(
"No cached assets matched, falling back to manual lookup",
event_code="reviewable_next_fallback_manual",
campaign=campaign,
user=user,
)
spawn_task = True
asset_query = find_new_reviewable_campaign_assets(campaign, user)
next_case = (
Case(
When(id__gt=after_pk, then=1),
default=0,
output_field=IntegerField(),
)
if after_pk is not None
else Value(0, output_field=IntegerField())
)
asset_query = asset_query.annotate(
same_project=Case(
When(item__project__slug=project_slug, then=1),
default=0,
output_field=IntegerField(),
),
same_item=Case(
When(item__item_id=item_id, then=1),
default=0,
output_field=IntegerField(),
),
next_asset=next_case,
).order_by("-next_asset", "-same_project", "-same_item", "sequence")
asset = (
asset_query.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if spawn_task:
# Spawn a task to populate the table for this campaign
# We wait to do this until after getting an asset because otherwise there's
# a chance all valid assets get grabbed by the task and our query will return
# nothing
structured_logger.debug(
"Spawned background task to populate cache",
event_code="reviewable_next_cache_population",
campaign=campaign,
user=user,
)
populate_task = get_registered_task(
"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_campaign"
)
populate_task.delay(campaign.id)
return asset
def find_invalid_next_reviewable_campaign_assets(
campaign_id: int,
) -> "QuerySet[concordia_models.NextReviewableCampaignAsset]":
"""
Return cache rows that are invalid for review for a given campaign.
Behavior:
Identifies NextReviewableCampaignAsset rows that are no longer valid
because the underlying asset is not `SUBMITTED` or because the asset is
currently reserved.
Args:
campaign_id (int): Identifier of the campaign.
Returns:
QuerySet[concordia_models.NextReviewableCampaignAsset]: Distinct
invalid cache rows.
"""
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(
asset__campaign_id=campaign_id
).values("asset_id")
status_filtered = concordia_models.NextReviewableCampaignAsset.objects.exclude(
asset__transcription_status=concordia_models.TranscriptionStatus.SUBMITTED
).filter(campaign_id=campaign_id)
reserved_filtered = concordia_models.NextReviewableCampaignAsset.objects.filter(
campaign_id=campaign_id, asset_id__in=Subquery(reserved_asset_ids)
)
return (status_filtered | reserved_filtered).distinct()
================================================
FILE: concordia/utils/next_asset/reviewable/topic.py
================================================
from typing import Dict
from django.contrib.auth.models import User
from django.db import transaction
from django.db.models import Case, IntegerField, Q, QuerySet, Subquery, Value, When
from concordia import models as concordia_models
from concordia.logging import ConcordiaLogger
from concordia.utils.celery import get_registered_task
structured_logger = ConcordiaLogger.get_logger(__name__)
def _reserved_asset_ids_subq() -> "QuerySet[Dict[str, int]]":
"""
Return a subquery of reserved asset identifiers.
Behavior:
Produces a subquery suitable for use with `Subquery(...)` and
`exclude(pk__in=...)` to filter out assets that currently have an
active reservation. This is not filtered to the topic to avoid
additional joins.
Returns:
QuerySet[Dict[str, int]]: A queryset of dictionaries with a single key
"asset_id" corresponding to reserved assets.
"""
return concordia_models.AssetTranscriptionReservation.objects.values("asset_id")
def _eligible_reviewable_base_qs(
topic: concordia_models.Topic,
user: User | None = None,
) -> "QuerySet[concordia_models.Asset]":
"""
Build the base queryset of reviewable assets for a topic.
Behavior:
Restricts to published projects, items, and assets, and to assets whose
transcription status is `SUBMITTED`. Optionally excludes assets
transcribed by the supplied user.
Args:
topic (concordia_models.Topic): Topic scope for filtering.
user (User | None): If provided, exclude assets transcribed by this user.
Returns:
QuerySet[concordia_models.Asset]: Reviewable assets, with `item` and
`item__project` selected via `select_related`.
"""
qs = concordia_models.Asset.objects.filter(
item__project__topics=topic.id,
item__project__published=True,
item__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,
).select_related("item", "item__project")
if user:
qs = qs.exclude(transcription__user=user.id)
return qs
def _next_seq_after(pk: int | None) -> int | None:
"""
Resolve the sequence number for a given asset primary key.
Behavior:
Convenience utility for ordering logic when advancing within a series
of assets.
Args:
pk (int | None): Asset primary key whose sequence to resolve.
Returns:
int | None: The asset's sequence number, or None if `pk` is falsy
or the asset does not exist.
"""
if not pk:
return None
return (
concordia_models.Asset.objects.filter(pk=pk)
.values_list("sequence", flat=True)
.first()
)
@transaction.atomic
def _find_reviewable_in_item(
topic: concordia_models.Topic,
user: User,
*,
item_id: str,
after_asset_pk: int | None,
) -> "concordia_models.Asset | None":
"""
Select the next reviewable asset within the same item.
Behavior:
Attempts a short-circuit within the user's current item to provide a
locally contiguous review flow.
Eligibility:
- Asset, Item, and Project are published.
- Asset transcription status is `SUBMITTED`.
- Asset is not reserved.
- Asset was not transcribed by the current user.
Ordering:
- If `after_asset_pk` refers to an asset in the same item whose project
is in `topic`, select the earliest asset whose (sequence, id) is
strictly greater than the current asset's pair.
- Otherwise, select the earliest eligible by (sequence, id).
Args:
topic (concordia_models.Topic): Topic scope.
user (User): Current user; used to exclude their own work.
item_id (str): Identifier of the item to stay within.
after_asset_pk (int | None): Asset primary key to advance from.
Returns:
concordia_models.Asset | None: A locked eligible asset, or
None if no match is available.
"""
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(
asset__item__item_id=item_id,
asset__item__project__topics=topic,
).values("asset_id")
eligible = (
concordia_models.Asset.objects.filter(
item__item_id=item_id,
item__project__topics=topic,
item__project__published=True,
item__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,
)
.exclude(pk__in=Subquery(reserved_asset_ids))
.exclude(transcription__user=user.id)
)
seq_gt_filter = None
if after_asset_pk is not None:
try:
current = (
concordia_models.Asset.objects.only("id", "sequence", "item_id", "item")
.select_related("item", "item__project")
.get(pk=after_asset_pk)
)
if (
current.item.item_id == item_id
and current.item.project.topics.filter(pk=topic.pk).exists()
):
seq_gt_filter = Q(sequence__gt=current.sequence) | (
Q(sequence=current.sequence) & Q(id__gt=after_asset_pk)
)
except concordia_models.Asset.DoesNotExist:
pass
if seq_gt_filter is not None:
eligible = eligible.filter(seq_gt_filter)
asset = (
eligible.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.order_by("sequence", "id")
.first()
)
structured_logger.debug(
"Item short-circuit (topic reviewable) resolved.",
event_code="reviewable_item_short_circuit_topic",
topic=topic,
item_id=item_id,
after_asset_pk=after_asset_pk,
chosen_asset_id=getattr(asset, "id", None),
)
return asset
@transaction.atomic
def _find_reviewable_in_project(
topic: concordia_models.Topic,
user: User,
*,
project_slug: str,
after_asset_pk: int | None,
) -> "concordia_models.Asset | None":
"""
Select the first eligible reviewable asset within the same project.
Behavior:
Short-circuit when staying within a project. Sequence is per item,
so this returns the first eligible asset, not strictly "after" a given
asset.
Eligibility:
- Same topic and project.
- Asset, Item, and Project are published.
- Asset transcription status is `SUBMITTED`.
- Asset is not reserved.
- Asset was not transcribed by the current user.
Ordering:
Deterministic by (item__item_id, sequence, id).
Args:
topic (concordia_models.Topic): Topic scope.
user (User): Current user; used to exclude their own work.
project_slug (str): Slug of the project to stay within.
after_asset_pk (int | None): Present for parity with the item
variant; not used for ordering here.
Returns:
concordia_models.Asset | None: A locked eligible asset, or
None if no match is available.
"""
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(
asset__item__project__slug=project_slug,
asset__item__project__topics=topic,
).values("asset_id")
eligible = (
concordia_models.Asset.objects.filter(
item__project__topics=topic,
item__project__slug=project_slug,
item__project__published=True,
item__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.SUBMITTED,
)
.exclude(pk__in=Subquery(reserved_asset_ids))
.exclude(transcription__user=user.id)
.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.order_by("item__item_id", "sequence", "id")
.first()
)
structured_logger.debug(
"Project short-circuit (topic reviewable) resolved.",
event_code="reviewable_project_short_circuit_topic",
topic=topic,
project_slug=project_slug,
after_asset_pk=after_asset_pk,
chosen_asset_id=getattr(eligible, "id", None),
)
return eligible
def find_new_reviewable_topic_assets(
topic: concordia_models.Topic,
user: User | None = None,
) -> "QuerySet[concordia_models.Asset]":
"""
Return assets in a topic that are eligible to be added to the cache.
Behavior:
Builds the candidate set for the `NextReviewableTopicAsset` cache by
excluding assets that are not `SUBMITTED`, assets already reserved, and
assets already present in the cache. Optionally excludes assets
transcribed by the provided user.
Args:
topic (concordia_models.Topic): Topic to filter by.
user (User | None): If provided, exclude assets transcribed by this user.
Returns:
QuerySet[concordia_models.Asset]: Eligible assets ordered by sequence.
"""
# Filtering this to the topic would be more costly than just getting all ids
# in most cases because it requires joining the asset table to the item table to
# the project table to the topic table.
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(
"asset_id"
)
next_asset_ids = concordia_models.NextReviewableTopicAsset.objects.filter(
topic=topic
).values("asset_id")
queryset = (
concordia_models.Asset.objects.filter(
item__project__topics=topic.id,
item__project__published=True,
item__published=True,
published=True,
)
.filter(transcription_status=concordia_models.TranscriptionStatus.SUBMITTED)
.exclude(pk__in=Subquery(reserved_asset_ids))
.exclude(pk__in=Subquery(next_asset_ids))
.order_by("sequence")
)
if user:
queryset = queryset.exclude(transcription__user=user.id)
return queryset
def find_next_reviewable_topic_assets(
topic: concordia_models.Topic,
user: User,
) -> "QuerySet[concordia_models.NextReviewableTopicAsset]":
"""
Return cached reviewable assets in a topic not transcribed by the user.
Behavior:
Reads from the `NextReviewableTopicAsset` cache table and filters out
assets where the requesting user appears in `transcriber_ids`.
Args:
topic (concordia_models.Topic): Topic to retrieve cached assets from.
user (User): Requesting user.
Returns:
QuerySet[concordia_models.NextReviewableTopicAsset]: Cached candidate rows
for the given user.
"""
return concordia_models.NextReviewableTopicAsset.objects.filter(
topic=topic
).exclude(transcriber_ids__contains=[user.id])
@transaction.atomic
def find_reviewable_topic_asset(
topic: concordia_models.Topic,
user: User,
) -> "concordia_models.Asset | None":
"""
Retrieve a single reviewable asset for a user from a topic.
Behavior:
First attempts to select a cached asset from `NextReviewableTopicAsset`.
If none is available, falls back to a direct query over `Asset` and
triggers a background task to replenish the cache.
Concurrency:
Uses `select_for_update(skip_locked=True, of=("self",))` so only the
`Asset` row is locked and concurrent consumers skip locked rows.
Args:
topic (concordia_models.Topic): Topic to search within.
user (User): Requesting user; their own transcriptions are excluded.
Returns:
concordia_models.Asset | None: A locked eligible asset, or None
if unavailable.
"""
next_asset = (
find_next_reviewable_topic_assets(topic, user)
.select_for_update(skip_locked=True, of=("self",))
.values_list("asset_id", flat=True)
.first()
)
spawn_task = False
if next_asset:
asset_query = concordia_models.Asset.objects.filter(id=next_asset)
else:
# No asset in the NextReviewableTopicAsset table for this topic,
# so fallback to manually finding one
structured_logger.debug(
"No cached assets available, falling back to manual lookup",
event_code="reviewable_fallback_manual_lookup",
topic=topic,
user=user,
)
spawn_task = True
asset_query = find_new_reviewable_topic_assets(topic, user)
# select_for_update(of=("self",)) causes the row locking only to
# apply to the Asset table, rather than also locking joined item table
asset = (
asset_query.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if spawn_task:
# Spawn a task to populate the table for this topic
# We wait to do this until after getting an asset because otherwise there's a
# a chance all valid assets get grabbed by the task and our query will return
# nothing
structured_logger.debug(
"Spawned background task to populate cache",
event_code="reviewable_cache_population_triggered",
topic=topic,
user=user,
)
populate_task = get_registered_task(
"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_topic"
)
populate_task.delay(topic.id)
return asset
def find_and_order_potential_reviewable_topic_assets(
topic: concordia_models.Topic,
user: User,
project_slug: str,
item_id: str,
asset_pk: int | None,
) -> "QuerySet[concordia_models.NextReviewableTopicAsset]":
"""
Retrieve and prioritize cached reviewable assets for proximity.
Behavior:
Orders cached candidates from `NextReviewableTopicAsset` to prefer
continuity with the user's current location.
Annotations added to each row (transient fields):
- next_asset (int): 1 if the candidate's asset_id is greater than
asset_pk, else 0.
- same_project (int): 1 if the candidate shares the given
project_slug, else 0.
- same_item (int): 1 if the candidate shares the given item_id, else 0.
Prioritization (descending on the following keys, then ascending by sequence):
- next_asset
- same_project
- same_item
- sequence
Args:
topic (concordia_models.Topic): Topic to filter by.
user (User): Requesting user.
project_slug (str): Slug of the user's current project.
item_id (str): Identifier of the user's current item.
asset_pk (int | None): Identifier of the current asset, if any.
Returns:
QuerySet[concordia_models.NextReviewableTopicAsset]: Prioritized
cached candidates.
"""
potential_next_assets = find_next_reviewable_topic_assets(topic, user)
# Handle None safely for the "next" signal
next_case = (
Case(
When(asset_id__gt=asset_pk, then=1),
default=0,
output_field=IntegerField(),
)
if asset_pk is not None
else Value(0, output_field=IntegerField())
)
# We'll favor assets which are in the same item or project as the original:
potential_next_assets = potential_next_assets.annotate(
same_project=Case(
When(project_slug=project_slug, then=1),
default=0,
output_field=IntegerField(),
),
same_item=Case(
When(item_item_id=item_id, then=1), default=0, output_field=IntegerField()
),
next_asset=next_case,
).order_by("-next_asset", "-same_project", "-same_item", "sequence")
return potential_next_assets
@transaction.atomic
def find_next_reviewable_topic_asset(
topic: concordia_models.Topic,
user: User,
project_slug: str,
item_id: str,
original_asset_id: int | None,
) -> "concordia_models.Asset | None":
"""
Retrieve the next best reviewable asset for a user within a topic.
Strategy:
1. If `item_id` is provided, try a same-item short-circuit that advances
by (sequence, id) relative to `original_asset_id`.
2. Else, if `project_slug` is provided, select the first eligible asset
within that project (short-circuit).
3. Else, prioritize cached candidates, and if none are suitable, fall
back to computing from `Asset` and trigger cache population.
Concurrency:
Uses `select_for_update(skip_locked=True, of=("self",))` to avoid
double-assignments across concurrent consumers.
Args:
topic (concordia_models.Topic): Topic to search within.
user (User): Requesting user.
project_slug (str): Slug of the user's current project.
item_id (str): Identifier of the user's current item.
original_asset_id (int | None): Identifier of the asset just reviewed.
Returns:
concordia_models.Asset | None: A locked eligible asset, or None if
unavailable.
"""
# Normalize the "after" reference
try:
after_pk = int(original_asset_id) if original_asset_id else None
except (TypeError, ValueError):
after_pk = None
# Short-circuit: same item
if item_id:
asset = _find_reviewable_in_item(
topic, user, item_id=item_id, after_asset_pk=after_pk
)
if asset:
return asset
# Short-circuit: same project
if project_slug:
asset = _find_reviewable_in_project(
topic, user, project_slug=project_slug, after_asset_pk=after_pk
)
if asset:
return asset
# Cache-backed selection, then manual fallback
potential_next_assets = find_and_order_potential_reviewable_topic_assets(
topic, user, project_slug, item_id, after_pk
)
asset_id = (
potential_next_assets.select_for_update(skip_locked=True, of=("self",))
.values_list("asset_id", flat=True)
.first()
)
spawn_task = False
if asset_id:
asset_query = concordia_models.Asset.objects.filter(id=asset_id)
else:
# Since we had no potential next assets in the caching table, we have to check
# the asset table directly.
structured_logger.debug(
"No cached assets matched, falling back to manual lookup",
event_code="reviewable_next_fallback_manual",
topic=topic,
user=user,
)
spawn_task = True
asset_query = find_new_reviewable_topic_assets(topic, user)
next_case = (
Case(
When(id__gt=after_pk, then=1),
default=0,
output_field=IntegerField(),
)
if after_pk is not None
else Value(0, output_field=IntegerField())
)
asset_query = asset_query.annotate(
same_project=Case(
When(item__project__slug=project_slug, then=1),
default=0,
output_field=IntegerField(),
),
same_item=Case(
When(item__item_id=item_id, then=1),
default=0,
output_field=IntegerField(),
),
next_asset=next_case,
).order_by("-next_asset", "-same_project", "-same_item", "sequence")
asset = (
asset_query.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if spawn_task:
# Spawn a task to populate the table for this topic
# We wait to do this until after getting an asset because otherwise there's
# a chance all valid assets get grabbed by the task and our query will return
# nothing
structured_logger.debug(
"Spawned background task to populate cache",
event_code="reviewable_next_cache_population",
topic=topic,
user=user,
)
populate_task = get_registered_task(
"concordia.tasks.next_asset.reviewable.populate_next_reviewable_for_topic"
)
populate_task.delay(topic.id)
return asset
def find_invalid_next_reviewable_topic_assets(
topic_id: int,
) -> "QuerySet[concordia_models.NextReviewableTopicAsset]":
"""
Return cache rows that are invalid for review for a given topic.
Behavior:
Identifies `NextReviewableTopicAsset` rows that are no longer valid
because the underlying asset is not `SUBMITTED` or because the asset is
currently reserved.
Args:
topic_id (int): Identifier of the topic.
Returns:
QuerySet[concordia_models.NextReviewableTopicAsset]: Distinct invalid
cache rows.
"""
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(
asset__item__project__topics=topic_id
).values("asset_id")
status_filtered = concordia_models.NextReviewableTopicAsset.objects.exclude(
asset__transcription_status=concordia_models.TranscriptionStatus.SUBMITTED
).filter(topic_id=topic_id)
reserved_filtered = concordia_models.NextReviewableTopicAsset.objects.filter(
topic_id=topic_id, asset_id__in=Subquery(reserved_asset_ids)
)
return (status_filtered | reserved_filtered).distinct()
================================================
FILE: concordia/utils/next_asset/transcribable/__init__.py
================================================
from .campaign import (
find_and_order_potential_transcribable_campaign_assets,
find_invalid_next_transcribable_campaign_assets,
find_new_transcribable_campaign_assets,
find_next_transcribable_campaign_asset,
find_next_transcribable_campaign_assets,
find_transcribable_campaign_asset,
)
from .topic import (
find_and_order_potential_transcribable_topic_assets,
find_invalid_next_transcribable_topic_assets,
find_new_transcribable_topic_assets,
find_next_transcribable_topic_asset,
find_next_transcribable_topic_assets,
find_transcribable_topic_asset,
)
__all__ = [
"find_new_transcribable_campaign_assets",
"find_next_transcribable_campaign_assets",
"find_transcribable_campaign_asset",
"find_and_order_potential_transcribable_campaign_assets",
"find_next_transcribable_campaign_asset",
"find_and_order_potential_transcribable_topic_assets",
"find_new_transcribable_topic_assets",
"find_next_transcribable_topic_asset",
"find_next_transcribable_topic_assets",
"find_transcribable_topic_asset",
"find_invalid_next_transcribable_campaign_assets",
"find_invalid_next_transcribable_topic_assets",
]
================================================
FILE: concordia/utils/next_asset/transcribable/campaign.py
================================================
from typing import Dict
from django.db import transaction
from django.db.models import Case, IntegerField, Q, QuerySet, Subquery, When
from concordia import models as concordia_models
from concordia.logging import ConcordiaLogger
from concordia.utils.celery import get_registered_task
structured_logger = ConcordiaLogger.get_logger(__name__)
def _reserved_asset_ids_subq(
campaign: concordia_models.Campaign,
) -> "QuerySet[Dict[str, int]]":
"""
Return a subquery of reserved asset identifiers for a campaign.
Behavior:
Produces a subquery suitable for use with `Subquery(...)` and
`exclude(pk__in=...)` clauses to filter out assets that currently have
an active reservation.
Args:
campaign (concordia_models.Campaign): Campaign whose reserved assets
should be excluded.
Returns:
QuerySet[Dict[str, int]]: A queryset of dictionaries with a single key
"asset_id" corresponding to reserved assets.
"""
return concordia_models.AssetTranscriptionReservation.objects.filter(
asset__campaign=campaign
).values("asset_id")
def _eligible_transcribable_base_qs(
campaign: concordia_models.Campaign,
) -> "QuerySet[concordia_models.Asset]":
"""
Build the base queryset of transcribable assets for a campaign.
Behavior:
Restricts to published projects, items, and assets, and to assets whose
transcription status is either `NOT_STARTED` or `IN_PROGRESS`.
Args:
campaign (concordia_models.Campaign): Campaign scope for filtering.
Returns:
QuerySet[concordia_models.Asset]: Transcribable assets, with `item` and
`item__project` selected via `select_related`.
"""
return concordia_models.Asset.objects.filter(
campaign_id=campaign.id,
item__project__published=True,
item__published=True,
published=True,
transcription_status__in=[
concordia_models.TranscriptionStatus.NOT_STARTED,
concordia_models.TranscriptionStatus.IN_PROGRESS,
],
).select_related("item", "item__project")
def _next_seq_after(pk: int | None) -> int | None:
"""
Resolve the sequence number for a given asset primary key.
Behavior:
Convenience utility for ordering logic when advancing within a series
of assets.
Args:
pk (int | None): Asset primary key whose sequence to resolve.
Returns:
int | None: The asset's sequence number, or None if `pk` is falsy
or the asset does not exist.
"""
if not pk:
return None
return (
concordia_models.Asset.objects.filter(pk=pk)
.values_list("sequence", flat=True)
.first()
)
def _order_unstarted_first(
qs: "QuerySet[concordia_models.Asset]",
) -> "QuerySet[concordia_models.Asset]":
"""
Apply a stable ordering that prefers `NOT_STARTED` over `IN_PROGRESS`,
then orders by `sequence`.
Args:
qs (QuerySet[concordia_models.Asset]): Base queryset to annotate and sort.
Returns:
QuerySet[concordia_models.Asset]: Annotated and ordered queryset with a
transient `unstarted` field (1 for `NOT_STARTED`, else 0).
"""
return qs.annotate(
unstarted=Case(
When(
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
then=1,
),
default=0,
output_field=IntegerField(),
)
).order_by("-unstarted", "sequence")
@transaction.atomic
def _find_transcribable_in_item(
campaign: concordia_models.Campaign,
*,
item_id: str,
after_asset_pk: int | None,
) -> "concordia_models.Asset | None":
"""
Fast path: find the next transcribable asset in the same item.
Behavior:
- Exclude the current asset.
- Advance by `(sequence, id)` within the item.
- Return only `NOT_STARTED` here (defer `IN_PROGRESS` to later fallbacks).
- Skip reserved assets.
- Respect published flags on campaign, project, item, and asset.
Args:
campaign (concordia_models.Campaign): Campaign scope.
item_id (str): Identifier of the item to stay within.
after_asset_pk (int | None): Asset primary key to advance from.
Returns:
concordia_models.Asset | None: The next eligible asset, or None if none.
"""
if not item_id:
return None
# Find current sequence to advance correctly within the item
cur_seq = None
if after_asset_pk:
cur_seq = (
concordia_models.Asset.objects.filter(pk=after_asset_pk)
.values_list("sequence", flat=True)
.first()
)
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(
"asset_id"
)
base = concordia_models.Asset.objects.filter(
item__item_id=item_id,
item__published=True,
item__project__published=True,
published=True,
campaign_id=campaign.id,
).exclude(pk__in=Subquery(reserved_asset_ids))
if after_asset_pk:
if cur_seq is not None:
base = base.filter(
Q(sequence__gt=cur_seq)
| (Q(sequence=cur_seq) & Q(id__gt=after_asset_pk))
)
else:
base = base.exclude(id=after_asset_pk)
# ONLY NOT_STARTED in this short-circuit
return (
base.filter(
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED
)
.order_by("sequence", "id")
.first()
)
def _find_transcribable_not_started_in_project(
campaign: concordia_models.Campaign,
*,
project_slug: str,
exclude_item_id: str | None = None,
) -> "concordia_models.Asset | None":
"""
Fast path: find the first `NOT_STARTED` asset in the same project.
Behavior:
Allows different items (optionally excluding the current item to avoid
bouncing back). Uses a stable ordering by `(item_id, sequence, id)`.
Args:
campaign (concordia_models.Campaign): Campaign scope.
project_slug (str): Slug of the project to stay within.
exclude_item_id (str | None): If provided, exclude this item.
Returns:
concordia_models.Asset | None: The first eligible asset, or None if none.
"""
if not project_slug:
return None
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(
"asset_id"
)
base = concordia_models.Asset.objects.filter(
campaign_id=campaign.id,
item__project__slug=project_slug,
item__published=True,
item__project__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
).exclude(pk__in=Subquery(reserved_asset_ids))
if exclude_item_id:
base = base.exclude(item__item_id=exclude_item_id)
return base.order_by("item__item_id", "sequence", "id").first()
def find_new_transcribable_campaign_assets(
campaign: concordia_models.Campaign,
) -> "QuerySet[concordia_models.Asset]":
"""
Return assets in a campaign that are eligible to be added to the cache.
Behavior:
Builds the candidate set for the `NextTranscribableCampaignAsset` cache
by excluding assets that are not `NOT_STARTED` or `IN_PROGRESS`, assets
already reserved, and assets already present in the cache.
Args:
campaign (concordia_models.Campaign): Campaign to filter by.
Returns:
QuerySet[concordia_models.Asset]: Eligible assets ordered by `sequence`.
"""
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(
asset__campaign=campaign
).values("asset_id")
next_asset_ids = concordia_models.NextTranscribableCampaignAsset.objects.filter(
campaign=campaign
).values("asset_id")
return (
concordia_models.Asset.objects.filter(
campaign_id=campaign.id,
item__project__published=True,
item__published=True,
published=True,
)
.filter(
Q(transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED)
| Q(transcription_status=concordia_models.TranscriptionStatus.IN_PROGRESS)
)
.exclude(pk__in=Subquery(reserved_asset_ids))
.exclude(pk__in=Subquery(next_asset_ids))
.order_by("sequence")
)
def find_next_transcribable_campaign_assets(
campaign: concordia_models.Campaign,
) -> "QuerySet[concordia_models.NextTranscribableCampaignAsset]":
"""
Return all cached transcribable assets for a campaign.
Behavior:
Reads from the `NextTranscribableCampaignAsset` cache table for the
given campaign.
Args:
campaign (concordia_models.Campaign): Campaign to retrieve cached assets for.
Returns:
QuerySet[concordia_models.NextTranscribableCampaignAsset]: Cached candidates.
"""
return concordia_models.NextTranscribableCampaignAsset.objects.filter(
campaign=campaign
)
@transaction.atomic
def find_transcribable_campaign_asset(
campaign: concordia_models.Campaign,
) -> "concordia_models.Asset | None":
"""
Retrieve a single transcribable asset from the campaign.
Behavior:
First attempts to select a cached asset from
`NextTranscribableCampaignAsset`. If none is available, falls back to a
direct query over `Asset` and triggers a background task to replenish
the cache.
Concurrency:
Uses `select_for_update(skip_locked=True, of=("self",))` so only the
`Asset` row is locked and concurrent consumers skip locked rows.
Args:
campaign (concordia_models.Campaign): Campaign to search within.
Returns:
concordia_models.Asset | None: A locked eligible asset, or None if
unavailable.
"""
next_asset = (
find_next_transcribable_campaign_assets(campaign)
.select_for_update(skip_locked=True, of=("self",))
.values_list("asset_id", flat=True)
.first()
)
spawn_task = False
if next_asset:
asset_query = concordia_models.Asset.objects.filter(id=next_asset)
else:
# No asset in the NextTranscribableCampaignAsset table for this campaign,
# so fallback to manually finding on
structured_logger.debug(
"No cached assets available, falling back to manual lookup",
event_code="transcribable_fallback_manual_lookup",
campaign=campaign,
)
asset_query = find_new_transcribable_campaign_assets(campaign)
spawn_task = True
# select_for_update(of=("self",)) causes the row locking only to
# apply to the Asset table, rather than also locking joined item table
asset = (
asset_query.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if spawn_task:
# Spawn a task to populate the table for this campaign
# We wait to do this until after getting an asset because otherwise there's a
# a chance all valid assets get grabbed by the task and our query will return
# nothing
structured_logger.debug(
"Spawned background task to populate cache",
event_code="transcribable_cache_population_triggered",
campaign=campaign,
)
populate_task = get_registered_task(
"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_campaign"
)
populate_task.delay(campaign.id)
return asset
def find_and_order_potential_transcribable_campaign_assets(
campaign: concordia_models.Campaign,
project_slug: str,
item_id: str,
asset_pk: int,
) -> "QuerySet[concordia_models.NextTranscribableCampaignAsset]":
"""
Retrieve and prioritize cached transcribable assets based on proximity
and status.
Behavior:
Orders cached candidates from `NextTranscribableCampaignAsset` to prefer:
- `NOT_STARTED` over `IN_PROGRESS` (via transient `unstarted` flag),
- same project,
- same item,
then by `sequence` and `asset_id` for stability.
Annotations added to each row (transient fields):
- unstarted (int): 1 if transcription status is `NOT_STARTED`, else 0.
- same_project (int): 1 if the candidate shares `project_slug`, else 0.
- same_item (int): 1 if the candidate shares `item_id`, else 0.
Args:
campaign (concordia_models.Campaign): Campaign to filter by.
project_slug (str): Slug of the original asset's project.
item_id (str): Item identifier of the original asset.
asset_pk (int): Primary key of the original asset.
Returns:
QuerySet[concordia_models.NextTranscribableCampaignAsset]: Prioritized
cached candidates.
"""
potential_next_assets = find_next_transcribable_campaign_assets(campaign)
potential_next_assets = potential_next_assets.annotate(
unstarted=Case(
When(
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
then=1,
),
default=0,
output_field=IntegerField(),
),
same_project=Case(
When(project_slug=project_slug, then=1),
default=0,
output_field=IntegerField(),
),
same_item=Case(
When(item_item_id=item_id, then=1),
default=0,
output_field=IntegerField(),
),
).order_by(
"-unstarted",
"-same_project",
"-same_item",
"sequence",
"asset_id",
)
return potential_next_assets
@transaction.atomic
def find_next_transcribable_campaign_asset(
campaign: concordia_models.Campaign,
project_slug: str,
item_id: str,
original_asset_id: int | None,
) -> "concordia_models.Asset | None":
"""
Retrieve the next best transcribable asset within a campaign.
Priority for short-circuit selection (before cache and fallback):
1) If `item_id` is provided, return the next `NOT_STARTED` asset in
that item by sequence (strictly after the original asset when known).
2) If `project_slug` is provided, return the first `NOT_STARTED` asset
in that project (ordered by item id, then sequence), excluding the
current item to keep moving forward.
If none of the above match, fall back to the cache-backed path:
Attempts to retrieve a candidate from `NextTranscribableCampaignAsset`. If
none is found, compute from `Asset` and trigger cache population.
After exhausting `NOT_STARTED` options, consider `IN_PROGRESS` assets in the
same item (strictly after the original when known).
Concurrency:
Uses `select_for_update(skip_locked=True, of=("self",))` to avoid
double-assignments across concurrent consumers.
Args:
campaign (concordia_models.Campaign): Campaign to search within.
project_slug (str): Slug of the current project.
item_id (str): Identifier of the current item.
original_asset_id (int | None): Identifier of the asset just transcribed.
Returns:
concordia_models.Asset | None: A locked eligible asset, or None if
unavailable.
"""
# Normalize original_asset_id for safe use in filters/comparisons
try:
original_pk = int(original_asset_id) if original_asset_id is not None else None
except (TypeError, ValueError):
original_pk = None
# Resolve "after sequence" only when the original asset belongs to the same item.
after_seq = None
if item_id and original_pk is not None:
try:
orig = (
concordia_models.Asset.objects.select_related("item")
.only("id", "sequence", "item__item_id")
.get(pk=original_pk)
)
if getattr(orig.item, "item_id", None) == item_id:
after_seq = orig.sequence
except concordia_models.Asset.DoesNotExist:
after_seq = None
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(
"asset_id"
)
# Short-circuit: same item and NOT_STARTED after current sequence
if item_id:
qs = concordia_models.Asset.objects.filter(
campaign_id=campaign.id,
item__item_id=item_id,
item__published=True,
item__project__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
).exclude(pk__in=Subquery(reserved_asset_ids))
if original_pk is not None:
qs = qs.exclude(pk=original_pk)
if after_seq is not None:
qs = qs.filter(
Q(sequence__gt=after_seq)
| (Q(sequence=after_seq) & Q(id__gt=original_pk))
)
asset = (
qs.order_by("sequence", "id")
.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if asset:
return asset
# Short-circuit: same project and NOT_STARTED (avoid current item and original)
if project_slug:
candidate = concordia_models.Asset.objects.filter(
campaign_id=campaign.id,
item__project__slug=project_slug,
item__published=True,
item__project__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
).exclude(pk__in=Subquery(reserved_asset_ids))
if original_pk is not None:
candidate = candidate.exclude(pk=original_pk)
if item_id:
candidate = candidate.exclude(item__item_id=item_id)
asset = (
candidate.order_by("item__item_id", "sequence", "id")
.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if asset:
return asset
# Cache-backed selection (NOT_STARTED), then manual fallback (also NOT_STARTED)
potential_next_assets = find_and_order_potential_transcribable_campaign_assets(
campaign, project_slug, item_id, original_asset_id
)
if original_pk is not None:
potential_next_assets = potential_next_assets.exclude(asset_id=original_pk)
if item_id:
# Keep moving forward: avoid bouncing to the same item
potential_next_assets = potential_next_assets.exclude(item_item_id=item_id)
asset_id = (
potential_next_assets.select_for_update(skip_locked=True, of=("self",))
.values_list("asset_id", flat=True)
.first()
)
spawn_task = False
if asset_id:
asset_query = concordia_models.Asset.objects.filter(id=asset_id)
else:
structured_logger.debug(
"No cached assets matched, falling back to manual lookup",
event_code="transcribable_next_fallback_manual",
campaign=campaign,
)
spawn_task = True
asset_query = find_new_transcribable_campaign_assets(campaign)
if original_pk is not None:
asset_query = asset_query.exclude(pk=original_pk)
if item_id:
asset_query = asset_query.exclude(item__item_id=item_id)
asset_query = asset_query.annotate(
unstarted=Case(
When(
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
then=1,
),
default=0,
output_field=IntegerField(),
),
same_project=Case(
When(item__project__slug=project_slug, then=1),
default=0,
output_field=IntegerField(),
),
same_item=Case(
When(item__item_id=item_id, then=1),
default=0,
output_field=IntegerField(),
),
).order_by(
"-unstarted",
"-same_project",
"-same_item",
"sequence",
"id",
)
asset = (
asset_query.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if asset:
if spawn_task:
structured_logger.debug(
"Spawned background task to populate cache",
event_code="transcribable_next_cache_population",
campaign=campaign,
)
populate_task = get_registered_task(
"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_campaign"
)
populate_task.delay(campaign.id)
return asset
# Only now consider same-item IN_PROGRESS after current sequence
if item_id:
qs = concordia_models.Asset.objects.filter(
campaign_id=campaign.id,
item__item_id=item_id,
item__published=True,
item__project__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.IN_PROGRESS,
).exclude(pk__in=Subquery(reserved_asset_ids))
if original_pk is not None:
qs = qs.exclude(pk=original_pk)
if after_seq is not None:
qs = qs.filter(
Q(sequence__gt=after_seq)
| (Q(sequence=after_seq) & Q(id__gt=original_pk))
)
asset = (
qs.order_by("sequence", "id")
.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if asset:
return asset
return None
def find_invalid_next_transcribable_campaign_assets(
campaign_id: int,
) -> "QuerySet[concordia_models.NextTranscribableCampaignAsset]":
"""
Return cached rows that are invalid for transcription for a campaign.
Behavior:
Identifies `NextTranscribableCampaignAsset` rows that are no longer valid
because the underlying asset is neither `NOT_STARTED` nor `IN_PROGRESS`,
or because the asset is currently reserved.
Args:
campaign_id (int): Identifier of the campaign.
Returns:
QuerySet[concordia_models.NextTranscribableCampaignAsset]: Distinct invalid
cache rows.
"""
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(
asset__campaign_id=campaign_id
).values("asset_id")
# Assets with transcription_status not eligible for transcription
status_filtered = concordia_models.NextTranscribableCampaignAsset.objects.filter(
campaign_id=campaign_id
).exclude(
asset__transcription_status__in=[
concordia_models.TranscriptionStatus.NOT_STARTED,
concordia_models.TranscriptionStatus.IN_PROGRESS,
]
)
# Assets that are reserved
reserved_filtered = concordia_models.NextTranscribableCampaignAsset.objects.filter(
campaign_id=campaign_id, asset_id__in=Subquery(reserved_asset_ids)
)
return (status_filtered | reserved_filtered).distinct()
================================================
FILE: concordia/utils/next_asset/transcribable/topic.py
================================================
from typing import Dict
from django.db import transaction
from django.db.models import Case, IntegerField, Q, QuerySet, Subquery, When
from concordia import models as concordia_models
from concordia.logging import ConcordiaLogger
from concordia.utils.celery import get_registered_task
structured_logger = ConcordiaLogger.get_logger(__name__)
def _reserved_asset_ids_subq() -> "QuerySet[Dict[str, int]]":
"""
Return a subquery of reserved asset identifiers.
Behavior:
Not filtered to the topic to avoid extra joins. Produces a subquery
suitable for use with `Subquery(...)` and `exclude(pk__in=...)`
clauses to filter out assets that currently have an active
reservation.
Returns:
QuerySet[Dict[str, int]]: A queryset of dictionaries with a single key
"asset_id" corresponding to reserved assets.
"""
return concordia_models.AssetTranscriptionReservation.objects.values("asset_id")
def _eligible_transcribable_base_qs(
topic: "concordia_models.Topic",
) -> "QuerySet[concordia_models.Asset]":
"""
Build the base queryset of transcribable assets for a topic.
Behavior:
Restricts to published projects, items, and assets, and to assets whose
transcription status is either `NOT_STARTED` or `IN_PROGRESS`.
Args:
topic (concordia_models.Topic): Topic scope for filtering.
Returns:
QuerySet[concordia_models.Asset]: Transcribable assets, with `item` and
`item__project` selected via `select_related`.
"""
return concordia_models.Asset.objects.filter(
item__project__topics=topic.id,
item__project__published=True,
item__published=True,
published=True,
transcription_status__in=[
concordia_models.TranscriptionStatus.NOT_STARTED,
concordia_models.TranscriptionStatus.IN_PROGRESS,
],
).select_related("item", "item__project")
def _next_seq_after(pk: int | None) -> int | None:
"""
Resolve the sequence number for a given asset primary key.
Behavior:
Convenience utility for ordering logic when advancing within a series
of assets.
Args:
pk (int | None): Asset primary key whose sequence to resolve.
Returns:
int | None: The asset's sequence number, or None if `pk` is falsy
or the asset does not exist.
"""
if not pk:
return None
return (
concordia_models.Asset.objects.filter(pk=pk)
.values_list("sequence", flat=True)
.first()
)
def _order_unstarted_first(
qs: "QuerySet[concordia_models.Asset]",
) -> "QuerySet[concordia_models.Asset]":
"""
Apply a stable ordering that prefers `NOT_STARTED` over `IN_PROGRESS`,
then orders by `sequence`.
Args:
qs (QuerySet[concordia_models.Asset]): Base queryset to annotate and sort.
Returns:
QuerySet[concordia_models.Asset]: Annotated and ordered queryset with a
transient `unstarted` field (1 for `NOT_STARTED`, else 0).
"""
return qs.annotate(
unstarted=Case(
When(
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
then=1,
),
default=0,
output_field=IntegerField(),
)
).order_by("-unstarted", "sequence")
def _find_transcribable_in_item_for_topic(
topic: "concordia_models.Topic",
*,
item_id: str,
after_asset_pk: int | None,
) -> "concordia_models.Asset | None":
"""
Fast path: find the next transcribable asset in the same item, constrained
to the topic.
Behavior:
- Asset must belong to a project that is in this topic.
- Exclude the current asset.
- Advance by `(sequence, id)` within the item.
- Return only `NOT_STARTED` here (defer `IN_PROGRESS` to later fallbacks).
- Skip reserved assets.
- Respect published flags.
Args:
topic (concordia_models.Topic): Topic scope.
item_id (str): Identifier of the item to stay within.
after_asset_pk (int | None): Asset primary key to advance from.
Returns:
concordia_models.Asset | None: The next eligible asset, or None if none.
"""
if not item_id:
return None
cur_seq = None
if after_asset_pk:
cur_seq = (
concordia_models.Asset.objects.filter(pk=after_asset_pk)
.values_list("sequence", flat=True)
.first()
)
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(
"asset_id"
)
base = concordia_models.Asset.objects.filter(
item__item_id=item_id,
item__project__topics=topic.id,
item__published=True,
item__project__published=True,
published=True,
).exclude(pk__in=Subquery(reserved_asset_ids))
if after_asset_pk:
if cur_seq is not None:
base = base.filter(
Q(sequence__gt=cur_seq)
| (Q(sequence=cur_seq) & Q(id__gt=after_asset_pk))
)
else:
base = base.exclude(id=after_asset_pk)
# ONLY NOT_STARTED in this short-circuit
return (
base.filter(
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED
)
.order_by("sequence", "id")
.first()
)
def _find_transcribable_not_started_in_project_for_topic(
topic: "concordia_models.Topic",
*,
project_slug: str,
exclude_item_id: str | None = None,
) -> "concordia_models.Asset | None":
"""
Fast path: find the first `NOT_STARTED` asset in the same project within
this topic.
Behavior:
Optionally exclude the current item. Uses a stable ordering by
`(item__item_id, sequence, id)`.
Args:
topic (concordia_models.Topic): Topic scope.
project_slug (str): Slug of the project to stay within.
exclude_item_id (str | None): If provided, exclude this item.
Returns:
concordia_models.Asset | None: The first eligible asset, or None if none.
"""
if not project_slug:
return None
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(
"asset_id"
)
base = concordia_models.Asset.objects.filter(
item__project__topics=topic.id,
item__project__slug=project_slug,
item__published=True,
item__project__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
).exclude(pk__in=Subquery(reserved_asset_ids))
if exclude_item_id:
base = base.exclude(item__item_id=exclude_item_id)
return base.order_by("item__item_id", "sequence", "id").first()
def find_new_transcribable_topic_assets(
topic: "concordia_models.Topic",
) -> "QuerySet[concordia_models.Asset]":
"""
Return assets in a topic that are eligible to be added to the cache.
Behavior:
Builds the candidate set for the `NextTranscribableTopicAsset` cache by
excluding assets that are not `NOT_STARTED` or `IN_PROGRESS`, assets
already reserved, and assets already present in the cache.
Args:
topic (concordia_models.Topic): Topic to filter by.
Returns:
QuerySet[concordia_models.Asset]: Eligible assets ordered by `sequence`.
"""
# Filtering this to the topic would be more costly than just getting all ids
# in most cases because it requires joining the asset table to the item table to
# the project table to the topic table.
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(
"asset_id"
)
next_asset_ids = concordia_models.NextTranscribableTopicAsset.objects.filter(
topic=topic
).values("asset_id")
return (
concordia_models.Asset.objects.filter(
item__project__topics=topic.id,
item__project__published=True,
item__published=True,
published=True,
)
.filter(
Q(transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED)
| Q(transcription_status=concordia_models.TranscriptionStatus.IN_PROGRESS)
)
.exclude(pk__in=Subquery(reserved_asset_ids))
.exclude(pk__in=Subquery(next_asset_ids))
.order_by("sequence")
)
def find_next_transcribable_topic_assets(
topic: "concordia_models.Topic",
) -> "QuerySet[concordia_models.NextTranscribableTopicAsset]":
"""
Return all cached transcribable assets for a topic.
Behavior:
Reads from the `NextTranscribableTopicAsset` cache table for the
given topic.
Args:
topic (concordia_models.Topic): Topic to retrieve cached assets for.
Returns:
QuerySet[concordia_models.NextTranscribableTopicAsset]: Cached candidates.
"""
return concordia_models.NextTranscribableTopicAsset.objects.filter(topic=topic)
@transaction.atomic
def find_transcribable_topic_asset(
topic: "concordia_models.Topic",
) -> "concordia_models.Asset | None":
"""
Retrieve a single transcribable asset from the topic.
Behavior:
First attempts to select a cached asset from
`NextTranscribableTopicAsset`. If none is available, falls back to a
direct query over `Asset` and triggers a background task to replenish
the cache.
Concurrency:
Uses `select_for_update(skip_locked=True, of=("self",))` so only the
`Asset` row is locked and concurrent consumers skip locked rows.
Args:
topic (concordia_models.Topic): Topic to search within.
Returns:
concordia_models.Asset | None: A locked eligible asset, or None if
unavailable.
"""
next_asset = (
find_next_transcribable_topic_assets(topic)
.select_for_update(skip_locked=True, of=("self",))
.values_list("asset_id", flat=True)
.first()
)
spawn_task = False
if next_asset:
asset_query = concordia_models.Asset.objects.filter(id=next_asset)
else:
# No asset in the NextTranscribableTopicAsset table for this topic,
# so fallback to manually finding one
structured_logger.debug(
"No cached assets available, falling back to manual lookup",
event_code="transcribable_fallback_manual_lookup",
topic=topic,
)
asset_query = find_new_transcribable_topic_assets(topic)
spawn_task = True
# select_for_update(of=("self",)) causes the row locking only to
# apply to the Asset table, rather than also locking joined item table
asset = (
asset_query.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if spawn_task:
# Spawn a task to populate the table for this topic
# We wait to do this until after getting an asset because otherwise there's a
# a chance all valid assets get grabbed by the task and our query will return
# nothing
structured_logger.debug(
"Spawned background task to populate cache",
event_code="transcribable_cache_population_triggered",
topic=topic,
)
populate_task = get_registered_task(
"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_topic"
)
populate_task.delay(topic.id)
return asset
def find_and_order_potential_transcribable_topic_assets(
topic: "concordia_models.Topic",
project_slug: str,
item_id: str,
asset_pk: int,
) -> "QuerySet[concordia_models.NextTranscribableTopicAsset]":
"""
Retrieve and prioritize cached transcribable assets based on proximity
and status.
Behavior:
Orders cached candidates from `NextTranscribableTopicAsset` to prefer:
- `NOT_STARTED` over `IN_PROGRESS` (via transient `unstarted` flag),
- same project,
- same item,
then by `sequence` and `asset_id` for stability.
Annotations added to each row (transient fields):
- unstarted (int): 1 if transcription status is `NOT_STARTED`, else 0.
- same_project (int): 1 if the candidate shares `project_slug`, else 0.
- same_item (int): 1 if the candidate shares `item_id`, else 0.
Args:
topic (concordia_models.Topic): Topic to filter by.
project_slug (str): Slug of the original asset's project.
item_id (str): Item identifier of the original asset.
asset_pk (int): Primary key of the original asset.
Returns:
QuerySet[concordia_models.NextTranscribableTopicAsset]: Prioritized
cached candidates.
"""
potential_next_assets = find_next_transcribable_topic_assets(topic)
potential_next_assets = potential_next_assets.annotate(
unstarted=Case(
When(
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
then=1,
),
default=0,
output_field=IntegerField(),
),
same_project=Case(
When(project_slug=project_slug, then=1),
default=0,
output_field=IntegerField(),
),
same_item=Case(
When(item_item_id=item_id, then=1),
default=0,
output_field=IntegerField(),
),
).order_by(
"-unstarted",
"-same_project",
"-same_item",
"sequence",
"asset_id",
)
return potential_next_assets
@transaction.atomic
def find_next_transcribable_topic_asset(
topic: "concordia_models.Topic",
project_slug: str,
item_id: str,
original_asset_id: int | None,
) -> "concordia_models.Asset | None":
"""
Retrieve the next best transcribable asset within a topic.
Priority for short-circuit selection (before cache and fallback):
1) If `item_id` is provided, return the next `NOT_STARTED` asset in
that item by sequence (strictly after the original asset when known).
2) If `project_slug` is provided, return the first `NOT_STARTED` asset
in that project (ordered by item id, then sequence), excluding the
current item to keep moving forward.
If none of the above match, fall back to the cache-backed path:
Attempts to retrieve a candidate from `NextTranscribableTopicAsset`. If
none is found, compute from `Asset` and trigger cache population.
After exhausting `NOT_STARTED` options, consider `IN_PROGRESS` assets in the
same item (strictly after the original when known).
Concurrency:
Uses `select_for_update(skip_locked=True, of=("self",))` to avoid
double-assignments across concurrent consumers.
Args:
topic (concordia_models.Topic): Topic to search within.
project_slug (str): Slug of the current project.
item_id (str): Identifier of the current item.
original_asset_id (int | None): Identifier of the asset just transcribed.
Returns:
concordia_models.Asset | None: A locked eligible asset, or None if
unavailable.
"""
# Resolve original context safely (int or digit-string only)
after_seq = None
orig = None
orig_item_id = None
orig_id_valid = isinstance(original_asset_id, int) or (
isinstance(original_asset_id, str) and original_asset_id.isdigit()
)
if orig_id_valid:
try:
orig = (
concordia_models.Asset.objects.select_related("item")
.only("id", "sequence", "item__item_id")
.get(pk=original_asset_id)
)
orig_item_id = getattr(orig.item, "item_id", None)
# Keep sequence handy for same-item gating in any path
after_seq = orig.sequence
except concordia_models.Asset.DoesNotExist:
orig = None
orig_item_id = None
after_seq = None
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.values(
"asset_id"
)
# Short-circuit: same item and NOT_STARTED after current sequence
if item_id:
qs = concordia_models.Asset.objects.filter(
item__project__topics=topic.id,
item__item_id=item_id,
item__published=True,
item__project__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
).exclude(pk__in=Subquery(reserved_asset_ids))
if orig_id_valid:
qs = qs.exclude(pk=original_asset_id)
if after_seq is not None and orig_item_id == item_id:
qs = qs.filter(
Q(sequence__gt=after_seq)
| (Q(sequence=after_seq) & Q(id__gt=int(original_asset_id)))
)
asset = (
qs.order_by("sequence", "id")
.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if asset:
return asset
# Short-circuit: same project and NOT_STARTED (topic-constrained)
if project_slug:
candidate = concordia_models.Asset.objects.filter(
item__project__topics=topic.id,
item__project__slug=project_slug,
item__published=True,
item__project__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
).exclude(pk__in=Subquery(reserved_asset_ids))
if orig_id_valid:
candidate = candidate.exclude(pk=original_asset_id)
if item_id:
candidate = candidate.exclude(item__item_id=item_id)
asset = (
candidate.order_by("item__item_id", "sequence", "id")
.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if asset:
return asset
# Cache-backed selection (NOT_STARTED anywhere), then manual fallback.
potential_next_assets = find_and_order_potential_transcribable_topic_assets(
topic, project_slug, item_id, original_asset_id
)
if orig_id_valid:
potential_next_assets = potential_next_assets.exclude(
asset_id=original_asset_id
)
if item_id:
potential_next_assets = potential_next_assets.exclude(item_item_id=item_id)
asset_id = (
potential_next_assets.select_for_update(skip_locked=True, of=("self",))
.values_list("asset_id", flat=True)
.first()
)
spawn_task = False
if asset_id:
asset_query = concordia_models.Asset.objects.filter(id=asset_id)
else:
structured_logger.debug(
"No cached assets matched, falling back to manual lookup",
event_code="transcribable_next_fallback_manual",
topic=topic,
)
spawn_task = True
asset_query = find_new_transcribable_topic_assets(topic)
if orig_id_valid:
asset_query = asset_query.exclude(pk=original_asset_id)
if item_id:
asset_query = asset_query.exclude(item__item_id=item_id)
# If we know the original's item/seq, keep moving forward within that item
if orig_item_id and after_seq is not None:
asset_query = asset_query.exclude(
Q(item__item_id=orig_item_id, sequence__lte=after_seq)
)
# Prefer same project and same item; if item_id is blank, prefer original's item
ref_item_id = item_id or orig_item_id
asset_query = asset_query.annotate(
unstarted=Case(
When(
transcription_status=concordia_models.TranscriptionStatus.NOT_STARTED,
then=1,
),
default=0,
output_field=IntegerField(),
),
same_project=Case(
When(item__project__slug=project_slug, then=1),
default=0,
output_field=IntegerField(),
),
same_item=Case(
When(item__item_id=ref_item_id, then=1),
default=0,
output_field=IntegerField(),
),
).order_by(
"-unstarted",
"-same_project",
"-same_item",
"sequence",
"id",
)
asset = (
asset_query.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if asset:
if spawn_task:
structured_logger.debug(
"Spawned background task to populate cache",
event_code="transcribable_next_cache_population",
topic=topic,
)
populate_task = get_registered_task(
"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_topic"
)
populate_task.delay(topic.id)
return asset
# Only now consider same-item IN_PROGRESS after current sequence
if item_id:
qs = concordia_models.Asset.objects.filter(
item__project__topics=topic.id,
item__item_id=item_id,
item__published=True,
item__project__published=True,
published=True,
transcription_status=concordia_models.TranscriptionStatus.IN_PROGRESS,
).exclude(pk__in=Subquery(reserved_asset_ids))
if orig_id_valid:
qs = qs.exclude(pk=original_asset_id)
if after_seq is not None and orig_item_id == item_id:
qs = qs.filter(
Q(sequence__gt=after_seq)
| (Q(sequence=after_seq) & Q(id__gt=int(original_asset_id)))
)
asset = (
qs.order_by("sequence", "id")
.select_for_update(skip_locked=True, of=("self",))
.select_related("item", "item__project")
.first()
)
if asset:
if spawn_task:
structured_logger.debug(
"Spawned background task to populate cache",
event_code="transcribable_next_cache_population",
topic=topic,
)
populate_task = get_registered_task(
"concordia.tasks.next_asset.transcribable.populate_next_transcribable_for_topic"
)
populate_task.delay(topic.id)
return asset
return None
def find_invalid_next_transcribable_topic_assets(
topic_id: int,
) -> "QuerySet[concordia_models.NextTranscribableTopicAsset]":
"""
Return cached rows that are invalid for transcription for a topic.
Behavior:
Identifies `NextTranscribableTopicAsset` rows that are no longer valid
because the underlying asset is neither `NOT_STARTED` nor
`IN_PROGRESS`, or because the asset is currently reserved.
Args:
topic_id (int): Identifier of the topic.
Returns:
QuerySet[concordia_models.NextTranscribableTopicAsset]: Distinct invalid
cache rows.
"""
reserved_asset_ids = concordia_models.AssetTranscriptionReservation.objects.filter(
asset__item__project__topics=topic_id
).values("asset_id")
status_filtered = concordia_models.NextTranscribableTopicAsset.objects.filter(
topic_id=topic_id
).exclude(
asset__transcription_status__in=[
concordia_models.TranscriptionStatus.NOT_STARTED,
concordia_models.TranscriptionStatus.IN_PROGRESS,
]
)
reserved_filtered = concordia_models.NextTranscribableTopicAsset.objects.filter(
topic_id=topic_id, asset_id__in=Subquery(reserved_asset_ids)
)
return (status_filtered | reserved_filtered).distinct()
================================================
FILE: concordia/validators.py
================================================
from django.conf import settings
from django.utils.translation import gettext_lazy as _
from .passwords.validators import ComplexityValidator
class DjangoPasswordsValidator(object):
"""
Wrapper for the django-passwords complexity validator which is compatible
with the Django 1.9+ password validation API
Because django-passwords is not compatible with Django 4+, the validator
has been integrated into the concordia.passwords module instead.
"""
message = _("Must be more complex (%s)")
code = "complexity"
def __init__(self):
self.validator = ComplexityValidator(settings.PASSWORD_COMPLEXITY)
def get_help_text(self):
return _("Your password fails to meet our complexity requirements.")
def validate(self, value, user=None):
return self.validator(value)
================================================
FILE: concordia/version.py
================================================
import functools
from setuptools_scm import get_version
@functools.lru_cache(maxsize=None)
def get_concordia_version():
return get_version()
================================================
FILE: concordia/views/README.md
================================================
## **init**.py
Initializes the `views` module and re-exports submodules for chained attribute access
such as `views.campaigns`. Also includes some basic views.
### Class-based Views
- **HomeView** - A `ListView` displaying featured campaigns on the homepage
### Function-based Views
- **healthz** - Returns a JSON response with system and application status
## accounts.py
Views related to user account management, including login, registration and profile updates
### Class-based Views
- **ConcordiaPasswordResetConfirmView** - Customized password reset confirmation
- **ConcordiaPasswordResetRequestView** - Customized password reset requests
- **ConcordiaRegistrationView** - Custom registration view with rate limiting
- **ConcordiaLoginView** - Login view with Turnstile challenge validation
- **AccountProfileView** - View for managing a user's profile and displaying contributions
- **AccountDeletionView** - View for users to delete their account (or anonymize it)
- **EmailReconfirmationView** - Handles confirming a user's changed email address
### Function-based Views
- **account_letter** - Generates and returns a PDF letter summarizing contributions
- **get_pages** - Renders a fragment of recent contributed pages
### Functions
- **registration_rate** - Rate-limit for failed registration attempts
## ajax.py
AJAX endpoints for dynamic client interactions
### Function-based Views
- **ajax_session_status** - Returns user-specific session data used by the frontend
- **ajax_messages** - Returns any queued messages for the current user
- **generate_ocr_transcription** - Generates a new transcription using OCR
- **rollback_transcription** - Reverts transcription to the previous version
- **rollforward_transcription** - Reverts the most recent transcription rollback
- **save_transcription** - Saves a new transcription
- **submit_transcription** - Marks a transcription as submitted for review
- **review_transcription** - Accepts or rejects a submitted transcription
- **submit_tags** - Updates the tag list for an asset
- **reserve_asset** - Manages reservation of an asset to prevent conflicts
### Functions
- **get_transcription_superseded** - Determines if the superseded transcription is valid
- **update_reservation** - Updates the timestamp for a reservation
- **obtain_reservation** - Creates a new reservation
## assets.py
Views for displaying asset detail pages and redirecting users to the next appropriate asset
### Class-based Views
- **AssetDetailView** - Displays the transcription interface for a single asset
### Function-based Views
- **redirect_to_next_asset** - Redirects to provided asset
- **redirect_to_next_reviewable_asset** - Finds and redirects to a reviewable asset
- **redirect_to_next_transcribable_asset** - Finds and redirects to a transcribable asset
- **redirect_to_next_reviewable_campaign_asset** - Finds and redirects to the a reviewable asset for a campaign
- **redirect_to_next_transcribable_campaign_asset** - Finds and redirects to a transcribable asset for a campaign
- **redirect_to_next_reviewable_topic_asset** - Finds and redirects to a reviewable asset for a topic
- **redirect_to_next_transcribable_topic_asset** - Finds and redirects to a transcribable asset for a topic
## campaigns.py
Views for listing campaigns, rendering campaign details, showing reports and filtering by reviewable status
### Class-based Views
- **CampaignListView** - Lists all active campaigns (unused)
- **CompletedCampaignListView** - Lists all completed and retired campaigns
- **CampaignTopicListView** - Primary active campaign list view; also includes active topics
- **CampaignDetailView** - Shows full details about a single campaign
- **FilteredCampaignDetailView** - Variant of `CampaignDetailView` that applies filtering based on the user
- **ReportCampaignView** - Displays a campaign report summarizing stats such as asset counts and contributors
## decorators.py
Custom decorators used by views
### Functions
- **default_cache_control** - Applies default public caching headers for pages that don't vary per user
- **user_cache_control** - Applies public caching headers with variation for logged-in users
- **validate_anonymous_user** - Validates anonymous users via Turnstile before processing requests
- **reserve_rate** - Returns a rate-limit value for unauthenticated users for reserving assets
- **next_asset_rate** - Returns a rate-limit value for unauthenicated users for next\_\*\_asset views
## items.py
Views for displaying individual item detail pages
### Class-based Views
- **ItemDetailView** - Displays a paginated list of assets within an item
- **FilteredItemDetailView** - Variant of `ItemDetailView` that applies filtering based on the user
## maintenance_mode.py
Views for toggling the site's maintenance mode. Only accessible to superusers
### Function-based Views
- **maintenance_mode_off** - Disables maintenance mode
- **maintenance_mode_on** - Enables maintenance mode
- **maintenance_mode_frontend_available** - Enables access to the frontend for staff while in maintenance mode
- **maintenance_mode_frontend_unavailable** - Disables access to the frontend for staff while in maintenance mode
## projects.py
Views for displaying project detail pages
### Class-based Views
- **ProjectDetailView** - Displays a project and its items
- **FilteredProjectDetailView** - Variant of `ProjectDetailView` that applies filtering based on the user
## rate_limit.py
Custom handler for responding to requests that exceed rate limits
### Function-based Views
- **ratelimit_view** - Returns a 429 response when a user is rate-limited
## simple_pages.py
Views and redirects for rendering static pages stored in the database
### Function-based Views
- **simple_page** - Renders a simple static page from the database
- **about_simple_page** - Renders the "about" simple page, which includes some additional data
### Class-based Views
- **HelpCenterRedirectView** - Redirects old help center URLs to new equivalents
- **HelpCenterSpanishRedirectView** - Redirects old Spanish help center URLs to new equivalents
## topics.py
View for displaying a topic's detail page
### Class-based Views
- **TopicDetailView** - Displays a topic's associated projects
## utils.py
Utility functions, constants and mixins used throughout the views module
### Constants
- **ASSETS_PER_PAGE** - Default number of assets to show per page
- **PROJECTS_PER_PAGE** - Default number of projects to show per page
- **ITEMS_PER_PAGE** - Default number of items to show per page
- **URL_REGEX** - Regular expression used to detect URLs in transcription text
- **MESSAGE_LEVEL_NAMES** - Dictionary mapping Django message levels to lowercase names
### Functions
- **\_get_pages** - Returns a queryset of assets a user has worked on
- **calculate_asset_stats** - Adds contributor and transcription status to the provided assets
- **annotate_children_with_progress_stats** - Annotates a list of objects with progress information
### Classes
- **AnonymousUserValidationCheckMixin** - Requires anonymous users to pass Turnstile validation
## visualization.py
Views for displaying visualizations
### Classes
- **VisualizationDataView** - Returns JSON representing the visualization `name`
================================================
FILE: concordia/views/__init__.py
================================================
import json
import logging
import os
from time import time
from django.conf import settings
from django.http import HttpResponse
from django.utils.decorators import method_decorator
from django.views.decorators.cache import never_cache
from django.views.generic import ListView
from concordia.models import Banner, Campaign, CarouselSlide
from concordia.version import get_concordia_version
# These imports are required to make chainted attribute access like, e.g.,
# views.campaigns.CampaignDetailView work correctly
from . import (
accounts, # noqa: F401
ajax, # noqa: F401
assets, # noqa: F401
campaigns, # noqa: F401
items, # noqa: F401
maintenance_mode, # noqa: F401
projects, # noqa: F401
rate_limit, # noqa: F401
simple_pages, # noqa: F401
topics, # noqa: F401
visualizations, # noqa: F401
)
from .decorators import default_cache_control
logger = logging.getLogger(__name__)
@never_cache
def healthz(request):
status = {
"current_time": time(),
"load_average": os.getloadavg(),
"debug": settings.DEBUG,
}
# We don't want to query a large table but we do want to hit the database
# at last once:
status["database_has_data"] = Campaign.objects.count() > 0
status["application_version"] = get_concordia_version()
return HttpResponse(content=json.dumps(status), content_type="application/json")
@method_decorator(default_cache_control, name="dispatch")
class HomeView(ListView):
template_name = "home.html"
queryset = (
Campaign.objects.published()
.listed()
.filter(display_on_homepage=True)
.order_by("ordering", "title")
)
context_object_name = "campaigns"
def get_context_data(self, *args, **kwargs):
ctx = super().get_context_data(*args, **kwargs)
banner = Banner.objects.filter(active=True).first()
if banner is not None:
ctx["banner"] = banner
ctx["slides"] = CarouselSlide.objects.published().order_by("ordering")
if ctx["slides"]:
ctx["firstslide"] = ctx["slides"][0]
return ctx
================================================
FILE: concordia/views/accounts.py
================================================
import gc
import logging
import tracemalloc
import uuid
from smtplib import SMTPException
from typing import Any, Optional, Type
from django.conf import settings
from django.contrib import messages
from django.contrib.auth import logout
from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin
from django.contrib.auth.views import (
LoginView,
PasswordResetConfirmView,
PasswordResetView,
)
from django.contrib.sites.shortcuts import get_current_site
from django.core import signing
from django.core.exceptions import ValidationError
from django.core.mail import send_mail
from django.core.paginator import Paginator
from django.db.models import Sum
from django.forms import Form
from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, JsonResponse
from django.shortcuts import redirect
from django.template import loader
from django.template.loader import render_to_string
from django.urls import reverse_lazy
from django.utils.decorators import method_decorator
from django.utils.translation import gettext_lazy as _
from django.views.decorators.cache import never_cache
from django.views.generic import FormView, ListView, TemplateView
from django_ratelimit.decorators import ratelimit
from django_registration.backends.activation.views import RegistrationView
from weasyprint import HTML
from concordia.forms import (
AccountDeletionForm,
ActivateAndSetPasswordForm,
AllowInactivePasswordResetForm,
TurnstileForm,
UserLoginForm,
UserNameForm,
UserProfileForm,
UserRegistrationForm,
)
from concordia.logging import ConcordiaLogger
from concordia.models import Campaign, ConcordiaUser, UserProfileActivity
from .utils import _get_pages
logger = logging.getLogger(__name__)
structured_logger = ConcordiaLogger.get_logger(__name__)
class ConcordiaPasswordResetConfirmView(PasswordResetConfirmView):
"""
Confirm a password reset and automatically log in the user.
Extends Django’s built-in
[PasswordResetConfirmView](https://docs.djangoproject.com/en/stable/topics/auth/default/#django.contrib.auth.views.PasswordResetConfirmView)
to use a custom form and enable automatic login after a successful reset.
Attributes:
post_reset_login (bool): Whether to log the user in after resetting
the password.
form_class (Form): The form used to set the new password and activate
the account.
Returns:
response (HttpResponse): Renders the password reset confirmation page or
redirects after successful password change and login.
"""
post_reset_login: bool = True
form_class: type[Form] = ActivateAndSetPasswordForm
class ConcordiaPasswordResetRequestView(PasswordResetView):
"""
Request a password reset, supporting inactive users.
Extends Django’s built-in
[`PasswordResetView`](https://docs.djangoproject.com/en/stable/topics/auth/default/#django.contrib.auth.views.PasswordResetView)
to use a custom form that allows inactive users to reset their password
and activate their account in one step.
Attributes:
form_class (Form): The form used to validate and process the password
reset request.
Returns:
response (HttpResponse): Renders the password reset form or redirects
after successful submission.
"""
form_class: type[Form] = AllowInactivePasswordResetForm
def registration_rate(group: str, request: HttpRequest) -> Optional[str]:
"""
Determine the throttling rate for registration attempts.
Used with the
[ratelimit](https://django-ratelimit.readthedocs.io/en/stable/usage.html#ratelimit)
decorator from `django-ratelimit` to dynamically adjust the request rate based
on form validation.
If the submitted form is invalid, limits requests to 10 per hour. If the
form is valid, allows the request without throttling.
Args:
group (str): The rate limit group name. Example: `"registration"`
request (HttpRequest): The request containing registration form data.
Returns:
rate (str or None): The rate limit string (e.g., "10/h") if the form is
invalid; otherwise `None` to indicate no throttling.
"""
registration_form = UserRegistrationForm(request.POST)
user = getattr(request, "user", None)
if registration_form.is_valid():
structured_logger.debug(
"Registration form valid.",
event_code="registration_rate_ok",
user=user,
)
return None
else:
structured_logger.debug(
"Registration form invalid, throttling.",
event_code="registration_rate_throttle",
user=user,
)
return "10/h"
@method_decorator(never_cache, name="dispatch")
@method_decorator(
ratelimit(
group="registration",
key="header:cf-connecting-ip",
rate=registration_rate,
method="POST",
block=settings.RATELIMIT_BLOCK,
),
name="post",
)
class ConcordiaRegistrationView(RegistrationView):
"""
User registration view with rate limiting.
Extends
[django_registration.views.RegistrationView](https://django-registration.readthedocs.io/en/stable/views.html#django_registration.views.RegistrationView)
to apply a POST-specific rate limit using the
[django-ratelimit](https://django-ratelimit.readthedocs.io/en/stable/usage.html#ratelimit)
decorator. This protects against abuse by restricting failed registration attempts
while allowing valid submissions to proceed freely.
Attributes:
form_class (Form): The form used to collect and validate user registration
data. Example: `UserRegistrationForm`.
Returns:
response (HttpResponse): Renders the registration form or redirects after
successful registration.
"""
form_class: Type[Form] = UserRegistrationForm
@method_decorator(never_cache, name="dispatch")
class ConcordiaLoginView(LoginView):
"""
Login view with Turnstile validation.
Extends Django's
[LoginView](https://docs.djangoproject.com/en/stable/topics/auth/default/#django.contrib.auth.views.LoginView)
to integrate Turnstile validation during POST requests.
Attributes:
form_class (Form): The login form used to authenticate users.
Returns:
response (HttpResponse): The rendered login form or redirect response,
depending on the validation outcome.
Return Behavior:
- On GET: Renders the login form with the embedded Turnstile widget.
- On POST:
- If both login and Turnstile succeed: redirects to the next page.
- If Turnstile fails: returns the login form with an error message.
- If the login form is invalid: returns the form with validation errors.
"""
form_class = UserLoginForm
def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
structured_logger.debug(
"Login POST received.",
event_code="login_post_entry",
user=request.user,
)
form = self.get_form()
if form.is_valid():
turnstile_form = TurnstileForm(request.POST)
if turnstile_form.is_valid():
structured_logger.debug(
"Login and Turnstile ok.",
event_code="login_success",
user=request.user,
)
return self.form_valid(form)
else:
structured_logger.warning(
"Turnstile failed for login.",
event_code="login_turnstile_failed",
reason="Turnstile validation failed",
reason_code="turnstile_failed",
user=request.user,
)
form.add_error(
None, "Unable to validate. Please login or complete the challenge."
)
return self.form_invalid(form)
else:
structured_logger.debug(
"Login form invalid.",
event_code="login_form_invalid",
user=request.user,
)
return self.form_invalid(form)
def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
ctx = super().get_context_data(**kwargs)
ctx["turnstile_form"] = TurnstileForm(auto_id=False)
structured_logger.debug(
"Added Turnstile form to context.",
event_code="login_context_turnstile",
user=self.request.user,
)
return ctx
@login_required
@never_cache
def account_letter(request: HttpRequest) -> HttpResponse:
"""
Generate and return a PDF letter summarizing a user's contributions.
This view creates a service letter for the logged-in user, summarizing their
transcription and review activity. It uses an HTML template rendered with
contribution data and converts it to a PDF using WeasyPrint.
Requires the user to be authenticated.
Returns:
response (HttpResponse): A PDF response with content type
`application/pdf` and a `Content-Disposition` header set to download
as `letter.pdf`.
Return Behavior:
- The generated PDF includes:
- User's name and join date.
- Total transcriptions and reviews.
- List of assets the user contributed to.
"""
structured_logger.debug(
"Building account letter.",
event_code="account_letter_start",
user=request.user,
)
image_url = "file://{0}/{1}/img/logo.jpg".format(
settings.SITE_ROOT_DIR, settings.STATIC_ROOT
)
user_profile_activity = UserProfileActivity.objects.filter(user=request.user)
aggregate_sums = user_profile_activity.aggregate(
Sum("review_count"), Sum("transcribe_count")
)
asset_list = _get_pages(request)
context = {
"user": request.user,
"join_date": request.user.date_joined,
"total_reviews": aggregate_sums["review_count__sum"],
"total_transcriptions": aggregate_sums["transcribe_count__sum"],
"image_url": image_url,
"asset_list": asset_list,
}
template = loader.get_template("documents/service_letter.html")
text = template.render(context)
html = HTML(string=text)
response = HttpResponse(
content=html.write_pdf(variant="pdf/ua-1"), content_type="application/pdf"
)
response["Content-Disposition"] = "attachment; filename=letter.pdf"
structured_logger.debug(
"Account letter generated.",
event_code="account_letter_success",
user=request.user,
total_reviews=aggregate_sums["review_count__sum"],
total_transcriptions=aggregate_sums["transcribe_count__sum"],
asset_count=len(asset_list),
)
return response
@login_required
@never_cache
def get_pages(request: HttpRequest) -> JsonResponse:
"""
Return a paginated and filtered list of the user's contributed assets as HTML.
Retrieves assets the current user has worked on, applies pagination, and
optionally filters by campaign, activity type, status, and date range. Renders
the results into a fragment of HTML for use in dynamic page updates.
Requires the user to be authenticated.
Args:
request (HttpRequest): The request from the authenticated user.
Request Parameters:
- `page` (int): Page number to display. Example: `2`
- `campaign` (int): Filter by campaign ID. Example: `17`
- `status` (list[str]): Filter by asset statuses. Example:
`["in_progress", "submitted"]`
- `activity` (str): Filter by activity type. Example: `"transcribe"`
- `order_by` (str): Sort order. Example: `"date-descending"`
- `start` (str): Start date in YYYY-MM-DD format. Example: `"2023-01-01"`
- `end` (str): End date in YYYY-MM-DD format. Example: `"2023-12-31"`
Returns:
response (JsonResponse): A JSON object containing rendered HTML for recent
contributed pages.
Response Format - Success:
- `content` (str): Rendered HTML for recent pages.
Example:
```json
{
"content": "...
"
}
```
"""
tracemalloc.start()
gc.collect()
structured_logger.debug(
"Fetching recent pages.",
event_code="recent_pages_entry",
user=request.user,
page=request.GET.get("page", "1"),
campaign=request.GET.get("campaign"),
activity=request.GET.get("activity"),
)
asset_list = _get_pages(request)
paginator = Paginator(asset_list, 30) # Show 30 assets per page.
page_number = int(request.GET.get("page", "1"))
context = {
"paginator": paginator,
"page_obj": paginator.get_page(page_number),
"is_paginated": True,
"recent_campaigns": Campaign.objects.filter(
project__item__asset__in=asset_list.values("pk")
)
.distinct()
.order_by("title")
.values("pk", "title"),
}
for param in ("activity", "end", "order_by", "start", "statuses"):
context[param] = request.GET.get(param, None)
campaign = request.GET.get("campaign", None)
context["statuses"] = request.GET.getlist("status")
if campaign is not None:
context["campaign"] = Campaign.objects.get(pk=int(campaign))
data = {}
data["content"] = loader.render_to_string(
"fragments/recent-pages.html", context, request=request
)
# Capture memory stats
current, peak = tracemalloc.get_traced_memory()
current_mb = current / 1024 / 1024
peak_mb = peak / 1024 / 1024
# For immediate visibility
structured_logger.info(
"Recent pages rendered.",
event_code="recent_pages_success",
user=request.user,
assets=asset_list.count(),
num_pages=paginator.num_pages,
page=page_number,
memory_current_mb=round(current_mb, 2),
memory_peak_mb=round(peak_mb, 2),
)
tracemalloc.stop()
return JsonResponse(data)
@method_decorator(never_cache, name="dispatch")
class AccountProfileView(LoginRequiredMixin, FormView, ListView):
"""
Display and update user account profile and contribution history.
Combines functionality from:
- [LoginRequiredMixin](https://docs.djangoproject.com/en/stable/topics/auth/default/#the-loginrequiredmixin-mixin)
- [FormView](https://docs.djangoproject.com/en/stable/ref/class-based-views/generic-editing/#formview)
- [ListView](https://docs.djangoproject.com/en/stable/ref/class-based-views/generic-display/#listview)
Allows authenticated users to:
- Update their email address and name
- View a paginated list of assets they have contributed to
- See aggregate statistics on their transcription and review activity
Email changes require confirmation unless the setting
`REQUIRE_EMAIL_RECONFIRMATION` is False.
Attributes:
template_name (str): Template used to render the profile page.
Example: `"account/profile.html"`
form_class (Form): Form used to update the user's email address.
Example: `UserProfileForm`
success_url (str): Redirect URL after successful form submission.
Example: `"/accounts/profile/"`
allow_empty (bool): Whether to render the page if the user has no
contributions. Default is `True`
paginate_by (int): Number of contributed assets to show per page.
Default is `30`.
reconfirmation_email_body_template (str): Path to the plain text email
body template. Example: `"emails/email_reconfirmation_body.txt"`
reconfirmation_email_subject_template (str): Path to the email subject
template. Example: `"emails/email_reconfirmation_subject.txt"`
Returns:
response (HttpResponse): The rendered profile page with contribution data
or a redirect to `#account` after successful form submission.
Request Parameters:
- `page` (int): Page number. Example: `1`
- `campaign` (int): Campaign filter. Example: `42`
- `activity` (str): Activity type filter. Example: `"transcribe"`
- `status` (list[str]): Asset statuses. Example: `["completed"]`
- `start` (str): Start date in YYYY-MM-DD format. Example: `"2023-01-01"`
- `end` (str): End date in YYYY-MM-DD format. Example: `"2023-12-31"`
- `order_by` (str): Sort field. Example: `"date-descending"`
- `tab` (str): Selected tab. Example: `"account"`
"""
template_name: str = "account/profile.html"
form_class: Type[Form] = UserProfileForm
success_url = reverse_lazy("user-profile")
reconfirmation_email_body_template: str = "emails/email_reconfirmation_body.txt"
reconfirmation_email_subject_template: str = (
"emails/email_reconfirmation_subject.txt"
)
# This view will list the assets which the user has contributed to
# along with their most recent action on each asset. This will be
# presented in the template as a standard paginated list of Asset
# instances with annotations
allow_empty: bool = True
paginate_by: int = 30
def post(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
structured_logger.debug(
"Profile POST received.",
event_code="profile_post_entry",
user=request.user,
)
self.object_list = self.get_queryset()
if "submit_name" in request.POST:
form = UserNameForm(request.POST)
if form.is_valid():
user = ConcordiaUser.objects.get(id=request.user.id)
user.first_name = form.cleaned_data["first_name"]
user.last_name = form.cleaned_data["last_name"]
user.save()
structured_logger.debug(
"Updated profile name.",
event_code="profile_name_updated",
user=request.user,
)
return redirect("user-profile")
else:
return super().post(request, *args, **kwargs)
def get_queryset(self) -> Any:
structured_logger.debug(
"Fetching pages for profile.",
event_code="profile_get_queryset",
user=self.request.user,
)
return _get_pages(self.request)
def get_context_data(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
ctx = super().get_context_data(*args, **kwargs)
page = self.request.GET.get("page", None)
campaign = self.request.GET.get("campaign", None)
activity = self.request.GET.get("activity", None)
status_list = self.request.GET.getlist("status")
start = self.request.GET.get("start", None)
end = self.request.GET.get("end", None)
order_by = self.request.GET.get("order_by", None)
if any([activity, campaign, page, status_list, start, end, order_by]):
ctx["active_tab"] = "recent"
if status_list:
ctx["status_list"] = status_list
ctx["order_by"] = self.request.GET.get("order_by", "date-descending")
elif "active_tab" not in ctx:
ctx["active_tab"] = self.request.GET.get("tab", "contributions")
ctx["activity"] = activity
if end is not None:
ctx["end"] = end
ctx["order_by"] = order_by
if start is not None:
ctx["start"] = start
ctx["valid"] = self.request.session.pop("valid", None)
user = self.request.user
concordia_user = ConcordiaUser.objects.get(id=user.id)
user_profile_activity = UserProfileActivity.objects.filter(user=user).order_by(
"campaign__title"
)
ctx["user_profile_activity"] = user_profile_activity
aggregate_sums = user_profile_activity.aggregate(
Sum("review_count"), Sum("transcribe_count"), Sum("asset_count")
)
ctx["totalReviews"] = aggregate_sums["review_count__sum"]
ctx["totalTranscriptions"] = aggregate_sums["transcribe_count__sum"]
ctx["pages_worked_on"] = aggregate_sums["asset_count__sum"]
if ctx["totalReviews"] is not None:
ctx["totalCount"] = ctx["totalReviews"] + ctx["totalTranscriptions"]
ctx["unconfirmed_email"] = concordia_user.get_email_for_reconfirmation()
ctx["name_form"] = UserNameForm()
structured_logger.debug(
"Profile context ready.",
event_code="profile_context_ready",
user=self.request.user,
total_reviews=ctx["totalReviews"],
total_transcriptions=ctx["totalTranscriptions"],
pages_worked=ctx["pages_worked_on"],
)
return ctx
def get_initial(self) -> dict[str, Any]:
initial = super().get_initial()
initial["email"] = self.request.user.email
return initial
def get_form_kwargs(self) -> dict[str, Any]:
# We'll expose the request object to the form so we can validate that an
# email is not in use:
kwargs = super().get_form_kwargs()
kwargs["request"] = self.request
return kwargs
def form_valid(self, form: Form) -> HttpResponse:
user = self.request.user
new_email = form.cleaned_data["email"]
structured_logger.info(
"Profile email update submitted.",
event_code="profile_email_update",
user=user,
require_reconfirm=settings.REQUIRE_EMAIL_RECONFIRMATION,
)
# This is annoying, but there's no better way to get the proxy model here
# without being hacky (changing user.__class__ directly.)
# Every method (such as using a user profile) would incur the same
# database request.
concordia_user = ConcordiaUser.objects.get(id=user.id)
if settings.REQUIRE_EMAIL_RECONFIRMATION:
concordia_user.set_email_for_reconfirmation(new_email)
structured_logger.debug(
"Email set for reconfirmation.",
event_code="email_reconfirm_set",
user=user,
new_email=new_email,
)
self.send_reconfirmation_email(concordia_user)
else:
concordia_user.email = new_email
concordia_user.full_clean()
concordia_user.save()
concordia_user.delete_email_for_reconfirmation()
structured_logger.debug(
"Email updated without reconfirmation.",
event_code="email_updated_no_reconfirm",
user=user,
new_email=new_email,
)
self.request.session["valid"] = True
return super().form_valid(form)
def form_invalid(self, form: Form) -> HttpResponse:
structured_logger.debug(
"Profile form invalid.",
event_code="profile_form_invalid",
user=self.request.user,
)
self.request.session["valid"] = False
return self.render_to_response(
self.get_context_data(form=form, active_tab="account")
)
def get_success_url(self) -> str:
# automatically open the Account Settings tab
return "{}#account".format(super().get_success_url())
def get_reconfirmation_email_context(self, confirmation_key: str) -> dict[str, Any]:
return {
"confirmation_key": confirmation_key,
"expiration_days": settings.EMAIL_RECONFIRMATION_DAYS,
"site": get_current_site(self.request),
}
def send_reconfirmation_email(self, user: ConcordiaUser) -> None:
confirmation_key = user.get_email_reconfirmation_key()
context = self.get_reconfirmation_email_context(confirmation_key)
context["user"] = user
subject = render_to_string(
template_name=self.reconfirmation_email_subject_template,
context=context,
request=self.request,
)
# Ensure subject is a single line
subject = "".join(subject.splitlines())
message = render_to_string(
template_name=self.reconfirmation_email_body_template,
context=context,
request=self.request,
)
try:
structured_logger.info(
"Sending reconfirmation email.",
event_code="email_reconfirm_send_start",
user=user,
email=user.get_email_for_reconfirmation(),
)
send_mail(
subject,
message=message,
from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[user.get_email_for_reconfirmation()],
)
structured_logger.debug(
"Reconfirmation email sent.",
event_code="email_reconfirm_send_success",
user=user,
email=user.get_email_for_reconfirmation(),
)
except SMTPException:
logger.exception(
"Unable to send email reconfirmation to %s",
user.get_email_for_reconfirmation(),
)
structured_logger.exception(
"Reconfirmation email send failed.",
event_code="email_reconfirm_send_failed",
reason="SMTPException",
reason_code="smtp_error",
user=user,
email=user.get_email_for_reconfirmation(),
)
messages.error(
self.request,
_("Email confirmation could not be sent."),
)
@method_decorator(never_cache, name="dispatch")
class AccountDeletionView(LoginRequiredMixin, FormView):
"""
Handle user-initiated account deletion.
Extends:
- [LoginRequiredMixin](https://docs.djangoproject.com/en/stable/topics/auth/default/#the-loginrequiredmixin-mixin)
- [FormView](https://docs.djangoproject.com/en/stable/ref/class-based-views/generic-editing/#formview)
Provides a confirmation form for deleting the user's account. If the user has
contributed transcriptions, their data is anonymized instead of being deleted.
Otherwise, the account is fully removed. A confirmation email is sent to the
user's address before deletion. After deletion, the user is logged out.
Requires the user to be authenticated.
Attributes:
template_name (str): Template used to render the confirmation form.
Example: `"account/account_deletion.html"`
form_class (Form): Form used to confirm account deletion.
Example: `AccountDeletionForm`
success_url (str): URL to redirect to after deletion.
Example: `"/"`
email_body_template (str): Template for the body of the confirmation email.
Example: `"emails/delete_account_body.txt"`
email_subject_template (str): Template for the subject of the confirmation
email. Example: `"emails/delete_account_subject.txt"`
Returns:
response (HttpResponse): A redirect to the homepage after deletion, or a
rendered form with errors if validation fails.
Return Behavior:
- If the user confirms deletion and has transcriptions: anonymizes their
account and logs them out.
- If the user has no transcriptions: deletes the account entirely and logs
them out.
- If the form is invalid: re-renders the confirmation form with errors.
"""
template_name: str = "account/account_deletion.html"
form_class: Type[Form] = AccountDeletionForm
success_url: str = reverse_lazy("homepage")
email_body_template: str = "emails/delete_account_body.txt"
email_subject_template: str = "emails/delete_account_subject.txt"
def get_form_kwargs(self) -> dict[str, Any]:
# We expose the request object to the form so we can use it
# to log the user out after deletion
kwargs = super().get_form_kwargs()
kwargs["request"] = self.request
return kwargs
def form_valid(self, form: Form) -> HttpResponse:
structured_logger.info(
"Account deletion confirmed.",
event_code="account_delete_confirmed",
user=form.request.user,
)
self.delete_user(form.request.user, form.request)
return super().form_valid(form)
def delete_user(self, user: ConcordiaUser, request: HttpRequest) -> None:
logger.info("Deletion request for %s", user)
structured_logger.info(
"Processing account deletion.",
event_code="account_delete_start",
user=user,
)
email = user.email
if user.transcription_set.exists():
logger.info("Anonymizing %s", user)
structured_logger.info(
"Anonymizing user account.",
event_code="account_anonymize",
user=user,
)
user.username = "Anonymized %s" % uuid.uuid4()
user.first_name = ""
user.last_name = ""
user.email = ""
user.set_unusable_password()
user.is_staff = False
user.is_superuser = False
user.is_active = False
user.save()
else:
logger.info("Deleting %s", user)
structured_logger.info(
"Deleting user account.",
event_code="account_delete",
user=user,
)
user.delete()
self.send_deletion_email(email)
logout(request)
structured_logger.info(
"Account deletion complete.",
event_code="account_delete_complete",
user=user,
)
def send_deletion_email(self, email: str) -> None:
context = {}
subject = render_to_string(
template_name=self.email_subject_template,
context=context,
request=self.request,
)
# Ensure subject is a single line
subject = "".join(subject.splitlines())
message = render_to_string(
template_name=self.email_body_template,
context=context,
request=self.request,
)
try:
structured_logger.info(
"Sending deletion email.",
event_code="account_delete_email_send_start",
user=self.request.user,
email=email,
)
send_mail(
subject,
message=message,
from_email=settings.DEFAULT_FROM_EMAIL,
recipient_list=[email],
)
structured_logger.debug(
"Deletion email sent.",
event_code="account_delete_email_send_success",
user=self.request.user,
email=email,
)
except SMTPException:
logger.exception(
"Unable to send account deletion email to %s",
email,
)
structured_logger.exception(
"Deletion email send failed.",
event_code="account_delete_email_send_failed",
reason="SMTPException",
reason_code="smtp_error",
user=self.request.user,
email=email,
)
messages.error(
self.request,
_("Email confirmation of deletion could not be sent."),
)
class EmailReconfirmationView(TemplateView):
"""
Handle email reconfirmation via a signed URL token.
Extends:
- [TemplateView](https://docs.djangoproject.com/en/stable/ref/class-based-views/base/#templateview)
Validates a confirmation key sent to the user's new email address during
an address change. If valid and not expired, applies the email update. If
invalid, expired or mismatched, renders an error message.
Attributes:
template_name (str): Template rendered if the confirmation fails.
Example: `"account/email_reconfirmation_failed.html"`
success_url (str): URL to redirect to on success.
Example: `"/accounts/profile/#account"`
BAD_USERNAME_MESSAGE (str): Error if the user account cannot be found.
BAD_EMAIL_MESSAGE (str): Error if the email does not match expectations.
EXPIRED_MESSAGE (str): Error if the key is expired.
INVALID_KEY_MESSAGE (str): Error if the key signature is invalid.
Returns:
response (HttpResponse): Redirects to the profile page with `#account`
on success, or renders the failure template with error details.
Request Parameters:
confirmation_key (str): A signed token containing the username and new
email. Example: `"ZHVtbXl1c2VyOnNvbWVvbmVAZXhhbXBsZS5jb20="`
"""
success_url = reverse_lazy("user-profile")
template_name = "account/email_reconfirmation_failed.html"
BAD_USERNAME_MESSAGE: str = _("The account you attempted to confirm is invalid.")
BAD_EMAIL_MESSAGE: str = _("The email you attempted to confirm is invalid.")
EXPIRED_MESSAGE: str = _(
"The confirmation key you provided is expired. Email confirmation links "
"expire after 7 days. If your key is expired, you will need to re-enter "
"your new email address"
)
INVALID_KEY_MESSAGE: str = _(
"The confirmation key you provided is invalid. Email confirmation links "
"expire after 7 days. If your key is expired, you will need to re-enter "
"your new email address."
)
def get_success_url(self) -> str:
return "{}#account".format(self.success_url)
def get(self, *args: Any, **kwargs: Any) -> HttpResponse:
extra_context = {}
try:
structured_logger.debug(
"Email reconfirmation GET received.",
event_code="email_reconfirm_entry",
)
self.confirm(*args, **kwargs)
except ValidationError as exc:
structured_logger.warning(
"Email reconfirmation failed.",
event_code="email_reconfirm_failed",
reason=str(exc.message),
reason_code=str(exc.code),
)
extra_context["reconfirmation_error"] = {
"message": exc.message,
"code": exc.code,
"params": exc.params,
}
context_data = self.get_context_data()
context_data.update(extra_context)
return self.render_to_response(context_data, status=403)
else:
structured_logger.debug(
"Email reconfirmation ok.",
event_code="email_reconfirm_success",
)
return HttpResponseRedirect(self.get_success_url())
def confirm(self, *args: Any, **kwargs: Any) -> ConcordiaUser:
username, email = self.validate_key(kwargs.get("confirmation_key"))
user = self.get_user(username)
if not user.validate_reconfirmation_email(email):
raise ValidationError(self.BAD_EMAIL_MESSAGE, code="bad_email") from None
try:
user.email = email
user.full_clean()
except ValidationError:
raise ValidationError(self.BAD_EMAIL_MESSAGE, code="bad_email") from None
user.save()
user.delete_email_for_reconfirmation()
structured_logger.info(
"Email reconfirmed and applied.",
event_code="email_reconfirm_applied",
user=user,
new_email=email,
)
return user
def validate_key(self, confirmation_key: str) -> tuple[str, str]:
try:
context = signing.loads(
confirmation_key, max_age=settings.EMAIL_RECONFIRMATION_TIMEOUT
)
return context["username"], context["email"]
except signing.SignatureExpired as exc:
raise ValidationError(self.EXPIRED_MESSAGE, code="expired") from exc
except signing.BadSignature as exc:
raise ValidationError(
self.INVALID_KEY_MESSAGE,
code="invalid_key",
params={"confirmation_key": confirmation_key},
) from exc
def get_user(self, username: str) -> ConcordiaUser:
try:
user = ConcordiaUser.objects.get(username=username)
structured_logger.debug(
"Loaded user for reconfirmation.",
event_code="email_reconfirm_user_loaded",
user=user,
)
return user
except ConcordiaUser.DoesNotExist as exc:
structured_logger.warning(
"User not found for reconfirmation.",
event_code="email_reconfirm_user_missing",
reason="User does not exist",
reason_code="user_missing",
)
raise ValidationError(
self.BAD_USERNAME_MESSAGE, code="bad_username"
) from exc
================================================
FILE: concordia/views/ajax.py
================================================
import logging
import re
from time import time
from typing import Union
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.contrib.messages import get_messages
from django.core.exceptions import ValidationError
from django.db import connection
from django.db.transaction import atomic
from django.http import HttpRequest, HttpResponse, JsonResponse
from django.shortcuts import get_object_or_404
from django.urls import reverse
from django.utils.timezone import now
from django.views.decorators.cache import cache_control, never_cache
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from django_ratelimit.decorators import ratelimit
from concordia.exceptions import RateLimitExceededError
from concordia.logging import ConcordiaLogger
from concordia.models import (
Asset,
AssetTranscriptionReservation,
ConcordiaUser,
Tag,
Transcription,
UserAssetTagCollection,
)
from concordia.signals.signals import (
reservation_obtained,
reservation_released,
)
from concordia.utils import (
get_anonymous_user,
get_or_create_reservation_token,
)
from concordia.utils.constants import MESSAGE_LEVEL_NAMES, URL_REGEX
from configuration.utils import configuration_value
from exporter.utils import remove_unacceptable_characters
from .decorators import reserve_rate, validate_anonymous_user
logger = logging.getLogger(__name__)
structured_logger = ConcordiaLogger.get_logger(__name__)
@cache_control(private=True, max_age=settings.DEFAULT_PAGE_TTL)
@csrf_exempt
def ajax_session_status(request: HttpRequest) -> JsonResponse:
"""
Return a JSON object describing the authenticated session state.
If the user is authenticated, this includes a truncated username and
navigational links to profile, logout, and admin pages (if staff or superuser).
If the user is anonymous, returns an empty dictionary.
Args:
request (HttpRequest): The HTTP request initiating the session status check.
Returns:
response (JsonResponse): A dictionary containing either session information
or empty data.
Response Format - Success:
- `username` (str): The first 15 characters of the user's username.
- `links` (list[dict]): A list of links relevant to the user's session.
- `title` (str): The label for the link (e.g., "Profile", "Logout").
- `url` (str): The absolute URL for the link.
Example:
```json
// If the user is authenticated:
{
"username": "johndoe",
"links": [
{"title": "Profile", "url": "https://example.com/accounts/profile/"},
{"title": "Logout", "url": "https://example.com/accounts/logout/"}
]
}
// If the user is anonymous:
{}
```
"""
user = request.user
if user.is_anonymous:
res = {}
else:
links = [
{
"title": "Profile",
"type": "link",
"url": request.build_absolute_uri(reverse("user-profile")),
}
]
if user.is_superuser or user.is_staff:
links.append(
{
"title": "Admin Area",
"type": "link",
"url": request.build_absolute_uri(reverse("admin:index")),
}
)
links.append(
{
"title": "Logout",
"type": "post",
"url": request.build_absolute_uri(reverse("logout")),
"fields": {"next": "/"},
}
)
res = {"username": user.username[:15], "links": links}
return JsonResponse(res)
@never_cache
@login_required
@csrf_exempt
def ajax_messages(request: HttpRequest) -> JsonResponse:
"""
Return a JSON object containing the user's queued messages.
Retrieves Django messages for the current request and formats them
as a list of dictionaries, each containing the message text and its
severity level.
Requires the user to be authenticated.
Args:
request (HttpRequest): The request from the authenticated user.
Returns:
response (JsonResponse): A dictionary with a `messages` field containing
a list of message entries.
Response Format - Success:
- `messages` (list[dict]): A list of user-visible messages.
- `level` (str): The severity level of the message
(e.g., "info", "warning", "error").
- `message` (str): The text content of the message.
Example:
```json
{
"messages": [
{"level": "info", "message": "You have been logged out."},
{"level": "warning", "message": "Your session is about to expire."}
]
}
```
"""
return JsonResponse(
{
"messages": [
{"level": MESSAGE_LEVEL_NAMES[i.level], "message": i.message}
for i in get_messages(request)
]
}
)
def get_transcription_superseded(
asset: Asset, supersedes_pk: Union[int, str, None]
) -> Union[Transcription, JsonResponse, None]:
"""
Determine the superseded transcription, if any, for a new transcription.
If a valid `supersedes_pk` is provided, returns the corresponding transcription
unless it has already been superseded. If no `supersedes_pk` is provided,
checks whether the asset already has an open transcription.
This helper may return an error response to be passed directly to the client,
or a transcription object used when saving a new one.
Args:
asset (Asset): The asset the transcription is associated with.
supersedes_pk (int or str or None): The primary key of the transcription
being superseded, or `None` if this is the first transcription.
Returns:
response (Transcription or JsonResponse or None): A valid transcription,
an error response, or `None`.
Return Behavior:
- If a valid transcription is found, a `Transcription` object is returned.
- If the request is invalid or the transcription has already been superseded,
a `JsonResponse` with an error is returned.
- If there is no previous transcription to supersede, `None` is returned.
Response Format - Error:
- `error` (str): Explanation of why the transcription cannot be created.
- "An open transcription already exists"
- "This transcription has been superseded"
- "Invalid supersedes value"
Example:
```json
{
"error": "An open transcription already exists"
}
```
"""
structured_logger.info(
"Checking for superseded transcription.",
event_code="transcription_supersede_check_start",
asset=asset,
supersedes_pk=supersedes_pk,
)
if not supersedes_pk:
if asset.transcription_set.filter(supersedes=None).exists():
structured_logger.warning(
"Open transcription already exists for asset.",
event_code="transcription_supersede_check_failed",
reason="An open transcription already exists",
reason_code="already_exists",
asset=asset,
)
return JsonResponse(
{"error": "An open transcription already exists"}, status=409
)
else:
superseded = None
else:
try:
if asset.transcription_set.filter(supersedes=supersedes_pk).exists():
structured_logger.warning(
"Transcription already superseded.",
event_code="transcription_supersede_check_failed",
reason="This transcription has been superseded",
reason_code="already_superseded",
asset=asset,
supersedes_pk=supersedes_pk,
)
return JsonResponse(
{"error": "This transcription has been superseded"}, status=409
)
try:
superseded = asset.transcription_set.get(pk=supersedes_pk)
except Transcription.DoesNotExist:
structured_logger.warning(
"Supersedes transcription not found.",
event_code="transcription_supersede_check_failed",
reason="Invalid supersedes value",
reason_code="not_found",
asset=asset,
supersedes_pk=supersedes_pk,
)
return JsonResponse({"error": "Invalid supersedes value"}, status=400)
except ValueError:
structured_logger.warning(
"Invalid supersedes value (non-integer).",
event_code="transcription_supersede_check_failed",
reason="Supersedes value must be an integer",
reason_code="invalid_pk_format",
asset=asset,
supersedes_pk=supersedes_pk,
)
return JsonResponse({"error": "Invalid supersedes value"}, status=400)
structured_logger.info(
"Superseded transcription found.",
event_code="transcription_supersede_check_success",
asset=asset,
supersedes_pk=supersedes_pk,
)
return superseded
@require_POST
@login_required
@atomic
@ratelimit(key="header:cf-connecting-ip", rate="1/m", block=settings.RATELIMIT_BLOCK)
def generate_ocr_transcription(
request: HttpRequest, *, asset_pk: Union[int, str]
) -> JsonResponse:
"""
Create and save a new OCR-generated transcription for an asset.
If no prior transcription exists, creates a blank transcription to serve as
the superseded record. Otherwise, the specified previous transcription is
superseded by the new OCR transcription.
Requires the user to be authenticated.
Request Parameters:
- `supersedes` (int or str, optional): The ID of the transcription being
superseded.
- `language` (str, optional): The language code to influence OCR output.
Returns:
response (JsonResponse): A dictionary describing the new transcription
and asset status.
Response Format - Success:
- `id` (int): ID of the new transcription.
- `sent` (float): UNIX timestamp when the transcription was created.
- `submissionUrl` (str): URL to submit the transcription.
- `text` (str): The OCR-generated transcription content.
- `asset` (dict):
- `id` (int): ID of the associated asset.
- `status` (str): Current transcription status.
- `contributors` (int): Number of users who have contributed.
- `undo_available` (bool): Whether the user can roll back this transcription.
- `redo_available` (bool): Whether the user can roll forward to another version.
Example:
```json
{
"id": 123,
"sent": 1716294920.927134,
"submissionUrl": "/transcriptions/123/submit/",
"text": "Detected OCR content...",
"asset": {
"id": 456,
"status": "in_progress",
"contributors": 2
},
"undo_available": true,
"redo_available": false
}
```
"""
asset = get_object_or_404(Asset, pk=asset_pk)
user = request.user
supersedes_pk = request.POST.get("supersedes")
language = request.POST.get("language", None)
structured_logger.info(
"Starting OCR transcription generation.",
event_code="ocr_generation_start",
user=user,
asset=asset,
supersedes_pk=supersedes_pk,
language=language,
)
superseded = get_transcription_superseded(asset, supersedes_pk)
if superseded:
# If superseded is an HttpResponse, that means
# this transcription has already been superseded, so
# we won't run OCR and instead send back an error
# Otherwise, we just have thr transcription the OCR
# is gong to supersede, so we can continue
if isinstance(superseded, HttpResponse):
structured_logger.warning(
"OCR generation aborted: superseded transcription is invalid.",
event_code="ocr_generation_aborted",
reason="Superseded transcription is invalid",
reason_code="superseded_invalid",
user=user,
asset=asset,
)
return superseded
else:
# This means this is the first transcription on this asset.
# To enable undoing of the OCR transcription, we create
# an empty transcription for the OCR transcription to supersede
structured_logger.info(
"No existing transcription; creating empty one for OCR supersession.",
event_code="ocr_blank_supersede",
user=user,
asset=asset,
)
superseded = Transcription(
asset=asset,
user=get_anonymous_user(),
text="",
)
superseded.full_clean()
superseded.save()
structured_logger.info(
"Blank superseded transcription created for OCR.",
event_code="ocr_blank_transcription_created",
user=user,
transcription=superseded,
)
transcription_text = asset.get_ocr_transcript(language)
transcription = Transcription(
asset=asset,
user=user,
supersedes=superseded,
text=transcription_text,
ocr_generated=True,
ocr_originated=True,
)
transcription.full_clean()
transcription.save()
structured_logger.info(
"OCR transcription successfully created.",
event_code="ocr_generation_success",
user=user,
transcription=transcription,
)
return JsonResponse(
{
"id": transcription.pk,
"sent": time(),
"submissionUrl": reverse("submit-transcription", args=(transcription.pk,)),
"text": transcription.text,
"asset": {
"id": transcription.asset.id,
"status": transcription.asset.transcription_status,
"contributors": transcription.asset.get_contributor_count(),
},
"undo_available": asset.can_rollback()[0],
"redo_available": asset.can_rollforward()[0],
},
status=201,
)
@require_POST
@validate_anonymous_user
@atomic
@ratelimit(key="header:cf-connecting-ip", rate="1/m", block=settings.RATELIMIT_BLOCK)
def rollback_transcription(
request: HttpRequest, *, asset_pk: Union[int, str]
) -> JsonResponse:
"""
Perform a rollback on the latest transcription for the given asset.
Restores the asset's transcription to the previous version in its history.
If rollback is not possible (e.g., no prior version exists), returns an error.
Anonymous users are supported and handled via `get_anonymous_user()`. The caller
must be validated via `validate_anonymous_user`.
Args:
request (HttpRequest): The POST request to initiate rollback.
asset_pk (int or str): The primary key of the asset being rolled back.
Returns:
response (JsonResponse): A dictionary containing the restored transcription
and asset status, or an error response if rollback fails.
Response Format - Success:
- `id` (int): ID of the restored transcription.
- `sent` (float): UNIX timestamp of the response.
- `submissionUrl` (str): URL to submit the transcription.
- `text` (str): The restored transcription text.
- `asset` (dict):
- `id` (int): ID of the asset.
- `status` (str): Current transcription status.
- `contributors` (int): Number of users who contributed.
- `message` (str): Confirmation message.
- `undo_available` (bool): Whether rollback is possible again.
- `redo_available` (bool): Whether rollforward is now available.
Response Format - Error:
- `error` (str): Explanation of the failure.
- "No previous transcription available"
Example:
```json
{
"id": 123,
"sent": 1716295121.113204,
"submissionUrl": "/transcriptions/123/submit/",
"text": "Previous transcription text",
"asset": {
"id": 456,
"status": "in_progress",
"contributors": 1
},
"message": "Successfully rolled back transcription to previous version",
"undo_available": false,
"redo_available": true
}
```
"""
asset = get_object_or_404(Asset, pk=asset_pk)
if request.user.is_anonymous:
user = get_anonymous_user()
else:
user = request.user
try:
transcription = asset.rollback_transcription(user)
except ValueError as e:
logger.exception("No previous transcription available for rollback", exc_info=e)
structured_logger.warning(
"Rollback failed: no previous transcription to revert to.",
event_code="rollback_failed",
reason_code="no_valid_target",
reason=str(e),
asset=asset,
user=user,
)
return JsonResponse(
{"error": "No previous transcription available"}, status=400
)
structured_logger.info(
"Rollback successfully performed.",
event_code="rollback_success",
user=user,
transcription=transcription,
)
return JsonResponse(
{
"id": transcription.pk,
"sent": time(),
"submissionUrl": reverse("submit-transcription", args=(transcription.pk,)),
"text": transcription.text,
"asset": {
"id": transcription.asset.id,
"status": transcription.asset.transcription_status,
"contributors": transcription.asset.get_contributor_count(),
},
"message": "Successfully rolled back transcription to previous version",
"undo_available": transcription.asset.can_rollback()[0],
"redo_available": transcription.asset.can_rollforward()[0],
},
status=201,
)
@require_POST
@validate_anonymous_user
@atomic
@ratelimit(key="header:cf-connecting-ip", rate="1/m", block=settings.RATELIMIT_BLOCK)
def rollforward_transcription(
request: HttpRequest, *, asset_pk: Union[int, str]
) -> JsonResponse:
"""
Perform a rollforward to the transcription previously replaced by a rollback.
Restores the asset's transcription to the next version in its history,
if a valid rollforward target exists. If not, returns an error response.
Anonymous users are supported and handled via `get_anonymous_user()`. The caller
must be validated via `validate_anonymous_user`.
Args:
request (HttpRequest): The POST request to initiate rollforward.
asset_pk (int or str): The primary key of the asset being rolled forward.
Returns:
response (JsonResponse): A dictionary containing the restored transcription
and asset status, or an error response if rollforward fails.
Response Format - Success:
- `id` (int): ID of the restored transcription.
- `sent` (float): UNIX timestamp of the response.
- `submissionUrl` (str): URL to submit the transcription.
- `text` (str): The restored transcription text.
- `asset` (dict):
- `id` (int): ID of the asset.
- `status` (str): Current transcription status.
- `contributors` (int): Number of users who contributed.
- `message` (str): Confirmation message.
- `undo_available` (bool): Whether rollback is now possible.
- `redo_available` (bool): Whether another rollforward is possible.
Response Format - Error:
- `error` (str): Explanation of the failure.
- "No transcription to restore"
Example:
```json
{
"id": 124,
"sent": 1716295243.029184,
"submissionUrl": "/transcriptions/124/submit/",
"text": "Next transcription text",
"asset": {
"id": 456,
"status": "in_progress",
"contributors": 1
},
"message": "Successfully restored transcription to next version",
"undo_available": true,
"redo_available": false
}
```
"""
asset = get_object_or_404(Asset, pk=asset_pk)
if request.user.is_anonymous:
user = get_anonymous_user()
else:
user = request.user
try:
transcription = asset.rollforward_transcription(user)
except ValueError as e:
logger.exception("No transcription available for rollforward", exc_info=e)
structured_logger.warning(
"Rollforward failed: no transcription available to restore.",
event_code="rollforward_failed",
reason_code="no_valid_target",
reason=str(e),
asset=asset,
user=user,
)
return JsonResponse({"error": "No transcription to restore"}, status=400)
structured_logger.info(
"Rollforward successfully performed.",
event_code="rollforward_success",
user=user,
transcription=transcription,
)
return JsonResponse(
{
"id": transcription.pk,
"sent": time(),
"submissionUrl": reverse("submit-transcription", args=(transcription.pk,)),
"text": transcription.text,
"asset": {
"id": transcription.asset.id,
"status": transcription.asset.transcription_status,
"contributors": transcription.asset.get_contributor_count(),
},
"message": "Successfully restored transcription to next version",
"undo_available": transcription.asset.can_rollback()[0],
"redo_available": transcription.asset.can_rollforward()[0],
},
status=201,
)
@require_POST
@validate_anonymous_user
@atomic
def save_transcription(
request: HttpRequest, *, asset_pk: Union[int, str]
) -> JsonResponse:
"""
Save a transcription draft for a given asset.
Validates the transcription text for disallowed content (e.g., URLs).
Non-printable characters are automatically removed before saving,
using the shared exporter sanitization utilities. The view also checks
for supersession rules. If valid, it creates and saves a new
transcription associated with the current or anonymous user.
Request Parameters:
- `text` (str): The transcription text.
- `supersedes` (int or str, optional): The ID of the transcription
being superseded. Example: `"123"`
Returns:
response (JsonResponse): A dictionary describing the saved transcription
and asset status, or an error response if validation fails.
Response Format - Success:
- `id` (int): ID of the saved transcription.
- `sent` (float): UNIX timestamp of the response.
- `submissionUrl` (str): URL to submit the transcription.
- `asset` (dict):
- `id` (int): ID of the associated asset.
- `status` (str): Current transcription status.
- `contributors` (int): Number of users who contributed.
- `undo_available` (bool): Whether rollback is currently possible.
- `redo_available` (bool): Whether rollforward is currently possible.
Response Format - Error:
- `error` (str): Explanation of the validation failure.
- "It looks like your text contains URLs."
- "An open transcription already exists"
- "This transcription has been superseded"
- "Invalid supersedes value"
Example:
```json
{
"id": 125,
"sent": 1716295310.743182,
"submissionUrl": "/transcriptions/125/submit/",
"text" : "Transcription text\r\nSecond line",
"asset": {
"id": 456,
"status": "in_progress",
"contributors": 1
},
"undo_available": true,
"redo_available": false
}
```
"""
asset = get_object_or_404(Asset, pk=asset_pk)
logger.info("Saving transcription for %s (%s)", asset, asset.id)
if request.user.is_anonymous:
user = get_anonymous_user()
else:
user = request.user
structured_logger.info(
"Starting transcription save.",
event_code="transcription_save_start",
user=user,
asset=asset,
)
transcription_text = request.POST["text"]
# Check whether this transcription text contains any URLs.
# If so, ask the user to correct the transcription by removing the URLs.
url_match = re.search(URL_REGEX, transcription_text)
if url_match:
structured_logger.warning(
"Transcription save rejected due to URL in text.",
event_code="transcription_save_rejected",
reason="Transcription text contains URLs",
reason_code="url_detected",
user=user,
asset=asset,
)
return JsonResponse(
{
"error": "It looks like your text contains URLs. "
"Please remove the URLs and try again.",
"error-code": "url_detected",
},
status=400,
)
# Sanitize the text by removing any unacceptable (non-printable) characters.
# This leverages the shared exporter whitelist and logic so behavior remains
# consistent across validation and export paths.
transcription_text = remove_unacceptable_characters(transcription_text)
supersedes_pk = request.POST.get("supersedes")
superseded = get_transcription_superseded(asset, supersedes_pk)
if superseded and isinstance(superseded, HttpResponse):
logger.info("Transcription superseded")
structured_logger.warning(
"Superseded transcription is invalid; aborting save.",
event_code="transcription_save_aborted",
reason="Superseded transcription is invalid",
reason_code="superseded_invalid",
user=user,
asset=asset,
)
return superseded
if superseded and (superseded.ocr_generated or superseded.ocr_originated):
ocr_originated = True
else:
ocr_originated = False
transcription = Transcription(
asset=asset,
user=user,
supersedes=superseded,
text=transcription_text,
ocr_originated=ocr_originated,
)
transcription.full_clean()
transcription.save()
logger.info("Transction %s saved", transcription.id)
structured_logger.info(
"Transcription saved successfully.",
event_code="transcription_save_success",
user=user,
transcription=transcription,
)
return JsonResponse(
{
"id": transcription.pk,
"sent": time(),
"submissionUrl": reverse("submit-transcription", args=(transcription.pk,)),
"text": transcription.text,
"asset": {
"id": transcription.asset.id,
"status": transcription.asset.transcription_status,
"contributors": transcription.asset.get_contributor_count(),
},
"undo_available": transcription.asset.can_rollback()[0],
"redo_available": transcription.asset.can_rollforward()[0],
},
status=201,
)
@require_POST
@validate_anonymous_user
def submit_transcription(request: HttpRequest, *, pk: Union[int, str]) -> JsonResponse:
"""
Submit a transcription for review.
Marks the transcription as submitted and clears any rejection state.
Prevents submission if the transcription has already been accepted or
superseded.
Anonymous users are supported and handled via `get_anonymous_user()`. The caller
must be validated via `validate_anonymous_user`.
Args:
request (HttpRequest): The POST request to submit the transcription.
pk (int or str): The primary key of the transcription to submit.
Returns:
response (JsonResponse): A dictionary with the asset status and submission
metadata, or an error response if submission is not allowed.
Response Format - Success:
- `id` (int): ID of the submitted transcription.
- `sent` (float): UNIX timestamp of the response.
- `asset` (dict):
- `id` (int): ID of the associated asset.
- `status` (str): Current transcription status.
- `contributors` (int): Number of users who contributed.
- `undo_available` (bool): Always `false` after submission.
- `redo_available` (bool): Always `false` after submission.
Response Format - Error:
- `error` (str): Explanation of the submission failure.
- "This transcription has already been updated."
Example:
```json
{
"id": 126,
"sent": 1716295421.019122,
"asset": {
"id": 456,
"status": "submitted",
"contributors": 1
},
"undo_available": false,
"redo_available": false
}
```
"""
transcription = get_object_or_404(Transcription, pk=pk)
asset = transcription.asset
logger.info(
"Transcription %s submitted for %s (%s)", transcription.id, asset, asset.id
)
is_superseded = transcription.asset.transcription_set.filter(supersedes=pk).exists()
is_already_submitted = transcription.submitted and not transcription.rejected
if is_already_submitted or is_superseded:
logger.warning(
(
"Submit for review was attempted for invalid transcription "
"record: submitted: %s pk: %d"
),
str(transcription.submitted),
pk,
)
structured_logger.warning(
"Submission rejected: transcription already submitted or superseded.",
event_code="transcription_submit_rejected",
reason="Transcription already submitted or superseded",
reason_code="already_updated",
user=request.user,
transcription=transcription,
)
return JsonResponse(
{
"error": "This transcription has already been updated."
" Reload the current status before continuing."
},
status=400,
)
transcription.submitted = now()
transcription.rejected = None
transcription.full_clean()
transcription.save()
logger.info("Transcription %s successfully submitted", transcription.id)
structured_logger.info(
"Transcription submitted successfully.",
event_code="transcription_submit_success",
user=request.user,
transcription=transcription,
)
return JsonResponse(
{
"id": transcription.pk,
"sent": time(),
"asset": {
"id": transcription.asset.id,
"status": transcription.asset.transcription_status,
"contributors": transcription.asset.get_contributor_count(),
},
"undo_available": False,
"redo_available": False,
},
status=200,
)
@require_POST
@login_required
@never_cache
def review_transcription(request: HttpRequest, *, pk: Union[int, str]) -> JsonResponse:
"""
Review and accept or reject a submitted transcription.
Only non-authors may accept a transcription. Users are limited by a
rate limit when accepting transcriptions. Review actions are rejected
if the transcription has already been reviewed or is invalid.
Args:
request (HttpRequest): The POST request containing the review action.
pk (int or str): The primary key of the transcription to review.
Returns:
response (JsonResponse): A dictionary with updated asset status and
metadata, or an error response if the review fails.
Response Format - Success:
- `id` (int): ID of the reviewed transcription.
- `sent` (float): UNIX timestamp of the response.
- `asset` (dict):
- `id` (int): ID of the associated asset.
- `status` (str): Updated transcription status.
- `contributors` (int): Number of users who contributed.
Response Format - Error:
- `error` (str): Explanation of the review failure.
- "Invalid action"
- "This transcription has already been reviewed"
- "You cannot accept your own transcription"
- Configuration-based rate limit messages
Example:
```json
{
"id": 127,
"sent": 1716295502.642184,
"asset": {
"id": 456,
"status": "completed",
"contributors": 2
}
}
```
"""
action = request.POST.get("action")
structured_logger.info(
"Starting transcription review.",
event_code="transcription_review_start",
user=request.user,
transcription_id=pk,
action=action,
)
if action not in ("accept", "reject"):
structured_logger.warning(
"Transcription review failed: invalid action.",
event_code="transcription_review_rejected",
reason="Invalid review action",
reason_code="invalid_action",
user=request.user,
transcription_id=pk,
action=action,
)
return JsonResponse({"error": "Invalid action"}, status=400)
transcription = get_object_or_404(Transcription, pk=pk)
asset = transcription.asset
logger.info(
"Transcription %s reviewed (%s) for %s (%s)",
transcription.id,
action,
asset,
asset.id,
)
if transcription.accepted or transcription.rejected:
structured_logger.warning(
"Review rejected: transcription already reviewed.",
event_code="transcription_review_rejected",
reason="Transcription has already been reviewed",
reason_code="already_reviewed",
user=request.user,
transcription=transcription,
)
return JsonResponse(
{"error": "This transcription has already been reviewed"}, status=400
)
if transcription.user.pk == request.user.pk and action == "accept":
logger.warning("Attempted self-acceptance for transcription %s", transcription)
structured_logger.warning(
"Review rejected: user attempted to accept their own transcription.",
event_code="transcription_review_rejected",
reason="User attempted to accept their own transcription",
reason_code="self_accept",
user=request.user,
transcription=transcription,
)
return JsonResponse(
{"error": "You cannot accept your own transcription"}, status=400
)
transcription.reviewed_by = request.user
if action == "accept":
concordia_user = ConcordiaUser.objects.get(id=request.user.id)
try:
concordia_user.check_and_track_accept_limit(transcription)
except RateLimitExceededError:
structured_logger.warning(
"Review rejected: user exceeded review rate limit.",
event_code="transcription_review_rejected",
reason="User exceeded review rate limit",
reason_code="rate_limit_exceeded",
user=request.user,
transcription=transcription,
)
return JsonResponse(
{
"error": configuration_value("review_rate_limit_banner_message"),
"popupTitle": configuration_value("review_rate_limit_popup_title"),
"popupError": configuration_value(
"review_rate_limit_popup_message"
),
},
status=429,
)
transcription.accepted = now()
else:
transcription.rejected = now()
transcription.full_clean()
transcription.save()
logger.info("Transcription %s successfully reviewed (%s)", transcription.id, action)
structured_logger.info(
"Transcription review successful.",
event_code="transcription_review_success",
user=request.user,
transcription=transcription,
action=action,
)
return JsonResponse(
{
"id": transcription.pk,
"sent": time(),
"asset": {
"id": transcription.asset.id,
"status": transcription.asset.transcription_status,
"contributors": transcription.asset.get_contributor_count(),
},
},
status=200,
)
@require_POST
@login_required
@atomic
def submit_tags(request: HttpRequest, *, asset_pk: Union[int, str]) -> JsonResponse:
"""
Submit a new set of tags for an asset from the current user.
Creates any new tags as needed and updates the user's tag collection
for the asset. Removes tags that are no longer present in the submission.
Args:
request (HttpRequest): The POST request containing tag values.
asset_pk (int or str): The primary key of the asset to tag.
Returns:
response (JsonResponse): A dictionary containing the updated user-specific
and global tag lists for the asset.
Response Format - Success:
- `user_tags` (list[str]): Tags currently assigned to the asset by this user.
- `all_tags` (list[str]): All tags currently applied to the asset by any user.
Response Format - Error:
- `error` (list[str]): Validation error messages for malformed/duplicate tags.
Example:
```json
{
"user_tags": ["map", "handwritten"],
"all_tags": ["handwritten", "map", "note"]
}
```
"""
asset = get_object_or_404(Asset, pk=asset_pk)
structured_logger.info(
"Starting tag submission.",
event_code="tag_submit_start",
user=request.user,
asset=asset,
)
user_tags, created = UserAssetTagCollection.objects.get_or_create(
asset=asset, user=request.user
)
tags = set(request.POST.getlist("tags"))
existing_tags = Tag.objects.filter(value__in=tags)
new_tag_values = tags.difference(i.value for i in existing_tags)
new_tags = [Tag(value=i) for i in new_tag_values]
try:
for i in new_tags:
i.full_clean()
except ValidationError as exc:
structured_logger.warning(
"Tag submission rejected: validation error on new tags.",
event_code="tag_submit_rejected",
reason="Tag failed validation",
reason_code="validation_error",
user=request.user,
asset=asset,
errors=str(exc.messages),
)
return JsonResponse({"error": exc.messages}, status=400)
Tag.objects.bulk_create(new_tags)
# At this point we now have Tag objects for everything in the POSTed
# request. We'll add anything which wasn't previously in this user's tag
# collection and remove anything which is no longer present.
all_submitted_tags = list(existing_tags) + new_tags
existing_user_tags = user_tags.tags.all()
for tag in all_submitted_tags:
if tag not in existing_user_tags:
user_tags.tags.add(tag)
all_tags_qs = Tag.objects.filter(userassettagcollection__asset__pk=asset_pk)
for tag in all_tags_qs:
if tag not in all_submitted_tags:
for collection in asset.userassettagcollection_set.all():
collection.tags.remove(tag)
all_tags = all_tags_qs.order_by("value")
final_user_tags = user_tags.tags.order_by("value").values_list("value", flat=True)
all_tags = all_tags.values_list("value", flat=True).distinct()
structured_logger.info(
"Tags submitted successfully.",
event_code="tag_submit_success",
user=request.user,
asset=asset,
user_tags=[tag.value for tag in user_tags.tags.all()],
)
return JsonResponse(
{"user_tags": list(final_user_tags), "all_tags": list(all_tags)}
)
@ratelimit(
key="header:cf-connecting-ip", rate=reserve_rate, block=settings.RATELIMIT_BLOCK
)
@require_POST
@never_cache
def reserve_asset(request: HttpRequest, *, asset_pk: Union[int, str]) -> JsonResponse:
"""
Attempt to reserve an asset for transcription by the current session.
If no active reservation exists, creates a new one using the session's
reservation token. If a reservation exists for this session, updates it.
If the asset is reserved by another session, returns a conflict response.
Handles reservation release if `release` is set in the request body.
Request Parameters:
- `release` (bool, optional): If present and true, releases the current
reservation instead of acquiring or updating it. Example: `"true"`
Returns:
response (JsonResponse or HttpResponse): A dictionary indicating the
reservation status and token, or an HTTP 408/409 response for timeout
or conflict.
Response Format - Success:
- `asset_pk` (int): The ID of the reserved asset.
- `reservation_token` (str): A unique identifier for the reservation session.
Response Format - Error:
- `408 Request Timeout`: The current session's reservation is tombstoned.
- `409 Conflict`: The asset is actively reserved by another session.
Example:
```json
{
"asset_pk": 789,
"reservation_token": "abc123xyz"
}
```
"""
reservation_token = get_or_create_reservation_token(request)
structured_logger.info(
"Handling reservation request.",
event_code="asset_reserve_start",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
# If the browser is letting us know of a specific reservation release,
# let it go even if it's within the grace period.
if request.POST.get("release"):
with connection.cursor() as cursor:
cursor.execute(
"""
DELETE FROM concordia_assettranscriptionreservation
WHERE asset_id = %s and reservation_token = %s
""",
[asset_pk, reservation_token],
)
# We'll pass the message to the WebSocket listeners before returning it:
msg = {"asset_pk": asset_pk, "reservation_token": reservation_token}
logger.info("Releasing reservation with token %s", reservation_token)
structured_logger.info(
"Releasing asset reservation via client request.",
event_code="asset_reserve_release",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
reservation_released.send(sender="reserve_asset", **msg)
return JsonResponse(msg)
# We're relying on the database to meet our integrity requirements and since
# this is called periodically we want to be fairly fast until we switch to
# something like Redis.
reservations = AssetTranscriptionReservation.objects.filter(
asset_id__exact=asset_pk
)
# Default: pretend there is no activity on the asset
is_it_already_mine = False
am_i_tombstoned = False
is_someone_else_tombstoned = False
is_someone_else_active = False
if reservations:
for reservation in reservations:
if reservation.tombstoned:
if reservation.reservation_token == reservation_token:
am_i_tombstoned = True
logger.debug("I'm tombstoned %s", reservation_token)
else:
is_someone_else_tombstoned = True
logger.debug(
"Someone else is tombstoned %s", reservation.reservation_token
)
else:
if reservation.reservation_token == reservation_token:
is_it_already_mine = True
logger.debug(
"I already have this active reservation %s", reservation_token
)
if not is_it_already_mine:
is_someone_else_active = True
logger.info(
"Someone else has this active reservation %s",
reservation.reservation_token,
)
if am_i_tombstoned:
structured_logger.warning(
"Reservation rejected: client is tombstoned.",
event_code="asset_reserve_rejected",
reason="Client reservation token is tombstoned",
reason_code="tombstoned_self",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
return HttpResponse(status=408) # Request Timed Out
if is_someone_else_active:
structured_logger.warning(
"Reservation rejected: asset is reserved by another client.",
event_code="asset_reserve_rejected",
reason="Asset is actively reserved by another session",
reason_code="conflict_active_other",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
return HttpResponse(status=409) # Conflict
if is_it_already_mine:
# This user already has the reservation and it's not tombstoned
structured_logger.info(
"Reservation updated for client.",
event_code="asset_reserve_updated",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
msg = update_reservation(asset_pk, reservation_token)
logger.debug("Updating reservation %s", reservation_token)
if is_someone_else_tombstoned:
# No reservations = no activity = go ahead and do an insert
structured_logger.info(
"Reservation acquired from tombstoned client.",
event_code="asset_reserve_from_tombstone",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
msg = obtain_reservation(asset_pk, reservation_token)
logger.debug(
"Obtaining reservation for %s from tombstoned user", reservation_token
)
else:
# No reservations = no activity = go ahead and do an insert
structured_logger.info(
"Initial reservation acquired (no existing reservations).",
event_code="asset_reserve_fresh",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
msg = obtain_reservation(asset_pk, reservation_token)
logger.debug("No activity, just get the reservation %s", reservation_token)
return JsonResponse(msg)
def update_reservation(
asset_pk: Union[int, str], reservation_token: str
) -> dict[str, Union[int, str]]:
"""
Update the timestamp on an existing active reservation for an asset.
Refreshes the reservation's `updated_on` field to extend its validity
and emits the `reservation_obtained` signal.
Args:
asset_pk (int or str): The primary key of the reserved asset.
reservation_token (str): The session's reservation token.
Returns:
response (dict): A dictionary confirming the updated reservation state.
Response Format - Success:
- `asset_pk` (int): The ID of the reserved asset.
- `reservation_token` (str): The reservation token used by the session.
Example:
```json
{
"asset_pk": 789,
"reservation_token": "abc123xyz"
}
```
"""
structured_logger.info(
"Attempting to update reservation timestamp.",
event_code="reservation_update_start",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
with connection.cursor() as cursor:
cursor.execute(
"""
UPDATE concordia_assettranscriptionreservation AS atr
SET updated_on = current_timestamp
WHERE (
atr.asset_id = %s
AND atr.reservation_token = %s
AND atr.tombstoned != TRUE
)
""".strip(),
[asset_pk, reservation_token],
)
structured_logger.info(
"Reservation update SQL executed.",
event_code="reservation_update_sql_executed",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
# We'll pass the message to the WebSocket listeners before returning it:
msg = {"asset_pk": asset_pk, "reservation_token": reservation_token}
reservation_obtained.send(sender="reserve_asset", **msg)
structured_logger.info(
"Reservation update completed; signal dispatched.",
event_code="reservation_update_success",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
return msg
def obtain_reservation(
asset_pk: Union[int, str], reservation_token: str
) -> dict[str, Union[int, str]]:
"""
Create a new reservation entry for an asset.
Inserts a new reservation row in the database for the given asset and
session token. Emits the `reservation_obtained` signal to notify listeners.
Args:
asset_pk (int or str): The primary key of the asset to reserve.
reservation_token (str): The session's reservation token.
Returns:
response (dict): A dictionary confirming the newly obtained reservation.
Response Format - Success:
- `asset_pk` (int): The ID of the reserved asset.
- `reservation_token` (str): The reservation token used by the session.
Example:
```json
{
"asset_pk": 789,
"reservation_token": "abc123xyz"
}
```
"""
structured_logger.info(
"Attempting to create new reservation.",
event_code="reservation_obtain_start",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
with connection.cursor() as cursor:
cursor.execute(
"""
INSERT INTO concordia_assettranscriptionreservation AS atr
(asset_id, reservation_token, tombstoned, created_on,
updated_on)
VALUES (%s, %s, FALSE, current_timestamp,
current_timestamp)
""".strip(),
[asset_pk, reservation_token],
)
structured_logger.info(
"Reservation INSERT executed successfully.",
event_code="reservation_insert_success",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
# We'll pass the message to the WebSocket listeners before returning it:
msg = {"asset_pk": asset_pk, "reservation_token": reservation_token}
reservation_obtained.send(sender="reserve_asset", **msg)
structured_logger.info(
"Reservation successfully obtained; signal dispatched.",
event_code="reservation_obtain_success",
asset_pk=asset_pk,
reservation_token=reservation_token,
)
return msg
================================================
FILE: concordia/views/assets.py
================================================
import logging
import random
from typing import Any
from urllib.parse import urlencode
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.models import User
from django.db.models import QuerySet
from django.db.transaction import atomic
from django.http import Http404, HttpRequest, HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, redirect
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.views.decorators.cache import never_cache
from django_ratelimit.decorators import ratelimit
from concordia.api_views import APIDetailView
from concordia.forms import TurnstileForm
from concordia.logging import ConcordiaLogger
from concordia.models import (
Asset,
AssetTranscriptionReservation,
Campaign,
CardFamily,
Guide,
Topic,
TranscriptionStatus,
TutorialCard,
UserAssetTagCollection,
)
from concordia.templatetags.concordia_media_tags import asset_media_url
from concordia.utils import (
get_anonymous_user,
get_or_create_reservation_token,
)
from concordia.utils.next_asset import (
find_next_reviewable_campaign_asset,
find_next_reviewable_topic_asset,
find_next_transcribable_campaign_asset,
find_next_transcribable_topic_asset,
find_reviewable_campaign_asset,
find_transcribable_campaign_asset,
remove_next_asset_objects,
)
from .decorators import next_asset_rate
from .utils import AnonymousUserValidationCheckMixin
logger = logging.getLogger(__name__)
structured_logger = ConcordiaLogger.get_logger(__name__)
@method_decorator(never_cache, name="dispatch")
class AssetDetailView(AnonymousUserValidationCheckMixin, APIDetailView):
"""
Display details for a single asset and handle missing assets.
This view handles `GET` and `POST` requests by retrieving the published
`Asset` that matches the campaign, project and item.
It uses `AnonymousUserValidationCheckMixin` for anonymous-user validation
and `APIDetailView` for API-driven detail behavior. It overrides
`dispatch` to log and redirect to the parent campaign page if the asset
is not found.
Attributes:
template_name (str): Template used to render the asset detail page.
"""
template_name = "transcriptions/asset_detail.html"
def dispatch(
self,
request: HttpRequest,
*args: Any,
**kwargs: Any,
) -> HttpResponse:
try:
return super().dispatch(request, *args, **kwargs)
except Http404:
structured_logger.info(
"AssetDetailView: asset not found, redirecting to campaign " "page.",
event_code="asset_detail_not_found_redirect",
user=request.user,
campaign_slug=self.kwargs.get("campaign_slug"),
project_slug=self.kwargs.get("project_slug"),
item_id=self.kwargs.get("item_id"),
asset_slug=self.kwargs.get("slug"),
)
campaign = get_object_or_404(
Campaign.objects.published(), slug=self.kwargs["campaign_slug"]
)
return redirect(campaign)
def get_queryset(self) -> QuerySet[Asset]:
asset_qs = Asset.objects.published().filter(
item__project__campaign__slug=self.kwargs["campaign_slug"],
item__project__slug=self.kwargs["project_slug"],
item__item_id=self.kwargs["item_id"],
slug=self.kwargs["slug"],
)
asset_qs = asset_qs.select_related("item__project__campaign")
return asset_qs
def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
"""
Build the context for the asset detail template.
Constructs a dictionary with the entries described in the context
format.
Context Format:
- `asset` (Asset): Asset instance being viewed.
- `item` (Item): Parent item of the asset.
- `project` (Project): Parent project of the item.
- `campaign` (Campaign): Campaign that contains the project.
- `transcription` (Transcription | None): Latest transcription or
`None`.
- `next_open_asset_url` (str): URL to the next transcribable
asset.
- `next_review_asset_url` (str): URL to the next reviewable
asset.
- `transcription_status` (str): One of the keys from
`TranscriptionStatus`.
- `activity_mode` (str): `"transcribe"` or `"review"`, based on
the transcription status.
- `disable_ocr` (bool): Whether OCR should be disabled for this
asset.
- `previous_asset_url` (str | None): URL to the previous asset, if
any.
- `next_asset_url` (str | None): URL to the next asset, if any.
- `asset_navigation` (list[tuple[int, str]]): Sequence and slug
pairs for navigation.
- `thumbnail_url` (str): URL of the asset thumbnail image.
- `current_asset_url` (str): Absolute URL of this asset detail
view.
- `tags` (list[str]): Sorted tag values applied to the asset.
- `registered_contributors` (int): Number of users who have
contributed to the asset.
- `cards` (list[Card]): Tutorial cards for the campaign or the
default card set.
- `guides` (QuerySet[dict[str, Any]] | None): Tutorial guide
entries.
- `languages` (list[tuple[str, str]]): Supported language
code and name pairs.
- `undo_available` (bool): Whether a rollback is possible.
- `redo_available` (bool): Whether a rollforward is possible.
- `turnstile_form` (TurnstileForm): Form for the Turnstile
widget.
Args:
**kwargs (Any): Additional keyword arguments passed to the
superclass implementation.
Returns:
dict[str, Any]: Context data for rendering the asset detail page.
"""
ctx = super().get_context_data(**kwargs)
asset = ctx["asset"]
# Bind a new logger so asset and user are always included
context_logger = structured_logger.bind(user=self.request.user, asset=asset)
context_logger.info(
"AssetDetailView: building context.",
event_code="asset_detail_context_start",
)
ctx["item"] = item = asset.item
ctx["project"] = project = item.project
ctx["campaign"] = project.campaign
transcription = asset.transcription_set.order_by("-pk").first()
context_logger.debug(
"AssetDetailView: latest transcription selected.",
event_code="asset_detail_latest_transcription",
transcription=transcription,
)
ctx["transcription"] = transcription
ctx["next_open_asset_url"] = "%s?%s" % (
reverse(
"transcriptions:redirect-to-next-transcribable-campaign-asset",
kwargs={"campaign_slug": project.campaign.slug},
),
urlencode(
{"project": project.slug, "item": item.item_id, "asset": asset.id}
),
)
ctx["next_review_asset_url"] = "%s?%s" % (
reverse(
"transcriptions:redirect-to-next-reviewable-campaign-asset",
kwargs={"campaign_slug": project.campaign.slug},
),
urlencode(
{"project": project.slug, "item": item.item_id, "asset": asset.id}
),
)
# We handle the case where an item with no transcriptions should be
# shown as status=not_started here so the logic does not need to be
# repeated in templates.
if transcription:
for choice_key, choice_value in TranscriptionStatus.CHOICE_MAP.items():
if choice_value == transcription.status:
transcription_status = choice_key
else:
transcription_status = TranscriptionStatus.NOT_STARTED
ctx["transcription_status"] = transcription_status
context_logger.debug(
"AssetDetailView: computed transcription status.",
event_code="asset_detail_transcription_status",
computed_status=transcription_status,
asset_status=asset.transcription_status,
)
if (
transcription_status == TranscriptionStatus.NOT_STARTED
or transcription_status == TranscriptionStatus.IN_PROGRESS
):
ctx["activity_mode"] = "transcribe"
ctx["disable_ocr"] = asset.turn_off_ocr()
else:
ctx["disable_ocr"] = True
if transcription_status == TranscriptionStatus.SUBMITTED:
ctx["activity_mode"] = "review"
previous_asset = (
item.asset_set.published()
.filter(sequence__lt=asset.sequence)
.order_by("sequence")
.last()
)
next_asset = (
item.asset_set.published()
.filter(sequence__gt=asset.sequence)
.order_by("sequence")
.first()
)
context_logger.debug(
"AssetDetailView: asset navigation resolved.",
event_code="asset_detail_navigation",
previous_asset_id=getattr(previous_asset, "pk", None),
next_asset_id=getattr(next_asset, "pk", None),
)
if previous_asset:
ctx["previous_asset_url"] = previous_asset.get_absolute_url()
if next_asset:
ctx["next_asset_url"] = next_asset.get_absolute_url()
ctx["asset_navigation"] = (
item.asset_set.published()
.order_by("sequence")
.values_list("sequence", "slug")
)
image_url = asset_media_url(asset)
if asset.download_url and "iiif" in asset.download_url:
thumbnail_url = asset.download_url.replace(
"http://tile.loc.gov", "https://tile.loc.gov"
)
thumbnail_url = thumbnail_url.replace("/pct:100/", "/!512,512/")
else:
thumbnail_url = image_url
context_logger.debug(
"AssetDetailView: thumbnail URL determined.",
event_code="asset_detail_thumbnail",
thumbnail_url=thumbnail_url,
)
ctx["thumbnail_url"] = thumbnail_url
ctx["current_asset_url"] = self.request.build_absolute_uri()
tag_groups = UserAssetTagCollection.objects.filter(asset__slug=asset.slug)
tags = set()
for tag_group in tag_groups:
for tag in tag_group.tags.all():
tags.add(tag.value)
ctx["tags"] = sorted(tags)
ctx["registered_contributors"] = asset.get_contributor_count()
if project.campaign.card_family:
card_family = project.campaign.card_family
else:
card_family = CardFamily.objects.filter(default=True).first()
if card_family is not None:
unordered_cards = TutorialCard.objects.filter(tutorial=card_family)
ordered_cards = unordered_cards.order_by("order")
ctx["cards"] = [tutorial_card.card for tutorial_card in ordered_cards]
guides = Guide.objects.order_by("order").values("title", "body")
if guides.count() > 0:
ctx["guides"] = guides
ctx["languages"] = list(settings.LANGUAGE_CODES.items())
ctx["undo_available"] = asset.can_rollback()[0] if transcription else False
ctx["redo_available"] = asset.can_rollforward()[0] if transcription else False
ctx["turnstile_form"] = TurnstileForm(auto_id=False)
context_logger.info(
"AssetDetailView: context ready.",
event_code="asset_detail_context_ready",
transcription=transcription,
transcription_status=transcription_status,
)
return ctx
def redirect_to_next_asset(
asset: Asset | None,
mode: str,
request: HttpRequest,
user: User,
) -> HttpResponseRedirect:
"""
Redirect the user to the appropriate asset view or the homepage.
If an asset is found, this helper creates a reservation for it and
removes the asset from the relevant caching tables. The user is then
redirected to the transcription page for that asset.
If no asset is provided, it redirects to the homepage and adds an
informational message.
Args:
asset (Asset | None): Asset to redirect to, or `None` if no asset is
available.
mode (str): Either `"transcribe"` or `"review"`, used for messaging.
request (HttpRequest): Request that initiated the redirect.
user (User): User being redirected.
Returns:
HttpResponseRedirect: Redirect to the asset detail page or the
homepage.
"""
structured_logger.info(
"Starting redirect to next asset.",
event_code="redirect_next_asset_start",
user=user,
mode=mode,
asset=asset,
)
reservation_token = get_or_create_reservation_token(request)
if asset:
# We previously created reservations for transcriptions but not
# reviews. This created a race condition with the next asset caching
# system because the non-reserved asset could be added into the cache
# table between when the user was redirected and when they made their
# own reservation. That could result in the asset being added to the
# caching system and sent to another user.
res = AssetTranscriptionReservation(
asset=asset, reservation_token=reservation_token
)
res.full_clean()
res.save()
structured_logger.info(
"Asset reserved and redirecting to asset detail view.",
event_code="redirect_next_asset_success",
asset=asset,
user=user,
)
remove_next_asset_objects(asset.id)
return redirect(
"transcriptions:asset-detail",
asset.item.project.campaign.slug,
asset.item.project.slug,
asset.item.item_id,
asset.slug,
)
else:
no_pages_message = f"There are no remaining pages to {mode}."
structured_logger.warning(
"No available asset to redirect to.",
event_code="redirect_next_asset_empty",
reason=("There were no eligible assets found to assign to the user."),
reason_code="no_asset_available",
asset=asset,
user=user,
mode=mode,
)
messages.info(request, no_pages_message)
return redirect("homepage")
@ratelimit(
key="header:cf-connecting-ip",
rate=next_asset_rate,
group="next_asset",
block=True,
)
@never_cache
@atomic
def redirect_to_next_reviewable_asset(
request: HttpRequest,
) -> HttpResponseRedirect:
"""
Redirect the user to a reviewable asset from any active reviewable
campaign.
This view iterates through campaigns marked as next-reviewable, then
falls back to other active campaigns if needed. It skips campaigns with
no eligible assets and uses asset caching when possible.
Args:
request (HttpRequest): Incoming HTTP request.
Returns:
HttpResponseRedirect: Redirect to the selected asset or the
homepage.
"""
structured_logger.info(
"Entered redirect_to_next_reviewable_asset view.",
event_code="redirect_reviewable_entry",
user=request.user,
)
if not request.user.is_authenticated:
user = get_anonymous_user()
else:
user = request.user
campaign_ids = list(
Campaign.objects.active()
.listed()
.published()
.get_next_review_campaigns()
.values_list("id", flat=True)
)
structured_logger.debug(
"Fetched candidate campaign IDs for reviewable assets.",
event_code="redirect_reviewable_campaign_ids",
user=user,
campaign_ids=campaign_ids,
)
asset = None
if campaign_ids:
random.shuffle(campaign_ids) # nosec
else:
logger.info("No configured reviewable campaigns")
structured_logger.info(
"No configured reviewable campaigns.",
event_code="redirect_reviewable_no_campaigns",
user=user,
)
for campaign_id in campaign_ids:
try:
campaign = Campaign.objects.get(id=campaign_id)
except IndexError:
logger.error("Next reviewable campaign %s not found", campaign_id)
structured_logger.error(
"Failed to retrieve next reviewable campaign by ID.",
event_code="redirect_reviewable_campaign_missing",
reason=("Reviewable campaign with specified ID was not found."),
reason_code="reviewable_campaign_not_found",
user=user,
campaign_id=campaign_id,
)
continue
asset = find_reviewable_campaign_asset(campaign, user)
if asset:
break
else:
logger.info("No reviewable assets found in %s", campaign)
structured_logger.info(
"No reviewable assets found in campaign.",
event_code="redirect_reviewable_campaign_empty_primary",
user=user,
campaign=campaign,
)
if not asset:
for campaign in (
Campaign.objects.active()
.listed()
.published()
.exclude(id__in=campaign_ids)
.order_by("launch_date")
):
asset = find_reviewable_campaign_asset(campaign, user)
if asset:
break
else:
logger.info("No reviewable assets found in %s", campaign)
structured_logger.info(
"No reviewable assets found in campaign.",
event_code="redirect_reviewable_campaign_empty_fallback",
user=user,
campaign=campaign,
)
structured_logger.info(
"Redirecting to next reviewable asset.",
event_code="redirect_reviewable_success",
user=user,
asset=asset,
)
return redirect_to_next_asset(asset, "review", request, user)
@ratelimit(
key="header:cf-connecting-ip",
rate=next_asset_rate,
group="next_asset",
block=True,
)
@never_cache
@atomic
def redirect_to_next_transcribable_asset(
request: HttpRequest,
) -> HttpResponseRedirect:
"""
Redirect the user to a transcribable asset from any active transcription
campaign.
This view iterates through campaigns marked as next-transcribable, then
falls back to other active campaigns if needed. It skips campaigns with
no eligible assets and uses asset caching when possible.
Args:
request (HttpRequest): Incoming HTTP request.
Returns:
HttpResponseRedirect: Redirect to the selected asset or the
homepage.
"""
structured_logger.info(
"Entered redirect_to_next_transcribable_asset view.",
event_code="redirect_transcribable_entry",
user=request.user,
)
campaign_ids = list(
Campaign.objects.active()
.listed()
.published()
.get_next_transcription_campaigns()
.values_list("id", flat=True)
)
structured_logger.debug(
"Fetched candidate campaign IDs for transcribable assets.",
event_code="redirect_transcribable_campaign_ids",
user=request.user,
campaign_ids=campaign_ids,
)
asset = None
if campaign_ids:
random.shuffle(campaign_ids) # nosec
else:
logger.info("No configured transcribable campaigns")
structured_logger.info(
"No configured transcribable campaigns.",
event_code="redirect_transcribable_no_campaigns",
user=request.user,
)
for campaign_id in campaign_ids:
try:
campaign = Campaign.objects.get(id=campaign_id)
except IndexError:
logger.error("Next transcribable campaign %s not found", campaign_id)
structured_logger.error(
"Next transcribable campaign ID not found.",
event_code="redirect_transcribable_campaign_missing",
reason=("Transcribable campaign with specified ID was not found."),
reason_code="transcribable_campaign_not_found",
user=request.user,
campaign_id=campaign_id,
)
continue
asset = find_transcribable_campaign_asset(campaign)
if asset:
break
else:
logger.info("No transcribable assets found in %s", campaign)
structured_logger.info(
"No transcribable assets found in campaign.",
event_code="redirect_transcribable_campaign_empty_primary",
user=request.user,
campaign=campaign,
)
if not asset:
for campaign in (
Campaign.objects.active()
.listed()
.published()
.exclude(id__in=campaign_ids)
.order_by("-launch_date")
):
asset = find_transcribable_campaign_asset(campaign)
if asset:
break
else:
logger.info("No transcribable assets found in %s", campaign)
structured_logger.info(
"No transcribable assets found in campaign (fallback " "loop).",
event_code="redirect_transcribable_campaign_empty_fallback",
user=request.user,
campaign=campaign,
)
if not asset:
logger.info("No transcribable assets found in any campaign")
structured_logger.info(
"No transcribable assets found in any campaign.",
event_code="redirect_transcribable_no_assets_anywhere",
user=request.user,
)
structured_logger.info(
"Redirecting to next transcribable asset.",
event_code="redirect_transcribable_success",
user=request.user,
asset=asset,
)
return redirect_to_next_asset(asset, "transcribe", request, request.user)
@ratelimit(
key="header:cf-connecting-ip",
rate=next_asset_rate,
group="next_asset",
block=True,
)
@never_cache
@atomic
def redirect_to_next_reviewable_campaign_asset(
request: HttpRequest,
*,
campaign_slug: str,
) -> HttpResponseRedirect:
"""
Redirect the user to the next reviewable asset within a campaign.
This view redirects within a specific campaign, which may be listed or
unlisted. It can use optional query parameters to influence which asset
is prioritized.
Request Parameters:
project (str): Current project slug.
item (str): Current item identifier. This is `item_id`, not the item
primary key.
asset (int): ID of the most recently reviewed asset.
Args:
request (HttpRequest): Incoming HTTP request.
campaign_slug (str): Slug for the target campaign.
Returns:
HttpResponseRedirect: Redirect to the selected asset or the
homepage.
"""
structured_logger.info(
"Entered redirect_to_next_reviewable_campaign_asset view.",
event_code="redirect_reviewable_campaign_entry",
user=request.user,
campaign_slug=campaign_slug,
)
# Campaign is specified: may be listed or unlisted
campaign = get_object_or_404(Campaign.objects.published(), slug=campaign_slug)
project_slug = request.GET.get("project", "")
item_id = request.GET.get("item", "")
asset_pk = request.GET.get("asset", 0)
structured_logger.debug(
"Parsed query parameters for reviewable asset redirection.",
event_code="redirect_reviewable_campaign_query_params",
user=request.user,
campaign=campaign,
project_slug=project_slug,
item_id=item_id,
asset_pk=asset_pk,
)
if not request.user.is_authenticated:
user = get_anonymous_user()
else:
user = request.user
# We pass request.user instead of user here to maintain pre-existing
# behavior (though it is probably unintended).
# TODO: Re-evaluate whether we should pass in user instead.
asset = find_next_reviewable_campaign_asset(
campaign, request.user, project_slug, item_id, asset_pk
)
structured_logger.info(
"Redirecting to next reviewable asset in campaign.",
event_code="redirect_reviewable_campaign_success",
user=user,
request_user=request.user,
asset=asset,
campaign=campaign, # We log campaign because asset might be None.
)
return redirect_to_next_asset(asset, "review", request, user)
@ratelimit(
key="header:cf-connecting-ip",
rate=next_asset_rate,
group="next_asset",
block=True,
)
@never_cache
@atomic
def redirect_to_next_transcribable_campaign_asset(
request: HttpRequest,
*,
campaign_slug: str,
) -> HttpResponseRedirect:
"""
Redirect the user to the next transcribable asset within a campaign.
This view redirects within a specific campaign, which may be listed or
unlisted. It can use optional query parameters to influence which asset
is prioritized.
Request Parameters:
project (str): Current project slug.
item (str): Current item identifier. This is `item_id`, not the item
primary key.
asset (int): ID of the most recently transcribed asset.
Args:
request (HttpRequest): Incoming HTTP request.
campaign_slug (str): Slug for the target campaign.
Returns:
HttpResponseRedirect: Redirect to the selected asset or the
homepage.
"""
structured_logger.info(
"Entered redirect_to_next_transcribable_campaign_asset view.",
event_code="redirect_transcribable_campaign_entry",
user=request.user,
campaign_slug=campaign_slug,
)
# Campaign is specified: may be listed or unlisted
campaign = get_object_or_404(Campaign.objects.published(), slug=campaign_slug)
project_slug = request.GET.get("project", "")
item_id = request.GET.get("item", "")
asset_pk = request.GET.get("asset", 0)
structured_logger.debug(
"Parsed query parameters for transcribable asset redirection.",
event_code="redirect_transcribable_campaign_query_params",
user=request.user,
campaign=campaign,
project_slug=project_slug,
item_id=item_id,
asset_pk=asset_pk,
)
if not request.user.is_authenticated:
user = get_anonymous_user()
else:
user = request.user
asset = find_next_transcribable_campaign_asset(
campaign, project_slug, item_id, asset_pk
)
structured_logger.info(
"Redirecting to next transcribable asset in campaign.",
event_code="redirect_transcribable_campaign_success",
user=user,
asset=asset,
campaign=campaign, # We log campaign because asset may be None.
)
return redirect_to_next_asset(asset, "transcribe", request, user)
@ratelimit(
key="header:cf-connecting-ip",
rate=next_asset_rate,
group="next_asset",
block=True,
)
@never_cache
@atomic
def redirect_to_next_reviewable_topic_asset(
request: HttpRequest,
*,
topic_slug: str,
) -> HttpResponseRedirect:
"""
Redirect the user to the next reviewable asset within a topic.
This view redirects within a specific topic, which may be listed or
unlisted. It can use optional query parameters to influence which asset
is prioritized.
Request Parameters:
project (str): Current project slug.
item (str): Current item identifier. This is `item_id`, not the item
primary key.
asset (int): ID of the most recently reviewed asset.
Args:
request (HttpRequest): Incoming HTTP request.
topic_slug (str): Slug for the target topic.
Returns:
HttpResponseRedirect: Redirect to the selected asset or the
homepage.
"""
structured_logger.info(
"Entered redirect_to_next_reviewable_topic_asset view.",
event_code="redirect_reviewable_topic_entry",
user=request.user,
topic_slug=topic_slug,
)
# Topic is specified: may be listed or unlisted
topic = get_object_or_404(Topic.objects.published(), slug=topic_slug)
project_slug = request.GET.get("project", "")
item_id = request.GET.get("item", "")
asset_pk = request.GET.get("asset", 0)
structured_logger.debug(
"Parsed query parameters for reviewable topic redirection.",
event_code="redirect_reviewable_topic_query_params",
user=request.user,
topic=topic,
project_slug=project_slug,
item_id=item_id,
asset_pk=asset_pk,
)
if not request.user.is_authenticated:
user = get_anonymous_user()
else:
user = request.user
# We pass request.user instead of user here to maintain pre-existing
# behavior (though it is probably unintended).
# TODO: Re-evaluate whether we should pass in user instead.
asset = find_next_reviewable_topic_asset(
topic, request.user, project_slug, item_id, asset_pk
)
structured_logger.info(
"Redirecting to next reviewable asset in topic.",
event_code="redirect_reviewable_topic_success",
user=user,
request_user=request.user,
asset=asset,
topic=topic,
)
return redirect_to_next_asset(asset, "review", request, user)
@ratelimit(
key="header:cf-connecting-ip",
rate=next_asset_rate,
group="next_asset",
block=True,
)
@never_cache
@atomic
def redirect_to_next_transcribable_topic_asset(
request: HttpRequest,
*,
topic_slug: str,
) -> HttpResponseRedirect:
"""
Redirect the user to the next transcribable asset within a topic.
This view redirects within a specific topic, which may be listed or
unlisted. It can use optional query parameters to influence which asset
is prioritized.
Request Parameters:
project (str): Current project slug.
item (str): Current item identifier. This is `item_id`, not the item
primary key.
asset (int): ID of the most recently transcribed asset.
Args:
request (HttpRequest): Incoming HTTP request.
topic_slug (str): Slug for the target topic.
Returns:
HttpResponseRedirect: Redirect to the selected asset or the
homepage.
"""
structured_logger.info(
"Entered redirect_to_next_transcribable_topic_asset view.",
event_code="redirect_transcribable_topic_entry",
user=request.user,
topic_slug=topic_slug,
)
# Topic is specified: may be listed or unlisted
topic = get_object_or_404(Topic.objects.published(), slug=topic_slug)
project_slug = request.GET.get("project", "")
item_id = request.GET.get("item", "")
asset_pk = request.GET.get("asset", 0)
structured_logger.debug(
"Parsed query parameters for transcribable topic redirection.",
event_code="redirect_transcribable_topic_query_params",
user=request.user,
topic=topic,
project_slug=project_slug,
item_id=item_id,
asset_pk=asset_pk,
)
if not request.user.is_authenticated:
user = get_anonymous_user()
else:
user = request.user
asset = find_next_transcribable_topic_asset(topic, project_slug, item_id, asset_pk)
structured_logger.info(
"Redirecting to next transcribable asset in topic.",
event_code="redirect_transcribable_topic_success",
user=user,
asset=asset,
topic=topic,
)
return redirect_to_next_asset(asset, "transcribe", request, user)
================================================
FILE: concordia/views/campaigns.py
================================================
from typing import Any, Iterable
from urllib.parse import urlencode
from django.core.paginator import Paginator
from django.db.models import Count, Q, QuerySet
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, render
from django.utils.decorators import method_decorator
from django.views.generic import TemplateView
from concordia.api_views import APIDetailView, APIListView
from concordia.models import (
STATUS_COUNT_KEYS,
Asset,
Campaign,
Project,
ResearchCenter,
SiteReport,
Topic,
Transcription,
TranscriptionStatus,
)
from concordia.utils.constants import ASSETS_PER_PAGE
from .decorators import default_cache_control, user_cache_control
from .utils import (
annotate_children_with_progress_stats,
calculate_asset_stats,
)
@method_decorator(default_cache_control, name="dispatch")
class CampaignListView(APIListView):
"""
Display a list of active campaigns.
Renders a list of published, listed, and active campaigns ordered by
their configured ordering and title. Adds context entries for topics
and completed campaigns for secondary display.
Inherits from APIListView to support both HTML rendering and API
serialization of campaigns.
Attributes:
template_name (str): Template used to render the campaign list.
queryset (QuerySet[Campaign]): The base queryset of campaigns.
context_object_name (str): The name of the context variable for campaigns.
Returns:
HttpResponse: Renders the campaign list template with context.
"""
template_name = "transcriptions/campaign_list.html"
queryset = (
Campaign.objects.published()
.listed()
.filter(status=Campaign.Status.ACTIVE)
.order_by("ordering", "title")
)
context_object_name = "campaigns"
def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
"""
Build context data for the campaign list template.
Adds:
- 'topics': Ordered list of published topics.
- 'completed_campaigns': Ordered list of completed or retired campaigns.
Args:
**kwargs: Additional context arguments.
Returns:
dict[str, Any]: Context data for rendering.
"""
data = super().get_context_data(**kwargs)
data["topics"] = (
Topic.objects.published().listed().order_by("ordering", "title")
)
data["completed_campaigns"] = (
Campaign.objects.published()
.listed()
.filter(status__in=[Campaign.Status.COMPLETED, Campaign.Status.RETIRED])
.order_by("ordering", "title")
)
return data
def serialize_context(self, context: dict[str, Any]) -> dict[str, Any]:
"""
Serialize context data for API responses.
Annotates each campaign object with its asset status counts.
Args:
context (dict[str, Any]): The view context.
Returns:
dict[str, Any]: Serialized context data for API output.
"""
data = super().serialize_context(context)
object_list = data["objects"]
campaign_stats_qs = (
Campaign.objects.filter(pk__in=[i["id"] for i in object_list])
.annotate(
**{
v: Count(
"project__item__asset",
filter=Q(
project__published=True,
project__item__published=True,
project__item__asset__published=True,
project__item__asset__transcription_status=k,
),
)
for k, v in STATUS_COUNT_KEYS.items()
}
)
.values("pk", *STATUS_COUNT_KEYS.values())
)
campaign_asset_counts = {}
for campaign_stats in campaign_stats_qs:
campaign_asset_counts[campaign_stats.pop("pk")] = campaign_stats
for obj in object_list:
obj["asset_stats"] = campaign_asset_counts[obj["id"]]
return data
@method_decorator(default_cache_control, name="dispatch")
class CompletedCampaignListView(APIListView):
"""
Display a list of completed and/or retired campaigns.
Renders a list of published, listed campaigns filtered by completion or
retirement status. Optionally filters by research center or campaign type.
Attributes:
model (Model): The Campaign model class.
template_name (str): Template used to render the campaign list.
context_object_name (str): The name of the context variable for campaigns.
Returns:
HttpResponse: Renders the completed campaign list template with context.
"""
model = Campaign
template_name = "transcriptions/campaign_list_small_blocks.html"
context_object_name = "campaigns"
def _get_all_campaigns(self) -> QuerySet[Campaign]:
"""
Retrieve all completed or retired campaigns, optionally filtered by type.
Returns:
QuerySet[Campaign]: Filtered campaigns.
"""
campaignType = self.request.GET.get("type", None)
campaigns = Campaign.objects.published().listed()
if campaignType is None:
return campaigns.filter(
status__in=[Campaign.Status.COMPLETED, Campaign.Status.RETIRED]
)
elif campaignType == "retired":
status = Campaign.Status.RETIRED
else:
status = Campaign.Status.COMPLETED
return campaigns.filter(status=status)
def get_queryset(self) -> QuerySet[Campaign]:
"""
Build the queryset of completed or retired campaigns.
Optionally filters by research center if provided.
Returns:
QuerySet[Campaign]: The queryset for completed campaigns.
"""
campaigns = self._get_all_campaigns()
research_center = self.request.GET.get("research_center", None)
if research_center is not None:
campaigns = campaigns.filter(research_centers=research_center)
return campaigns.order_by("-completed_date")
def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
"""
Build context data for the completed campaign list template.
Adds:
- 'result_count': The number of campaigns in the list.
- 'research_centers': Distinct research centers for these campaigns.
Args:
**kwargs: Additional context arguments.
Returns:
dict[str, Any]: Context data for rendering.
"""
campaigns = self._get_all_campaigns()
data = super().get_context_data(**kwargs)
data["result_count"] = self.object_list.count()
data["research_centers"] = ResearchCenter.objects.filter(
campaign__in=campaigns
).distinct()
return data
@method_decorator(default_cache_control, name="dispatch")
class CampaignTopicListView(TemplateView):
"""
Display a list of campaigns grouped by topic.
Renders active campaigns, a subset of topics and completed/retired campaigns
for navigation and discovery pages.
Attributes:
template_name (str): Template used to render the campaign-topic list page.
Returns:
HttpResponse: Renders the campaign topic list template with context.
"""
template_name = "transcriptions/campaign_topic_list.html"
def get(self, request, *args: Any, **kwargs: Any) -> HttpResponse:
"""
Handle GET requests for the campaign-topic list page.
Builds context containing:
- 'campaigns': Ordered list of active campaigns.
- 'topics': Ordered list of up to 5 topics.
- 'completed_campaigns': Ordered list of completed and retired campaigns.
Args:
request (HttpRequest): The incoming HTTP request.
*args: Additional positional arguments.
**kwargs: Additional keyword arguments.
Returns:
HttpResponse: Rendered campaign topic list page.
"""
data = {}
data["campaigns"] = (
Campaign.objects.published()
.listed()
.filter(status=Campaign.Status.ACTIVE)
.annotated()
.order_by("ordering", "title")
)
data["topics"] = (
Topic.objects.published().listed().order_by("ordering", "title")[:5]
)
data["completed_campaigns"] = (
Campaign.objects.published()
.listed()
.filter(status__in=[Campaign.Status.COMPLETED, Campaign.Status.RETIRED])
.order_by("ordering", "title")
)
return render(request, self.template_name, data)
@method_decorator(default_cache_control, name="dispatch")
class CampaignDetailView(APIDetailView):
"""
Display details for a single campaign.
Renders campaign information, associated projects, and aggregated asset
statistics. Selects different templates based on campaign status
(active, completed, or retired).
Attributes:
template_name (str): Template for active campaigns.
completed_template_name (str): Template for completed campaigns.
retired_template_name (str): Template for retired campaigns.
context_object_name (str): Context variable name for the campaign.
queryset (QuerySet[Campaign]): Base queryset of campaigns.
Returns:
HttpResponse: Renders the campaign detail template with context.
"""
template_name = "transcriptions/campaign_detail.html"
completed_template_name = "transcriptions/campaign_detail_completed.html"
retired_template_name = "transcriptions/campaign_detail_retired.html"
context_object_name = "campaign"
queryset = Campaign.objects.published().order_by("title")
def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
"""
Build context data for the campaign detail page.
Adds:
- For retired campaigns: contributor and completed counts from SiteReport.
- For active campaigns: filtered and annotated projects, asset statistics.
Args:
**kwargs: Additional context arguments.
Returns:
dict[str, Any]: Context data for rendering.
"""
ctx = super().get_context_data(**kwargs)
if self.object and self.object.status == Campaign.Status.RETIRED:
latest_report = SiteReport.objects.filter(campaign=ctx["campaign"]).latest(
"created_on"
)
ctx["completed_count"] = latest_report.assets_completed
ctx["contributor_count"] = latest_report.registered_contributors
else:
projects = (
ctx["campaign"].project_set.published().order_by("ordering", "title")
)
ctx["filters"] = filters = {}
filter_by_reviewable = kwargs.get("filter_by_reviewable", False)
if filter_by_reviewable:
projects = projects.filter(
item__asset__transcription__id__in=Transcription.objects.exclude(
user=self.request.user.id
).values_list("id", flat=True)
)
ctx["filter_assets"] = True
projects = projects.annotate(
**{
f"{key}_count": Count(
"item__asset",
filter=Q(
item__published=True,
item__asset__published=True,
item__asset__transcription_status=key,
),
)
for key in TranscriptionStatus.CHOICE_MAP
}
)
if filter_by_reviewable:
status = TranscriptionStatus.SUBMITTED
else:
status = self.request.GET.get("transcription_status")
if status in TranscriptionStatus.CHOICE_MAP:
projects = projects.exclude(**{f"{status}_count": 0})
# We only want to pass specific QS parameters
# to lower-level search pages:
filters["transcription_status"] = status
ctx["sublevel_querystring"] = urlencode(filters)
annotate_children_with_progress_stats(projects)
ctx["projects"] = projects
campaign_assets = Asset.objects.filter(
item__project__campaign=self.object,
item__project__published=True,
item__published=True,
published=True,
)
if filter_by_reviewable:
campaign_assets = campaign_assets.exclude(
transcription__user=self.request.user.id
)
ctx["transcription_status"] = TranscriptionStatus.SUBMITTED
else:
ctx["transcription_status"] = status
calculate_asset_stats(campaign_assets, ctx)
return ctx
def serialize_context(self, context: dict[str, Any]) -> dict[str, Any]:
"""
Serialize campaign context data for API responses.
Adds:
- 'related_links': Helpful Link title and URL pairs for the campaign.
Args:
context (dict[str, Any]): The view context.
Returns:
dict[str, Any]: Serialized context data for API output.
"""
ctx = super().serialize_context(context)
ctx["object"]["related_links"] = [
{"title": title, "url": url}
for title, url in self.object.helpfullink_set.values_list(
"title", "link_url"
)
]
return ctx
def get_template_names(self) -> list[str]:
"""
Determine the template to use based on campaign status.
Returns:
list[str]: List containing the selected template name.
"""
if self.object.status == Campaign.Status.COMPLETED:
return [self.completed_template_name]
elif self.object.status == Campaign.Status.RETIRED:
return [self.retired_template_name]
return super().get_template_names()
@method_decorator(user_cache_control, name="dispatch")
class FilteredCampaignDetailView(CampaignDetailView):
"""
Display campaign details with reviewable asset filtering for staff users.
Inherits from CampaignDetailView, overriding context data to include only
assets eligible for review by staff users when authenticated.
Returns:
HttpResponse: Renders the filtered campaign detail template with context.
"""
def get_context_data(self, **kwargs: Any) -> dict[str, Any]:
"""
Build context data with reviewable asset filtering for staff users.
Adds 'filter_by_reviewable' to kwargs when user is authenticated and staff.
Args:
**kwargs: Additional context arguments.
Returns:
dict[str, Any]: Context data for rendering.
"""
if self.request.user.is_authenticated and self.request.user.is_staff:
kwargs["filter_by_reviewable"] = True
return super().get_context_data(**kwargs)
@method_decorator(default_cache_control, name="dispatch")
class ReportCampaignView(TemplateView):
"""
Display a report summarizing campaign resources and status.
Renders a paginated report including project-level asset counts, tag counts,
contributor counts, reviewer counts and transcription status summaries.
Attributes:
template_name (str): Template used to render the campaign report page.
Returns:
HttpResponse: Renders the campaign report template with context.
"""
template_name = "transcriptions/campaign_report.html"
def get(
self, request, campaign_slug: str, *args: Any, **kwargs: Any
) -> HttpResponse:
"""
Handle GET requests for the campaign report page.
Builds context containing:
- Campaign title and slug
- Total asset count
- Paginated projects with asset, tag, transcriber and reviewer counts
- Transcription status summaries per project
Args:
request (HttpRequest): The incoming HTTP request.
campaign_slug (str): Slug for the campaign to report on.
*args: Additional positional arguments.
**kwargs: Additional keyword arguments.
Returns:
HttpResponse: Rendered campaign report page.
"""
campaign = get_object_or_404(Campaign.objects.published(), slug=campaign_slug)
try:
page = int(self.request.GET.get("page", "1"))
except ValueError:
page = 1
campaign_assets = Asset.objects.published().filter(
item__project__campaign=campaign
)
ctx = {
"title": campaign.title,
"campaign_slug": campaign.slug,
"total_asset_count": campaign_assets.count(),
}
projects_qs = campaign.project_set.published().order_by("title")
projects_qs = projects_qs.annotate(
asset_count=Count(
"item__asset",
filter=Q(item__published=True, item__asset__published=True),
distinct=True,
)
)
projects_qs = projects_qs.annotate(
tag_count=Count("item__asset__userassettagcollection__tags", distinct=True)
)
projects_qs = projects_qs.annotate(
transcriber_count=Count("item__asset__transcription__user", distinct=True),
reviewer_count=Count(
"item__asset__transcription__reviewed_by", distinct=True
),
)
paginator = Paginator(projects_qs, ASSETS_PER_PAGE)
if page > paginator.num_pages:
page = 1
projects_page = paginator.get_page(page)
self.add_transcription_status_summary_to_projects(projects_page)
ctx["paginator"] = paginator
ctx["projects"] = projects_page
return render(self.request, self.template_name, ctx)
def add_transcription_status_summary_to_projects(
self, projects: Iterable[Project]
) -> None:
"""
Annotate each project with a summary of transcription statuses.
Adds a 'transcription_statuses' attribute to each project, containing
status names and their respective counts, ordered by status.
Args:
projects (Iterable): Projects to annotate.
Returns:
None
"""
status_qs = Asset.objects.filter(
item__published=True, item__project__in=projects, published=True
)
status_qs = status_qs.values_list("item__project__id", "transcription_status")
status_qs = status_qs.annotate(Count("transcription_status"))
project_statuses = {}
for project_id, status_value, count in status_qs:
status_name = TranscriptionStatus.CHOICE_MAP[status_value]
project_statuses.setdefault(project_id, []).append((status_name, count))
# We'll sort the statuses in the same order they're presented in the choices
# list so the display order will be both stable and consistent with the way
# we talk about the workflow:
sort_order = [j for i, j in TranscriptionStatus.CHOICES]
for project in projects:
statuses = project_statuses.get(project.id, [])
statuses.sort(key=lambda i: sort_order.index(i[0]))
project.transcription_statuses = statuses
================================================
FILE: concordia/views/decorators.py
================================================
from collections.abc import Callable
from functools import wraps
from time import time
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.http import HttpRequest, JsonResponse
from django.views.decorators.cache import cache_control, never_cache
from django.views.decorators.vary import vary_on_headers
from concordia.forms import TurnstileForm
from concordia.logging import ConcordiaLogger
from configuration.utils import configuration_value
from configuration.validation import validate_rate
structured_logger = ConcordiaLogger.get_logger(__name__)
def default_cache_control(view_function: Callable) -> Callable:
"""
Decorator that applies default cache control headers to public-facing views.
This decorator sets `Cache-Control: public` with a max-age defined in the
`DEFAULT_PAGE_TTL` Django setting. It also varies the response by the
`Accept-Encoding` header.
Args:
view_function (Callable): The view function to decorate.
Returns:
Callable: The wrapped view function with cache control headers applied.
"""
@vary_on_headers("Accept-Encoding")
@cache_control(public=True, max_age=settings.DEFAULT_PAGE_TTL)
@wraps(view_function)
def inner(*args, **kwargs):
return view_function(*args, **kwargs)
return inner
def user_cache_control(view_function: Callable) -> Callable:
"""
Decorator that applies cache control headers for views varying by session.
This decorator is intended for views that may return different content
based on whether the user is authenticated. It sets
`Cache-Control: public` with the `DEFAULT_PAGE_TTL` setting and varies
the response by both `Accept-Encoding` and `Cookie` headers.
Args:
view_function (Callable): The view function to decorate.
Returns:
Callable: The wrapped view function with user-aware cache control
headers.
"""
@vary_on_headers("Accept-Encoding", "Cookie")
@cache_control(public=True, max_age=settings.DEFAULT_PAGE_TTL)
@wraps(view_function)
def inner(*args, **kwargs):
return view_function(*args, **kwargs)
return inner
def validate_anonymous_user(view: Callable) -> Callable:
"""
Decorator that applies anonymous user validation for `POST` requests.
If the user is unauthenticated and submits a `POST` request, this
decorator checks whether the user has recently passed Turnstile
validation. If not, it validates the request using a `TurnstileForm`.
Failing validation returns a 401 JSON response.
The timestamp of a successful validation is stored in the user session
to avoid re-validating within the configured interval.
Args:
view (Callable): The view function to wrap.
Returns:
Callable: The wrapped view function with anonymous user validation
logic.
"""
@wraps(view)
@never_cache
def inner(request, *args, **kwargs):
if not request.user.is_authenticated and request.method == "POST":
# First check if the user has already been validated within the
# time limit. If so, validation can be skipped.
turnstile_last_validated = request.session.get(
"turnstile_last_validated", 0
)
age = time() - turnstile_last_validated
if age > settings.ANONYMOUS_USER_VALIDATION_INTERVAL:
form = TurnstileForm(request.POST)
if not form.is_valid():
return JsonResponse(
{
"error": (
"Unable to validate. Please try again or " "login."
)
},
status=401,
)
else:
# User has been validated, so cache the time in the
# session.
request.session["turnstile_last_validated"] = time()
return view(request, *args, **kwargs)
return inner
def reserve_rate(group: str, request: HttpRequest) -> str | None:
"""
Determine the rate limit value for a request.
This helper is used to control throttling behavior. If the user is
anonymous, it returns a fixed rate limit string, for example "100/m".
Authenticated users are not rate-limited and it returns `None`.
The `group` parameter controls how rate limits are grouped. It defaults
to the dotted name of the view so each view is treated as its own rate
limit bucket unless explicitly overridden.
Args:
group (str): Group name used to bucket rate limits. Defaults to the
dotted view name if not set manually.
request (HttpRequest): The incoming HTTP request.
Returns:
str | None: A rate string such as "100/m" for anonymous users, or
`None` otherwise.
"""
return None if request.user.is_authenticated else "100/m"
def next_asset_rate(group: str, request: HttpRequest) -> str | None:
"""
Determine the rate limit value for a next-asset request.
If the user is anonymous, this helper returns a rate limit string from
the `next_asset_rate_limit` configuration value, for example "4/m".
Authenticated users are not rate-limited and it returns `None`.
The `group` parameter controls how rate limits are grouped. It is used
internally by `django-ratelimit`. It could be used to return different
rate limits based on the group, but that is not needed currently.
Args:
group (str): Group name used to bucket rate limits. Defaults to the
dotted view name if not set manually.
request (HttpRequest): The incoming HTTP request.
Returns:
str | None: A rate string such as "4/m" for anonymous users, or
`None` otherwise.
"""
if request.user.is_authenticated:
return None
try:
rate_limit = configuration_value("next_asset_rate_limit")
return validate_rate(rate_limit)
except (ObjectDoesNotExist, ValidationError) as exc:
structured_logger.warning(
"Falling back to default next-asset rate limit.",
event_code="next_asset_rate_config_fallback",
reason="Could not load or validate configured rate limit",
reason_code="config_missing_or_invalid",
group=group,
default_rate="4/m",
user=request.user,
error_type=exc.__class__.__name__,
error=str(exc),
)
return "4/m"
================================================
FILE: concordia/views/items.py
================================================
from urllib.parse import urlencode
from django.db.models import QuerySet
from django.http import Http404, HttpRequest, HttpResponse
from django.shortcuts import get_object_or_404, redirect
from django.utils.decorators import method_decorator
from concordia.api_views import APIListView
from concordia.models import Campaign, Item, TranscriptionStatus
from concordia.utils import get_image_urls_from_asset
from .decorators import default_cache_control, user_cache_control
from .utils import calculate_asset_stats
@method_decorator(default_cache_control, name="dispatch")
class ItemDetailView(APIListView):
"""
Display a paginated list of assets for a specific item.
This view handles GET requests and renders the item detail page,
which includes the item's assets, context for filtering, and transcription stats.
Uses `APIListView` to support both HTML rendering and optional JSON output for
frontend consumption.
Attributes:
template_name (str): Template used to render the asset list.
context_object_name (str): The variable name for assets in the template.
paginate_by (int): Number of assets to display per page.
http_method_names (list[str]): HTTP methods supported by the view.
"""
template_name = "transcriptions/item_detail.html"
context_object_name = "assets"
paginate_by = 10
http_method_names = ["get", "options", "head"]
def dispatch(self, request: HttpRequest, *args, **kwargs) -> HttpResponse:
"""
Handle incoming HTTP requests and redirect if the item or campaign is missing.
Args:
request (HttpRequest): The HTTP request object.
Returns:
HttpResponse: The response for the view or a redirect to the campaign page.
"""
try:
return super().dispatch(request, *args, **kwargs)
except Http404:
campaign = get_object_or_404(
Campaign.objects.published(), slug=self.kwargs["campaign_slug"]
)
return redirect(campaign)
def _get_assets(self) -> QuerySet:
"""
Retrieve the queryset of published assets for the current item.
If the `filter_by_reviewable` flag is set in `self.kwargs`, excludes any assets
already transcribed by the current user to allow for review filtering.
Returns:
QuerySet: The filtered set of published `Asset` objects for the item.
"""
assets = self.item.asset_set.published()
if self.kwargs.get("filter_by_reviewable", False):
assets = assets.exclude(transcription__user=self.request.user.id)
return assets
def get_queryset(self):
"""
Build and return the queryset of assets for the current item.
Retrieves the specified `Item` and filters its published assets based on
transcription status. If `filter_by_reviewable` is set in `self.kwargs`,
filters assets not yet transcribed by the current user and restricts the
status to SUBMITTED.
The resulting queryset is ordered by sequence and annotated for use in the view.
Also sets `self.filters` for use in context and querystring construction.
Returns:
QuerySet: The filtered and ordered queryset of `Asset` objects.
"""
self.item = get_object_or_404(
Item.objects.published().select_related("project__campaign"),
project__campaign__slug=self.kwargs["campaign_slug"],
project__slug=self.kwargs["project_slug"],
item_id=self.kwargs["item_id"],
)
asset_qs = self._get_assets().order_by("sequence")
asset_qs = asset_qs.select_related(
"item__project__campaign", "item__project", "item"
)
self.filters = {}
if self.kwargs.get("filter_by_reviewable", False):
status = TranscriptionStatus.SUBMITTED
else:
status = self.request.GET.get("transcription_status")
if status in TranscriptionStatus.CHOICE_MAP:
asset_qs = asset_qs.filter(transcription_status=status)
# We only want to pass specific QS parameters to lower-level search
# pages so we'll record those here:
self.filters["transcription_status"] = status
return asset_qs
def get_context_data(self, **kwargs):
"""
Construct the context dictionary for rendering the item detail page.
Adds campaign, project, item and transcription status to the context. Also
attaches filter state and querystring for pagination or navigation. If review
filtering is enabled, the context is marked accordingly.
Includes asset-level transcription statistics via `calculate_asset_stats()`.
Returns:
dict: The context data for the template rendering.
"""
ctx = super().get_context_data(**kwargs)
ctx.update(
{
"campaign": self.item.project.campaign,
"project": self.item.project,
"item": self.item,
"sublevel_querystring": urlencode(self.filters),
"filters": self.filters,
}
)
item_assets = self._get_assets()
if self.kwargs.get("filter_by_reviewable", False):
ctx["filter_assets"] = True
ctx["transcription_status"] = TranscriptionStatus.SUBMITTED
else:
ctx["transcription_status"] = self.request.GET.get("transcription_status")
calculate_asset_stats(item_assets, ctx)
return ctx
def serialize_context(self, context: dict) -> dict:
"""
Serialize the context data for JSON responses.
Enhances each serialized asset with its associated image and thumbnail URLs.
Also includes serialized data for the parent item.
Args:
context (dict): The original context data returned by `get_context_data()`.
Returns:
dict: The serialized version of the context suitable for API responses.
"""
data = super().serialize_context(context)
for i, asset in enumerate(context["object_list"]):
serialized_asset = data["objects"][i]
image_url, thumbnail_url = get_image_urls_from_asset(asset)
serialized_asset["image_url"] = image_url
serialized_asset["thumbnail_url"] = thumbnail_url
data["item"] = self.serialize_object(context["item"])
return data
@method_decorator(user_cache_control, name="dispatch")
class FilteredItemDetailView(ItemDetailView):
"""
View that displays only reviewable assets for an item.
Inherits from `ItemDetailView` but overrides queryset and context behavior to
exclude assets already transcribed by the current user. Used to present assets
eligible for review.
"""
def get_queryset(self):
"""
Modify the queryset to include only reviewable assets.
Sets the `filter_by_reviewable` flag in `self.kwargs` to enable filtering logic
in the parent view.
Returns:
QuerySet: A filtered queryset of `Asset` objects for review.
"""
self.kwargs["filter_by_reviewable"] = True
return super().get_queryset()
def get_context_data(self, **kwargs):
"""
Update the context to reflect that only reviewable assets are being shown.
Ensures `filter_by_reviewable` is set in both `self.kwargs` and `kwargs` so that
downstream logic (like filtering and labeling) behaves consistently.
Returns:
dict: The context dictionary for rendering the filtered item detail view.
"""
self.kwargs["filter_by_reviewable"] = True
kwargs["filter_by_reviewable"] = True
return super().get_context_data(**kwargs)
================================================
FILE: concordia/views/maintenance_mode.py
================================================
from time import time
from django.core.cache import cache
from django.http import HttpRequest, HttpResponseRedirect
from maintenance_mode.core import set_maintenance_mode
def maintenance_mode_off(request: HttpRequest) -> HttpResponseRedirect:
"""
Deactivates maintenance mode and redirects to the site root.
Only superusers are allowed to use this view. If the requesting user is not a
superuser, no change is made to the system state.
Returns:
HttpResponseRedirect: Redirect to the root path with a timestamp parameter
used for cache busting.
"""
if request.user.is_superuser:
set_maintenance_mode(False)
# Added cache busting to make sure maintenance mode banner is
# always displayed/removed
return HttpResponseRedirect("/?t={}".format(int(time())))
def maintenance_mode_on(request: HttpRequest) -> HttpResponseRedirect:
"""
Activates maintenance mode and redirects to the site root.
Only superusers are allowed to use this view. If the requesting user is not a
superuser, no change is made to the system state.
Returns:
HttpResponseRedirect: Redirect to the root path with a timestamp parameter
used for cache busting.
"""
if request.user.is_superuser:
set_maintenance_mode(True)
# Added cache busting to make sure maintenance mode banner is
# always displayed/removed
return HttpResponseRedirect("/?t={}".format(int(time())))
def maintenance_mode_frontend_available(request: HttpRequest) -> HttpResponseRedirect:
"""
Enables frontend access during maintenance mode and redirects to the site root.
This sets a cache key (`maintenance_mode_frontend_available`) to allow staff and
superusers to bypass maintenance restrictions while the site is otherwise disabled.
Only superusers are allowed to use this view.
Returns:
HttpResponseRedirect: Redirect to the root path with a timestamp parameter
used for cache busting.
"""
if request.user.is_superuser:
cache.set("maintenance_mode_frontend_available", True, None)
return HttpResponseRedirect("/?t={}".format(int(time())))
def maintenance_mode_frontend_unavailable(request: HttpRequest) -> HttpResponseRedirect:
"""
Disables frontend access during maintenance mode and redirects to the site root.
This clears the `maintenance_mode_frontend_available` cache key, fully locking out
all users (including staff) from the site frontend during maintenance mode.
Only superusers are allowed to use this view.
Returns:
HttpResponseRedirect: Redirect to the root path with a timestamp parameter
used for cache busting.
"""
if request.user.is_superuser:
cache.set("maintenance_mode_frontend_available", False, None)
return HttpResponseRedirect("/?t={}".format(int(time())))
================================================
FILE: concordia/views/projects.py
================================================
from urllib.parse import urlencode
from django.db.models import Count, Q, QuerySet
from django.http import Http404, HttpRequest, HttpResponse
from django.shortcuts import get_object_or_404, redirect
from django.utils.decorators import method_decorator
from concordia.api_views import APIListView
from concordia.models import Asset, Campaign, Project, TranscriptionStatus
from .decorators import default_cache_control, user_cache_control
from .utils import annotate_children_with_progress_stats, calculate_asset_stats
@method_decorator(default_cache_control, name="dispatch")
class ProjectDetailView(APIListView):
"""
Display a paginated list of items for a single project.
Handles GET requests for a published project scoped by campaign and project
slugs. Applies optional filtering to show only items with a specific
transcription status. Builds context including campaign/project metadata
and progress statistics.
Attributes:
template_name (str): Template used for project detail.
context_object_name (str): Context key under which the list of items is
exposed to templates.
paginate_by (int): Number of items per page.
Returns:
HttpResponse: Rendered project detail page or a redirect if the project
or campaign cannot be found.
"""
template_name = "transcriptions/project_detail.html"
context_object_name = "items"
paginate_by = 10
def dispatch(self, request: HttpRequest, *args, **kwargs) -> HttpResponse:
"""
Dispatch the request or redirect to campaign if the item is missing.
If an `Http404` occurs during normal dispatch (e.g., item not found),
redirect to the campaign page to keep navigation stable.
Args:
request (HttpRequest): The incoming request.
*args: Positional args forwarded to the superclass.
**kwargs: Keyword args forwarded to the superclass.
Returns:
HttpResponse: Normal response from `APIListView.dispatch` or a
redirect to the campaign page when the asset path is invalid.
"""
try:
return super().dispatch(request, *args, **kwargs)
except Http404:
campaign = get_object_or_404(
Campaign.objects.published(), slug=self.kwargs["campaign_slug"]
)
return redirect(campaign)
def get_queryset(self, filter_by_reviewable: bool = False) -> QuerySet:
"""
Return the queryset of items for the current project.
Loads the published project identified by `campaign_slug` and `slug`,
then builds an ordered queryset of its published items. When
`filter_by_reviewable` is true, excludes items already transcribed by
the requesting user. Each item is annotated with per-status counts
using `TranscriptionStatus.CHOICE_MAP`.
Request Parameters:
- `transcription_status` (str, optional): If present and valid,
restrict the queryset to items that have at least one asset in
that status (items with zero count for that status are excluded).
Args:
filter_by_reviewable (bool): If true, exclude items containing an
asset with a transcription by the current user.
Returns:
QuerySet: Ordered queryset of items annotated with per-status
counts. Also sets `self.filters` to the propagated filters that
should appear in sublevel navigation links.
"""
self.project = get_object_or_404(
Project.objects.published().select_related("campaign"),
slug=self.kwargs["slug"],
campaign__slug=self.kwargs["campaign_slug"],
)
item_qs = self.project.item_set.published().order_by("item_id")
if filter_by_reviewable:
item_qs = item_qs.exclude(asset__transcription__user=self.request.user.id)
item_qs = item_qs.annotate(
**{
f"{key}_count": Count(
"asset", filter=Q(asset__transcription_status=key)
)
for key in TranscriptionStatus.CHOICE_MAP
}
)
self.filters: dict[str, str] = {}
if filter_by_reviewable:
status = TranscriptionStatus.SUBMITTED
else:
status = self.request.GET.get("transcription_status")
if status in TranscriptionStatus.CHOICE_MAP:
item_qs = item_qs.exclude(**{f"{status}_count": 0})
# We only want to pass specific QS parameters to lower-level search
# pages so we'll record those here:
self.filters["transcription_status"] = status
return item_qs
def get_context_data(self, **kws) -> dict[str, object]:
"""
Build context for the project detail template.
Context Format:
- `items` (QuerySet): Paginated list of project items.
- `project` (Project): The current project.
- `campaign` (Campaign): The parent campaign.
- `filters` (dict[str, str] | absent): Filters applied to this view.
- `sublevel_querystring` (str | absent): URL-encoded filters to pass
into sublevel pages.
- `transcription_status` (str | None): Current status filter or
derived status when reviewable filtering is active.
- `filter_assets` (bool | absent): True when items are filtered to
exclude those transcribed by the current user.
- `total_assets` (int): Count of assets in the project (published).
- `completed_assets` (int): Count of completed assets.
- `in_progress_assets` (int): Count of in-progress assets.
- `not_started_assets` (int): Count of not-started assets.
- `submitted_assets` (int): Count of submitted assets.
- Progress statistics on each item are added in-place by
`annotate_children_with_progress_stats`.
Args:
**kws: Optional flags. Recognized:
- `filter_by_reviewable` (bool): When true, limit assets and
force `transcription_status` to `SUBMITTED`.
Returns:
dict[str, object]: Template context including project/campaign
metadata, filters, and computed statistics.
"""
ctx = super().get_context_data(**kws)
ctx["project"] = project = self.project
ctx["campaign"] = project.campaign
if self.filters:
ctx["sublevel_querystring"] = urlencode(self.filters)
ctx["filters"] = self.filters
project_assets = Asset.objects.filter(
item__project=project, published=True, item__published=True
)
filter_by_reviewable = kws.get("filter_by_reviewable", False)
if filter_by_reviewable:
project_assets = project_assets.exclude(
transcription__user=self.request.user.id
)
ctx["filter_assets"] = True
ctx["transcription_status"] = TranscriptionStatus.SUBMITTED
else:
ctx["transcription_status"] = self.request.GET.get("transcription_status")
calculate_asset_stats(project_assets, ctx)
annotate_children_with_progress_stats(ctx["items"])
return ctx
def serialize_context(self, context: dict[str, object]) -> dict[str, object]:
"""
Serialize context for API responses.
Extends the base list serialization by attaching the serialized project
object. Mirrors the behavior used elsewhere to pair list payloads with
their parent container.
Args:
context (dict[str, object]): The view context to serialize.
Returns:
dict[str, object]: A JSON-serializable structure including:
- `results`/pagination fields from the base serializer.
- `project` (dict): Serialized project metadata.
"""
data = super().serialize_context(context)
data["project"] = self.serialize_object(context["project"])
return data
@method_decorator(user_cache_control, name="dispatch")
class FilteredProjectDetailView(ProjectDetailView):
"""
Project detail view that filters to reviewable items for the user.
This variant restricts the queryset and context to prioritize items that
are ready for review by the current user (i.e., excludes items with assets
already transcribed by that user). It also sets the effective status filter
to `SUBMITTED` in context.
"""
def get_queryset(self) -> QuerySet:
"""
Return the review-focused queryset.
Delegates to the parent implementation with `filter_by_reviewable=True`.
Returns:
QuerySet: Item queryset annotated with status counts and filtered
for reviewable content.
"""
return super().get_queryset(filter_by_reviewable=True)
def get_context_data(self, **kws) -> dict[str, object]:
"""
Build context with reviewable filtering enabled.
Sets the `filter_by_reviewable` flag before delegating to the parent
implementation so that downstream context keys (e.g., status and
`filter_assets`) reflect review-mode behavior.
Args:
**kws: Context keyword arguments.
Returns:
dict[str, object]: Context dictionary with reviewable filtering.
"""
kws["filter_by_reviewable"] = True
return super().get_context_data(**kws)
================================================
FILE: concordia/views/rate_limit.py
================================================
from django.http import HttpRequest, HttpResponse, JsonResponse
from django.shortcuts import render
from concordia.utils import request_accepts_json
def ratelimit_view(
request: HttpRequest, exception: Exception | None = None
) -> HttpResponse:
"""
Handles requests blocked due to rate limiting (HTTP 429).
Determines whether to return a JSON or HTML response based on the request headers.
Adds a `Retry-After` header instructing clients to wait 15 minutes before retrying.
Args:
request (HttpRequest): The incoming request that triggered the rate limit.
exception (Exception | None): The exception that caused the view to trigger,
if available.
Returns:
HttpResponse: A JSON or HTML 429 response with a retry header.
"""
status_code = 429
ctx = {
"error": "You have been rate-limited. Please try again later.",
"status": status_code,
}
if exception is not None:
ctx["exception"] = str(exception)
if request.headers.get(
"x-requested-with"
) == "XMLHttpRequest" or request_accepts_json(request):
response = JsonResponse(ctx, status=status_code)
else:
response = render(request, "429.html", context=ctx, status=status_code)
response["Retry-After"] = 15 * 60
return response
================================================
FILE: concordia/views/simple_pages.py
================================================
import datetime
from typing import Any
import markdown
from django.core.cache import cache
from django.http import HttpRequest, HttpResponse
from django.shortcuts import get_object_or_404, render
from django.template import Context, Template
from django.utils.http import http_date
from django.utils.timezone import now
from django.views.generic import RedirectView
from concordia.models import Guide, SimplePage, SiteReport
from concordia.parser import paginate_blog_posts
from .decorators import default_cache_control
@default_cache_control
def simple_page(
request: HttpRequest,
path: str | None = None,
slug: str | None = None,
body_ctx: dict[str, Any] | None = None,
template: str = "static-page.html",
) -> HttpResponse:
"""
Renders a simple Markdown-based page stored in the `SimplePage` model.
If no `path` is provided, defaults to the current request path. Markdown is
rendered with optional associated guide content. Breadcrumbs and language
detection are computed from the URL structure.
Request Parameters:
path (str, optional): The database path of the page. Defaults to the
current request path.
slug (str, optional): Unused in current logic; passed for route compatibility.
body_ctx (dict[str, Any], optional): Additional context injected into the page
body during rendering.
template (str): Template used to render the page.
Returns:
HttpResponse: Rendered HTML of the simple page.
"""
if not path:
path = request.path
if body_ctx is None:
body_ctx = {}
page = get_object_or_404(SimplePage, path=path)
md = markdown.Markdown(extensions=["meta"])
breadcrumbs = []
path_components = request.path.strip("/").split("/")
for i, segment in enumerate(path_components[:-1], start=1):
breadcrumbs.append(
("/%s/" % "/".join(path_components[0:i]), segment.replace("-", " ").title())
)
breadcrumbs.append((request.path, page.title))
language_code = "en"
if request.path.replace("/", "").endswith("-esp"):
language_code = "es"
ctx = {
"language_code": language_code,
"title": page.title,
"breadcrumbs": breadcrumbs,
}
guide = page.guide_set.all().first()
if guide is not None:
html = "".join((page.body, guide.body))
ctx["add_navigation"] = True
else:
html = page.body
if "add_navigation" in ctx:
ctx["guides"] = Guide.objects.order_by("order")
body = Template(md.convert(html))
ctx["body"] = body.render(Context(body_ctx))
ctx.update(body_ctx)
resp = render(request, template, ctx)
resp["Created"] = http_date(page.created_on.timestamp())
return resp
@default_cache_control
def about_simple_page(
request: HttpRequest, path: str | None = None, slug: str | None = None
) -> HttpResponse:
"""
Renders the "about" simple page with additional cached campaign and blog stats.
Adds the following keys to the context:
- `report_date` (datetime): Yesterday’s date.
- `campaigns_published` (int): Count from active SiteReport.
- `assets_published` (int): Active + retired total.
- `assets_completed` (int): Active + retired total.
- `assets_waiting_review` (int): Active + retired total.
- `users_activated` (int): From active SiteReport.
- `blog_posts` (Callable): Reference to blog post fetcher.
Returns:
HttpResponse: Rendered HTML of the about page with campaign stats.
"""
context_cache_key = "about_simple_page-about_context"
about_context = cache.get(context_cache_key)
if not about_context:
try:
active_campaigns = SiteReport.objects.filter(
report_name=SiteReport.ReportName.TOTAL
).latest()
except SiteReport.DoesNotExist:
active_campaigns = SiteReport(
campaigns_published=0,
assets_published=0,
assets_completed=0,
assets_waiting_review=0,
users_activated=0,
)
try:
retired_campaigns = SiteReport.objects.filter(
report_name=SiteReport.ReportName.RETIRED_TOTAL
).latest()
except SiteReport.DoesNotExist:
retired_campaigns = SiteReport(
assets_published=0,
assets_completed=0,
assets_waiting_review=0,
)
about_context = {
"report_date": now() - datetime.timedelta(days=1),
"campaigns_published": active_campaigns.campaigns_published,
"assets_published": active_campaigns.assets_published
+ retired_campaigns.assets_published,
"assets_completed": active_campaigns.assets_completed
+ retired_campaigns.assets_completed,
"assets_waiting_review": active_campaigns.assets_waiting_review
+ retired_campaigns.assets_waiting_review,
"users_activated": active_campaigns.users_activated,
"blog_posts": paginate_blog_posts(),
"about_page": True,
}
cache.set(context_cache_key, about_context, 60 * 60)
return simple_page(request, path, slug, about_context)
# These views are to make sure various links to help-center URLs don't break
# when the URLs are changed to not include help-center and can be removed after
# all links are updated.
class HelpCenterRedirectView(RedirectView):
def get_redirect_url(self, *args, **kwargs):
path = kwargs["page_slug"]
return "/get-started/" + path + "/"
class HelpCenterSpanishRedirectView(RedirectView):
def get_redirect_url(self, *args, **kwargs):
path = kwargs["page_slug"]
return "/get-started-esp/" + path + "-esp/"
# End of help-center views
================================================
FILE: concordia/views/topics.py
================================================
from typing import Any
from urllib.parse import urlencode
from django.db.models import Count, F, FilteredRelation, Q
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_page
from concordia.api_views import APIDetailView
from concordia.models import Asset, Topic, TranscriptionStatus
from .decorators import default_cache_control
from .utils import annotate_children_with_progress_stats, calculate_asset_stats
@method_decorator(default_cache_control, name="dispatch")
@method_decorator(cache_page(60 * 60, cache="view_cache"), name="dispatch")
class TopicDetailView(APIDetailView):
"""
Display a topic and its projects with aggregated progress stats.
Renders the topic detail page with a list of published projects tied to
the topic, annotated with per-status asset counts. Supports an optional
transcription-status filter which narrows projects to those containing
assets in that status and respects per-topic URL filter overrides.
Attributes:
template_name (str): Template used for topic detail.
context_object_name (str): Context key for the main object (`topic`).
queryset (QuerySet[Topic]): Base queryset for lookup and ordering.
"""
template_name = "transcriptions/topic_detail.html"
context_object_name = "topic"
queryset = Topic.objects.published().order_by("title")
def get_context_data(self, **kwargs: Any) -> dict[str, object]:
"""
Build context for the topic detail template.
Computes project-level progress annotations and applies an optional
status filter. Also computes topic-wide asset statistics and prepares
sublevel querystring parameters for downstream pages.
Request Parameters:
- `transcription_status` (str, optional): When present and valid,
filters projects to those that:
* have at least one asset in the given status, and
* either have no `pt__url_filter` or have one matching the
requested status.
Context Format:
- `topic` (Topic): The topic being viewed.
- `projects` (QuerySet): Topic projects with:
* per-status counts (e.g., `submitted_count`), and
* `topic_ordering` and `topic_url_filter` from the through
relation.
- `filters` (dict[str, str]): Applied filter parameters.
- `sublevel_querystring` (str): URL-encoded `filters` for links.
- `transcription_status` (str | None): Reflected status filter.
- Aggregated asset stats for the topic (added by
`calculate_asset_stats`), including keys such as:
* `total_assets`, `completed_assets`, `in_progress_assets`,
`not_started_assets`, `submitted_assets`.
Args:
**kwargs: Additional context arguments passed by the base class.
Returns:
dict[str, object]: Context for rendering the topic detail page.
"""
ctx = super().get_context_data(**kwargs)
topic = ctx["topic"]
status = self.request.GET.get("transcription_status")
status_valid = status in TranscriptionStatus.CHOICE_MAP
projects = (
topic.project_set.published().annotate(
**{
f"{key}_count": Count(
"item__asset",
filter=Q(
item__published=True,
item__asset__published=True,
item__asset__transcription_status=key,
),
)
for key in TranscriptionStatus.CHOICE_MAP.keys()
}
)
# Pin the through relation to THIS topic, otherwise it will annotate for
# each ProjectTopic the project is part of
.annotate(
pt=FilteredRelation(
"projecttopic", condition=Q(projecttopic__topic=topic)
)
)
# Pull fields from the pinned alias
.annotate(
topic_ordering=F("pt__ordering"),
topic_url_filter=F("pt__url_filter"),
)
)
# If there's a status filter, we want to exclude any projects
# don't don't have assets in that status, as well as any
# that have a URL filter that's different than the status filter
if status_valid:
ctx["transcription_status"] = status
projects = projects.filter(
Q(pt__url_filter__isnull=True)
| Q(pt__url_filter="")
| Q(pt__url_filter=status)
).exclude(**{f"{status}_count": 0})
projects = projects.order_by("topic_ordering", "campaign__title", "title")
ctx["filters"] = filters = {}
if status_valid:
# We only want to pass specific QS parameters to lower-level search pages:
filters["transcription_status"] = status
ctx["sublevel_querystring"] = urlencode(filters)
annotate_children_with_progress_stats(projects)
ctx["projects"] = projects
topic_assets = Asset.objects.filter(
item__project__topics=self.object,
item__project__published=True,
item__published=True,
published=True,
)
calculate_asset_stats(topic_assets, ctx)
return ctx
def serialize_context(self, context: dict[str, object]) -> dict[str, object]:
"""
Serialize context for API consumers.
Extends the base serializer with a `related_links` list derived from the
topic's associated helpful links.
Args:
context (dict[str, object]): Fully built template context.
Returns:
dict[str, object]: JSON-serializable payload that includes the base
fields from `APIDetailView.serialize_context` and:
- `object.related_links` (list[dict]): Each with:
* `title` (str)
* `url` (str)
"""
ctx = super().serialize_context(context)
ctx["object"]["related_links"] = [
{"title": title, "url": url}
for title, url, sequence in self.object.helpfullink_set.values_list(
"title", "link_url"
)
]
return ctx
================================================
FILE: concordia/views/utils.py
================================================
import datetime
from collections.abc import Iterable
from time import time
from django.conf import settings
from django.db.models import Count, Max, Q, QuerySet
from django.db.models.functions import Greatest
from django.http import HttpRequest
from django.utils import timezone
from django.utils.timezone import now
from concordia.models import Asset, Transcription, TranscriptionStatus
def _get_pages(request: HttpRequest) -> QuerySet:
"""
Retrieve a filtered and annotated queryset of assets based on user activity.
Filters the Asset queryset by:
- Activity type (transcribed or reviewed)
- Transcription status
- Date range (start, end or both)
- Campaign ID
- Last six months of activity
Assets are annotated with:
- Timestamps of last transcription/review activity
- Combined latest activity timestamp
Also applies ordering based on the selected sort parameter.
Args:
request (HttpRequest): The incoming HTTP request with query parameters.
Returns:
QuerySet: A queryset of `Asset` objects with applied filters and annotations.
"""
user = request.user
activity = request.GET.get("activity", None)
if activity == "transcribed":
q = Q(transcription__user=user)
elif activity == "reviewed":
q = Q(transcription__reviewed_by=user)
else:
q = Q(transcription__user=user) | Q(transcription__reviewed_by=user)
assets = Asset.objects.filter(q)
status_list = request.GET.getlist("status")
if status_list and status_list != []:
if "completed" not in status_list:
assets = assets.exclude(transcription_status=TranscriptionStatus.COMPLETED)
if "submitted" not in status_list:
assets = assets.exclude(transcription_status=TranscriptionStatus.SUBMITTED)
if "in_progress" not in status_list:
assets = assets.exclude(
transcription_status=TranscriptionStatus.IN_PROGRESS
)
assets = assets.select_related("item", "item__project", "item__project__campaign")
assets = assets.annotate(
last_transcribed=Max(
"transcription__created_on",
filter=Q(transcription__user=user),
),
last_reviewed=Max(
"transcription__updated_on",
filter=Q(transcription__reviewed_by=user),
),
latest_activity=Greatest(
"last_transcribed",
"last_reviewed",
filter=Q(transcription__user=user) | Q(transcription__reviewed_by=user),
),
)
fmt = "%Y-%m-%d"
start_date = None
start = request.GET.get("start", None)
if start is not None and len(start) > 0:
start_date = timezone.make_aware(datetime.datetime.strptime(start, fmt))
end_date = None
end = request.GET.get("end", None)
if end is not None and len(end) > 0:
end_date = timezone.make_aware(datetime.datetime.strptime(end, fmt))
if start_date is not None and end_date is not None:
end_date += datetime.timedelta(days=1)
end = end_date.strftime(fmt)
assets = assets.filter(latest_activity__range=[start, end])
elif start_date is not None or end_date is not None:
date = start_date if start_date else end_date
assets = assets.filter(
latest_activity__year=date.year,
latest_activity__month=date.month,
latest_activity__day=date.day,
)
# CONCD-189 only show pages from the last 6 months
# This should be an aware datetime, not a date. A date is cast
# to a naive datetime when it's compared to a datetime
# field, as is being done here
SIX_MONTHS_AGO = now() - datetime.timedelta(days=6 * 30)
assets = assets.filter(latest_activity__gte=SIX_MONTHS_AGO)
order_by = request.GET.get("order_by", "date-descending")
if order_by == "date-ascending":
assets = assets.order_by("latest_activity", "-id")
else:
assets = assets.order_by("-latest_activity", "-id")
campaign_id = request.GET.get("campaign", None)
if campaign_id is not None:
assets = assets.filter(item__project__campaign__pk=campaign_id)
return assets
def calculate_asset_stats(asset_qs: QuerySet, ctx: dict) -> None:
"""
Annotates the context dictionary with asset statistics and contributor data.
Computes:
- Total number of unique contributors across all transcriptions.
- Count and percentage of assets per transcription status.
- Labeled status counts for use in progress displays.
Percentages are capped at 99% for values between 99.0 and 99.999... to avoid
showing 100% prematurely.
Args:
asset_qs (QuerySet): A queryset of `Asset` objects to calculate statistics on.
ctx (dict): The context dictionary to populate with computed values.
Returns:
None
"""
asset_count = asset_qs.count()
trans_qs = Transcription.objects.filter(asset__in=asset_qs).values_list(
"user_id", "reviewed_by"
)
user_ids = set()
for i, j in trans_qs.iterator():
user_ids.add(i)
user_ids.add(j)
# Remove null values from the set, if it exists
try:
user_ids.remove(None)
except KeyError:
pass
ctx["contributor_count"] = len(user_ids)
asset_state_qs = asset_qs.values_list("transcription_status")
asset_state_qs = asset_state_qs.annotate(Count("transcription_status")).order_by()
status_counts_by_key = dict(asset_state_qs)
ctx["transcription_status_counts"] = labeled_status_counts = []
for status_key, status_label in TranscriptionStatus.CHOICES:
value = status_counts_by_key.get(status_key, 0)
if value:
pct_raw = 100 * (value / asset_count)
if pct_raw >= 99 and pct_raw < 100:
pct = 99
else:
pct = round(pct_raw)
else:
pct = 0
ctx[f"{status_key}_percent"] = pct
ctx[f"{status_key}_count"] = value
labeled_status_counts.append((status_key, status_label, value))
def annotate_children_with_progress_stats(children: Iterable) -> None:
"""
Annotates child objects with transcription progress statistics.
Each object is expected to have attributes named `{status}_count` corresponding to
each transcription status key. This function calculates:
- `total_count`: Total asset count for the object.
- `{status}_percent`: Percentage of total for each transcription status.
- `lowest_transcription_status`: The first non-zero status in defined order.
Percentages are capped at 99% for values between 99.0 and 99.999... to avoid
rounding up to 100% prematurely.
Args:
children (Iterable): A sequence of objects with `{status}_count` attributes.
Returns:
None
"""
for obj in children:
counts = {}
for k, __ in TranscriptionStatus.CHOICES:
counts[k] = getattr(obj, f"{k}_count", 0)
obj.total_count = total = sum(counts.values())
lowest_status = None
for k, __ in TranscriptionStatus.CHOICES:
count = counts[k]
if total > 0:
pct_raw = 100 * (count / total)
if pct_raw >= 99 and pct_raw < 100:
pct = 99
else:
pct = round(pct_raw)
else:
pct = 0
setattr(obj, f"{k}_percent", pct)
if lowest_status is None and count > 0:
lowest_status = k
obj.lowest_transcription_status = lowest_status
class AnonymousUserValidationCheckMixin:
"""
Mixin that injects anonymous user validation context into class-based views.
Adds a boolean `anonymous_user_validation_required` to the context, indicating
whether a Turnstile validation prompt should be displayed based on the time since
the user's last successful validation.
Intended for use with views that already implement `get_context_data()`, such as
Django's TemplateView or DetailView subclasses.
"""
def get_context_data(self, *args, **kwargs) -> dict:
"""
Add anonymous user validation flag to the context.
If the user is unauthenticated and the time since their last validation exceeds
the configured interval, the flag is set to True. Otherwise, it is set to False.
Returns:
dict: The updated template context with the validation flag included.
"""
context = super().get_context_data(**kwargs)
if not self.request.user.is_authenticated:
turnstile_last_validated = self.request.session.get(
"turnstile_last_validated", 0
)
age = time() - turnstile_last_validated
context["anonymous_user_validation_required"] = (
age > settings.ANONYMOUS_USER_VALIDATION_INTERVAL
)
else:
context["anonymous_user_validation_required"] = False
return context
================================================
FILE: concordia/views/visualizations.py
================================================
from django.core.cache import caches
from django.http import JsonResponse
from django.utils.decorators import method_decorator
from django.views import View
from django.views.decorators.cache import never_cache
@method_decorator(never_cache, name="dispatch")
class VisualizationDataView(View):
"""
Serve cached visualization data as JSON, returning a 404 JSON error if missing.
A single endpoint that, given a `name` slug in the URL, looks up exactly
that key in the 'visualization_cache' and returns its contents as JSON.
If no entry exists under that key, responds with a 404 and a JSON error message.
Attributes:
cache (BaseCache): The Django cache used to retrieve data.
URL Parameters:
name (str): The slug identifying which visualization data to return.
Example: "daily-transcription-activity-by-campaign".
Returns:
JsonResponse:
- On success: the cached data (any JSON-serializable structure).
- On failure: a JSON object {"error": "..."} with HTTP status 404.
"""
cache = caches["visualization_cache"]
def get(self, request, name):
data = self.cache.get(name)
if data is None:
return JsonResponse(
{"error": f"No visualization data found for '{name}'"}, status=404
)
return JsonResponse(data)
================================================
FILE: concordia/widgets.py
================================================
from django import forms
class EmailWidget(forms.EmailInput):
template_name = "forms/widgets/email.html"
================================================
FILE: concordia/wsgi.py
================================================
"""
WSGI config for concordia project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/
"""
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
================================================
FILE: configuration/__init__.py
================================================
================================================
FILE: configuration/admin.py
================================================
from typing import Any
from django.contrib import admin, messages
from django.http import HttpRequest, HttpResponse
from django.template.response import TemplateResponse
from django.utils.html import format_html
from configuration.models import Configuration
@admin.register(Configuration)
class ConfigurationAdmin(admin.ModelAdmin):
"""
Admin configuration for the `Configuration` model.
Behavior:
- Displays the key, raw value, and description in the changelist.
- Provides a read-only `validated_value` field on the change form that
shows the interpreted value as returned by `Configuration.get_value`.
- Overrides `changeform_view` to add a two-step confirmation flow when
saving changes, including a preview of the parsed value. Also handles
an explicit cancel action by rebuilding the normal change form
context rather than delegating to the base implementation.
"""
list_display = ("key", "value", "description")
readonly_fields = ("validated_value",)
def validated_value(self, obj: Configuration) -> str:
"""
Render the parsed configuration value and explanatory text.
Notes:
This method does not alter the base `ModelAdmin` behavior. It is a
helper used by the change form to display both the interpreted
value from `Configuration.get_value()` and a short explanation
that this parsed value is what application code will consume.
Args:
obj (Configuration): The instance being edited.
Returns:
str: HTML-safe string produced by `format_html` containing the
parsed value and explanatory note.
"""
return format_html(
"{}
{}
",
obj.get_value(),
"This is the interpreted value based on the selected data type. "
"This value is what will be seen by the code that uses this "
"configuration.",
)
def changeform_view(
self,
request: HttpRequest,
object_id: str | None = None,
form_url: str = "",
extra_context: dict[str, Any] | None = None,
) -> HttpResponse:
"""
Override the base change form view to add a confirmation step.
Differences from the base implementation:
- On initial POST, validate the form and, if valid, render a
confirmation template that previews the parsed value produced by
`Configuration.get_value()`.
- On confirmation POST (`_confirm_update`), save the instance and
show a success message.
- On cancel POST (`cancel_update`), rebuild the standard change
form context manually and re-render the change form instead of
delegating to the base method (which would otherwise proceed with
the change because it is a POST).
- For all other flows, fall back to the base implementation.
Args:
request (HttpRequest): The current request.
object_id (str | None): Primary key of the object being edited.
form_url (str): Form action URL.
extra_context (dict[str, Any] | None): Extra template context.
Returns:
HttpResponse: Either the confirmation screen, the re-rendered
change form, or the default response from the base view.
"""
obj = self.get_object(request, object_id)
if request.method == "POST":
if "_confirm_update" in request.POST:
# Second POST: confirmation of update
form = self.get_form(request, obj)(request.POST, instance=obj)
if form.is_valid():
form.save()
self.message_user(request, "Configuration updated and cached.")
return self.response_post_save_change(request, form.instance)
else:
self.message_user(
request, "Invalid data on confirmation.", level=messages.ERROR
)
elif "cancel_update" in request.POST:
form = self.get_form(request, obj)(request.POST, instance=obj)
admin_form = admin.helpers.AdminForm(
form,
list(self.get_fieldsets(request, obj)),
self.get_prepopulated_fields(request, obj),
self.get_readonly_fields(request, obj),
model_admin=self,
)
# We unfortunately have to manually construct this context, since using
# super causes it to just perform the cancelled change, because this is
# a POST request
context = {
**self.admin_site.each_context(request),
"title": f"Edit Configuration: {obj.key}",
"adminform": admin_form,
"inline_admin_formsets": [],
"media": self.media + form.media,
"object_id": object_id,
"original": obj,
"opts": self.model._meta,
"add": False,
"change": True,
"is_popup": False,
"save_as": self.save_as,
"has_view_permission": self.has_view_permission(request, obj),
"has_add_permission": self.has_add_permission(request),
"has_change_permission": self.has_change_permission(request, obj),
"has_delete_permission": self.has_delete_permission(request, obj),
"form_url": form_url,
"to_field": None,
"has_editable_inline_admin_formsets": False,
}
return self.render_change_form(
request,
context=context,
add=False,
change=True,
form_url=form_url,
obj=obj,
)
else:
# First POST: validate and show confirmation screen
form = self.get_form(request, obj)(request.POST, instance=obj)
if form.is_valid():
new_instance = form.save(commit=False)
try:
parsed_value = new_instance.get_value()
except Exception as e:
self.message_user(
request, f"Validation failed: {e}", level=messages.ERROR
)
return super().changeform_view(
request, object_id, form_url, extra_context=extra_context
)
context = {
"title": (
f"Confirm Update of Configuration '{new_instance.key}'"
),
"original": self.model._default_manager.get(pk=obj.pk),
"new_instance": new_instance,
"parsed_value": parsed_value,
"opts": self.model._meta,
"object_id": object_id,
"form_url": form_url,
"request": request,
}
return TemplateResponse(
request, "admin/configuration_confirm_update.html", context
)
return super().changeform_view(request, object_id, form_url, extra_context)
================================================
FILE: configuration/apps.py
================================================
from django.apps import AppConfig
class ConfigurationConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "configuration"
def ready(self):
import configuration.signals # NOQA
================================================
FILE: configuration/management/__init__.py
================================================
================================================
FILE: configuration/management/commands/__init__.py
================================================
================================================
FILE: configuration/management/commands/configcache.py
================================================
from django.core.cache import caches
from django.core.management.base import BaseCommand
class Command(BaseCommand):
help = "Fetch a value from the configuration cache by key." # NOQA: A003
def add_arguments(self, parser):
parser.add_argument("key", type=str, help="The cache key to retrieve")
def handle(self, *args, **options):
config_cache = caches["configuration_cache"]
key = options["key"]
cache_key = f"config_{key}"
value = config_cache.get(cache_key)
if value is None:
self.stdout.write(self.style.WARNING(f"Key '{key}' not found in cache."))
else:
self.stdout.write(self.style.SUCCESS(f"Key '{key}' found:"))
self.stdout.write(str(value))
================================================
FILE: configuration/migrations/0001_initial.py
================================================
# Generated by Django 4.2.16 on 2025-02-25 19:21
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Configuration",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"key",
models.CharField(
help_text="Unique identifier for the configuration setting",
max_length=255,
unique=True,
),
),
(
"value",
models.TextField(help_text="Value of the configuration setting"),
),
(
"data_type",
models.CharField(
choices=[
("text", "Plain text"),
("number", "Number"),
("boolean", "Boolean"),
("json", "JSON"),
("html", "HTML"),
],
default="text",
help_text="Data type of the value",
max_length=10,
),
),
(
"description",
models.TextField(
blank=True,
help_text="Optional description of the configuration setting",
),
),
],
),
]
================================================
FILE: configuration/migrations/0002_populate_configurations.py
================================================
# Generated by Django 4.2.16 on 2025-02-25 20:15
from django.db import migrations
def populate_configuration(apps, schema_editor):
Configuration = apps.get_model("configuration", "Configuration")
# List of initial configuration entries
initial_data = [
{
"key": "review_rate_limit_popup_message",
"data_type": "html",
"value": "Volunteers can only accept {% configuration_value 'review_rate_limit' %} pages per minute.
\r\nPlease read all transcriptions completely to ensure they are whole and accurate. See review instructions.
",
"description": "Message shown in the pop-up when a user exceeds the review rate limit",
},
{
"key": "review_rate_limit_popup_title",
"data_type": "html",
"value": "You cannot yet accept this page",
"description": "Title of the error pop-up displayed when a user exceeds the review rate limit",
},
{
"key": "review_rate_limit",
"data_type": "number",
"value": "4",
"description": "Number of reviews allowed per minute",
},
{
"key": "review_rate_limit_banner_message",
"data_type": "html",
"value": "You cannot yet accept this page. Volunteers can only accept {% configuration_value 'review_rate_limit' %} pages per minute. See review instructions .",
"description": "Message to display on the banner when a user exceeds the review rate limit",
},
]
# Insert data into the database
for entry in initial_data:
Configuration.objects.update_or_create(key=entry["key"], defaults=entry)
def revert_populate_configuration(apps, schema_editor):
# We can't actually revert the data to the state it was before,
# and there's no actual need to, but we need this function to be
# able to reverse this migration
pass
class Migration(migrations.Migration):
dependencies = [
("configuration", "0001_initial"),
]
operations = [
migrations.RunPython(populate_configuration, revert_populate_configuration),
]
================================================
FILE: configuration/migrations/0003_populate_retry_configurations.py
================================================
# Generated by Django 4.2.16 on 2025-03-03 20:51
from django.db import migrations
def populate_retry_configuration(apps, schema_editor):
Configuration = apps.get_model("configuration", "Configuration")
# List of initial retry configuration entries
initial_data = [
{
"key": "asset_image_import_max_retries",
"data_type": "number",
"value": "3",
"description": "The maximum number of times to retry downloading "
"an asset image during import",
},
{
"key": "asset_image_import_max_retry_delay",
"data_type": "number",
"value": "10",
"description": "The number of minutes to wait before retrying downloading "
"an image during import. A non-positive number will disable retries. Should "
"not be greater than around 45 to avoid the system duplicating the task.",
},
]
# Insert data into the database
for entry in initial_data:
Configuration.objects.update_or_create(key=entry["key"], defaults=entry)
def revert_populate_retry_configuration(apps, schema_editor):
# We can't actually revert the data to the state it was before,
# and there's no actual need to, but we need this function to be
# able to reverse this migration
pass
class Migration(migrations.Migration):
dependencies = [
("configuration", "0002_populate_configurations"),
]
operations = [
migrations.RunPython(
populate_retry_configuration, revert_populate_retry_configuration
),
]
================================================
FILE: configuration/migrations/0004_alter_configuration_options.py
================================================
# Generated by Django 4.2.16 on 2025-03-18 20:01
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("configuration", "0003_populate_retry_configurations"),
]
operations = [
migrations.AlterModelOptions(
name="configuration",
options={"ordering": ["key"]},
),
]
================================================
FILE: configuration/migrations/0005_alter_configuration_data_type.py
================================================
# Generated by Django 4.2.22 on 2025-07-29 17:28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("configuration", "0004_alter_configuration_options"),
]
operations = [
migrations.AlterField(
model_name="configuration",
name="data_type",
field=models.CharField(
choices=[
("text", "Plain text"),
("number", "Number"),
("boolean", "Boolean"),
("json", "JSON"),
("html", "HTML"),
("rate", "Rate"),
],
default="text",
help_text="Data type of the value",
max_length=10,
),
),
]
================================================
FILE: configuration/migrations/0006_populate_next_asset_rate_limit.py
================================================
from django.db import migrations
def populate_configuration(apps, schema_editor):
Configuration = apps.get_model("configuration", "Configuration")
initial_data = [
{
"key": "next_asset_rate_limit",
"data_type": "rate",
"value": "4/m",
"description": "Rate limit of anonymous users for the next_*_asset views. Format is 'X/u', where 'X' is the number of requests and 'u' is 's', 'm', 'h' or'd' (second, minute, hour or day). '5/s' means 'five per second'.",
},
]
for entry in initial_data:
Configuration.objects.update_or_create(key=entry["key"], defaults=entry)
def revert_populate_configuration(apps, schema_editor):
# We can't actually revert the data to the state it was before,
# and there's no actual need to, but we need this function to be
# able to reverse this migration
pass
class Migration(migrations.Migration):
dependencies = [
("configuration", "0005_alter_configuration_data_type"),
]
operations = [
migrations.RunPython(populate_configuration, revert_populate_configuration),
]
================================================
FILE: configuration/migrations/__init__.py
================================================
================================================
FILE: configuration/models.py
================================================
import json
from typing import Any
from django import template
from django.core.exceptions import ValidationError
from django.db import models
from configuration.validation import validate_rate
class Configuration(models.Model):
"""
Key/value configuration model with typed decoding.
Purpose:
Store site configuration as string values and expose a helper that
converts the stored text into a concrete Python type based on
`data_type`.
Fields:
key (models.CharField): Unique identifier for the setting.
data_type (models.CharField): One of `DataType` choices indicating how
`value` should be interpreted.
value (models.TextField): Raw text representation of the value.
description (models.TextField): Optional human-readable description.
Meta:
ordering: Sorted by `key`.
"""
class DataType(models.TextChoices):
"""
Supported data types for decoding `value` in `get_value`.
"""
TEXT = "text", "Plain text"
NUMBER = "number", "Number"
BOOLEAN = "boolean", "Boolean"
JSON = "json", "JSON"
HTML = "html", "HTML"
RATE = "rate", "Rate"
key = models.CharField(
max_length=255,
unique=True,
help_text="Unique identifier for the configuration setting",
)
data_type = models.CharField(
max_length=10,
choices=DataType.choices,
default=DataType.TEXT,
help_text="Data type of the value",
)
value = models.TextField(help_text="Value of the configuration setting")
description = models.TextField(
blank=True, help_text="Optional description of the configuration setting"
)
class Meta:
ordering = ["key"]
def __str__(self) -> str:
"""
Return the configuration key for display.
"""
return self.key
def get_value(self) -> "Any":
"""
Decode and return `value` according to `data_type`.
Behavior:
- `NUMBER`: Try `int(value)`, else try `float(value)`, else return 0.
- `BOOLEAN`: Return True if `value.lower() == "true"`, else False.
- `JSON`: Parse with `json.loads(value)` and return the result.
- `HTML`: Render `value` through Django's template engine with an
empty context and return the rendered string.
- `RATE`: Validate using `validate_rate(value)`. If validation
fails, return an empty string. Otherwise return the validated
value as provided by `validate_rate`.
- `TEXT` or any unrecognized type: Return `value` unchanged.
Returns:
Any: Decoded value. The concrete type depends on `data_type` and
may be `int`, `float`, `bool`, `str`, `dict`, `list`, or a value
returned by `validate_rate`.
Raises:
json.JSONDecodeError: If `data_type` is `JSON` and `value` is not
valid JSON.
"""
if self.data_type == Configuration.DataType.NUMBER:
try:
return int(self.value)
except ValueError:
try:
return float(self.value)
except ValueError:
return 0
elif self.data_type == Configuration.DataType.BOOLEAN:
if self.value.lower() == "true":
return True
else:
return False
elif self.data_type == Configuration.DataType.JSON:
return json.loads(self.value)
elif self.data_type == Configuration.DataType.HTML:
value = template.Template(self.value)
return value.render(template.Context({}))
elif self.data_type == Configuration.DataType.RATE:
try:
return validate_rate(self.value)
except ValidationError:
return ""
else:
# DataType.TEXT or an unknown type,
# so just return the value itself
return self.value
================================================
FILE: configuration/signals.py
================================================
from django.db.models.signals import post_save
from django.dispatch import receiver
from configuration.models import Configuration
from configuration.utils import cache_configuration_value
@receiver(post_save, sender=Configuration)
def update_cached_configuration_value(
sender: type[Configuration], *, instance: Configuration, **kwargs
) -> None:
"""
Post-save signal handler that updates the cached configuration value.
Behavior:
- Parse the instance value using `Configuration.get_value()`.
- If parsing succeeds, write the parsed value to the cache via
`cache_configuration_value`.
- If parsing raises any exception, skip caching to avoid persisting an
invalid value.
Signals:
Connected to `django.db.models.signals.post_save` for
`configuration.models.Configuration`.
Args:
sender (type[Configuration]): The model class that sent the signal.
instance (Configuration): The saved instance whose parsed value should
be cached.
Returns:
None
"""
try:
value = instance.get_value()
except Exception:
# Do not cache if value is invalid
return
cache_configuration_value(instance.key, value)
================================================
FILE: configuration/templates/admin/configuration_confirm_update.html
================================================
{% extends "admin/base_site.html" %}
{% load i18n %}
{% block content %}
{% endblock %}
================================================
FILE: configuration/templatetags/__init__.py
================================================
================================================
FILE: configuration/templatetags/configuration_tags.py
================================================
from typing import Any
from django import template
from configuration.models import Configuration
from configuration.utils import configuration_value as _configuration_value
register = template.Library()
@register.simple_tag
def configuration_value(key: str) -> Any:
"""
Return the parsed configuration value for a key, for use in templates.
Behavior:
Delegates to `configuration.utils.configuration_value` to fetch and parse
the value (including any casting based on the configured data type).
If the configuration is missing or parsing raises an exception of any
kind, return an empty string to keep template rendering resilient.
Args:
key (str): The unique configuration key.
Returns:
Any: The parsed value when available and valid; otherwise an empty string.
"""
try:
return _configuration_value(key)
except (Configuration.DoesNotExist, Exception):
# Return an empty string if the key does not exist or parsing fails
return ""
================================================
FILE: configuration/tests/__init__.py
================================================
================================================
FILE: configuration/tests/test_admin.py
================================================
from django.contrib.auth import get_user_model
from django.test import TestCase
from django.urls import reverse
from configuration.models import Configuration
class TestConfigurationAdmin(TestCase):
def setUp(self):
self.superuser = get_user_model().objects.create_superuser(
username="admin",
email="admin@example.com",
password="adminpass", # nosec
)
self.client.force_login(self.superuser)
self.config = Configuration.objects.create(
key="test-key",
value="Initial value",
data_type=Configuration.DataType.TEXT,
description="Initial description",
)
self.url = reverse(
"admin:configuration_configuration_change", args=[self.config.pk]
)
def test_change_view_initial_get(self):
response = self.client.get(self.url)
self.assertEqual(response.status_code, 200)
self.assertContains(response, "test-key")
self.assertContains(response, "Initial value")
def test_save_triggers_confirmation(self):
response = self.client.post(
self.url,
{
"key": self.config.key,
"value": "Updated value",
"data_type": Configuration.DataType.TEXT,
"description": "Updated description",
},
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(response, "admin/configuration_confirm_update.html")
self.assertContains(response, "Confirm Update of Configuration")
def test_confirm_save_updates_object(self):
# Step 1: post to trigger confirmation
confirmation_response = self.client.post(
self.url,
{
"key": self.config.key,
"value": "Updated value",
"data_type": Configuration.DataType.TEXT,
"description": "Updated description",
},
)
self.assertEqual(confirmation_response.status_code, 200)
self.assertTemplateUsed(
confirmation_response, "admin/configuration_confirm_update.html"
)
# Step 2: confirm the update
confirm_response = self.client.post(
self.url,
{
"_confirm_update": "1",
"key": self.config.key,
"value": "Updated value",
"data_type": Configuration.DataType.TEXT,
"description": "Updated description",
},
follow=True,
)
changelist_url = reverse("admin:configuration_configuration_changelist")
self.assertRedirects(confirm_response, changelist_url)
self.assertContains(confirm_response, "Configuration updated and cached.")
self.config.refresh_from_db()
self.assertEqual(self.config.value, "Updated value")
self.assertEqual(self.config.description, "Updated description")
def test_cancel_preserves_input(self):
# Step 1: post to trigger confirmation
confirmation_response = self.client.post(
self.url,
{
"key": self.config.key,
"value": "New value",
"data_type": Configuration.DataType.TEXT,
"description": "Changed description",
},
)
self.assertEqual(confirmation_response.status_code, 200)
# Step 2: simulate "Cancel" by posting with cancel_update
cancel_response = self.client.post(
self.url,
{
"cancel_update": "1",
"key": self.config.key,
"value": "New value",
"data_type": Configuration.DataType.TEXT,
"description": "Changed description",
},
)
self.assertEqual(cancel_response.status_code, 200)
self.assertContains(cancel_response, "Changed description")
self.assertContains(cancel_response, "New value")
self.config.refresh_from_db()
self.assertEqual(self.config.value, "Initial value")
self.assertEqual(self.config.description, "Initial description")
def test_confirm_save_with_invalid_form(self):
# Step 1: trigger confirmation with valid initial post
self.client.post(
self.url,
{
"key": self.config.key,
"value": "value",
"data_type": Configuration.DataType.TEXT,
"description": "desc",
},
)
# Step 2: confirm with missing required field (invalid POST)
response = self.client.post(
self.url,
{
"_confirm_update": "1",
# Omit 'key' which is required
"value": "value",
"data_type": Configuration.DataType.TEXT,
"description": "desc",
},
follow=True,
)
self.assertEqual(response.status_code, 200)
self.assertContains(response, "Invalid data on confirmation.")
self.config.refresh_from_db()
self.assertEqual(self.config.value, "Initial value") # unchanged
def test_get_value_failure_on_confirmation(self):
# Create a config with data_type=JSON and invalid JSON
config = Configuration.objects.create(
key="bad-json-key",
value="Not JSON",
data_type=Configuration.DataType.JSON,
description="desc",
)
url = reverse("admin:configuration_configuration_change", args=[config.pk])
# Initial POST with invalid JSON triggers get_value failure
response = self.client.post(
url,
{
"key": config.key,
"value": "Still not JSON",
"data_type": Configuration.DataType.JSON,
"description": "desc",
},
follow=True,
)
self.assertEqual(response.status_code, 200)
self.assertContains(response, "Validation failed:")
def test_first_post_invalid_form(self):
response = self.client.post(
self.url,
{
"key": "", # key is required, so this makes the form invalid
"value": "Some value",
"data_type": Configuration.DataType.TEXT,
"description": "Bad post",
},
)
self.assertEqual(response.status_code, 200)
self.assertTemplateUsed(response, "admin/change_form.html")
self.assertContains(response, "This field is required.")
================================================
FILE: configuration/tests/test_models.py
================================================
import json
from django.core.cache import caches
from django.test import TestCase
from configuration.models import Configuration
class TestConfiguration(TestCase):
def setUp(self):
caches["configuration_cache"].clear()
def test_str(self):
config = Configuration.objects.create(
key="test-key", value="Test value", data_type=Configuration.DataType.TEXT
)
self.assertEqual(str(config), "test-key")
def test_text(self):
config = Configuration.objects.create(
key="test-key", value="Test value", data_type=Configuration.DataType.TEXT
)
self.assertEqual(config.get_value(), "Test value")
config2 = Configuration.objects.create(
key="test-key2", value="", data_type=Configuration.DataType.TEXT
)
self.assertEqual(config2.get_value(), "")
config3 = Configuration.objects.create(
key="test-key3",
value='{"key" : "value"}',
data_type=Configuration.DataType.TEXT,
)
self.assertEqual(config3.get_value(), '{"key" : "value"}')
def test_number(self):
config = Configuration.objects.create(
key="test-key", value="100", data_type=Configuration.DataType.NUMBER
)
self.assertEqual(config.get_value(), 100)
config2 = Configuration.objects.create(
key="test-key2", value="100.12", data_type=Configuration.DataType.NUMBER
)
self.assertEqual(config2.get_value(), 100.12)
config3 = Configuration.objects.create(
key="test-key3", value="Test value", data_type=Configuration.DataType.NUMBER
)
self.assertEqual(config3.get_value(), 0)
config4 = Configuration.objects.create(
key="test-key4", value="", data_type=Configuration.DataType.NUMBER
)
self.assertEqual(config4.get_value(), 0)
def test_boolean(self):
config = Configuration.objects.create(
key="test-key", value="True", data_type=Configuration.DataType.BOOLEAN
)
self.assertEqual(config.get_value(), True)
config2 = Configuration.objects.create(
key="test-key2", value="true", data_type=Configuration.DataType.BOOLEAN
)
self.assertEqual(config2.get_value(), True)
config3 = Configuration.objects.create(
key="test-key3", value="TrUe", data_type=Configuration.DataType.BOOLEAN
)
self.assertEqual(config3.get_value(), True)
config4 = Configuration.objects.create(
key="test-key4", value="", data_type=Configuration.DataType.BOOLEAN
)
self.assertEqual(config4.get_value(), False)
config5 = Configuration.objects.create(
key="test-key5", value="1", data_type=Configuration.DataType.BOOLEAN
)
self.assertEqual(config5.get_value(), False)
config6 = Configuration.objects.create(
key="test-key6",
value="Test value",
data_type=Configuration.DataType.BOOLEAN,
)
self.assertEqual(config6.get_value(), False)
def test_json(self):
config = Configuration.objects.create(
key="test-key", value="true", data_type=Configuration.DataType.JSON
)
self.assertEqual(config.get_value(), True)
config2 = Configuration.objects.create(
key="test-key2", value="True", data_type=Configuration.DataType.JSON
)
self.assertRaises(json.decoder.JSONDecodeError, config2.get_value)
config3 = Configuration.objects.create(
key="test-key3",
value='{"key" : "value"}',
data_type=Configuration.DataType.JSON,
)
self.assertEqual(config3.get_value(), {"key": "value"})
config4 = Configuration.objects.create(
key="test-key4", value="", data_type=Configuration.DataType.JSON
)
self.assertRaises(json.decoder.JSONDecodeError, config4.get_value)
config5 = Configuration.objects.create(
key="test-key5", value="1", data_type=Configuration.DataType.JSON
)
self.assertEqual(config5.get_value(), 1)
config6 = Configuration.objects.create(
key="test-key6", value="Test value", data_type=Configuration.DataType.JSON
)
self.assertRaises(json.decoder.JSONDecodeError, config6.get_value)
def test_html(self):
config = Configuration.objects.create(
key="test-key", value="Test value", data_type=Configuration.DataType.HTML
)
self.assertEqual(config.get_value(), "Test value")
config2 = Configuration.objects.create(
key="test-key2", value="", data_type=Configuration.DataType.HTML
)
self.assertEqual(config2.get_value(), "")
config3 = Configuration.objects.create(
key="test-key3",
value='{"key" : "value"}',
data_type=Configuration.DataType.HTML,
)
self.assertEqual(config3.get_value(), '{"key" : "value"}')
config4 = Configuration.objects.create(
key="test-key4",
value="Test value
",
data_type=Configuration.DataType.HTML,
)
self.assertEqual(config4.get_value(), "Test value
")
config5 = Configuration.objects.create(
key="test-key5",
value="{% configuration_value 'test-key' %}
",
data_type=Configuration.DataType.HTML,
)
self.assertEqual(config5.get_value(), "Test value
")
config6 = Configuration.objects.create(
key="test-key6",
value="{% url 'homepage' %}",
data_type=Configuration.DataType.HTML,
)
self.assertEqual(config6.get_value(), "/")
def test_rate(self):
# Valid rates
config1 = Configuration.objects.create(
key="test-key1", value="1/s", data_type=Configuration.DataType.RATE
)
self.assertEqual(config1.get_value(), "1/s")
config2 = Configuration.objects.create(
key="test-key2", value="100/m", data_type=Configuration.DataType.RATE
)
self.assertEqual(config2.get_value(), "100/m")
config3 = Configuration.objects.create(
key="test-key3", value="50/h", data_type=Configuration.DataType.RATE
)
self.assertEqual(config3.get_value(), "50/h")
config4 = Configuration.objects.create(
key="test-key4", value="1000/d", data_type=Configuration.DataType.RATE
)
self.assertEqual(config4.get_value(), "1000/d")
# Invalid formats
config5 = Configuration.objects.create(
key="test-key5", value="5/hour", data_type=Configuration.DataType.RATE
)
self.assertEqual(config5.get_value(), "")
config6 = Configuration.objects.create(
key="test-key6", value="ten/m", data_type=Configuration.DataType.RATE
)
self.assertEqual(config6.get_value(), "")
config7 = Configuration.objects.create(
key="test-key7", value="10", data_type=Configuration.DataType.RATE
)
self.assertEqual(config7.get_value(), "")
config8 = Configuration.objects.create(
key="test-key8", value="10/", data_type=Configuration.DataType.RATE
)
self.assertEqual(config8.get_value(), "")
config9 = Configuration.objects.create(
key="test-key9", value="/m", data_type=Configuration.DataType.RATE
)
self.assertEqual(config9.get_value(), "")
# Zero and negative values
config10 = Configuration.objects.create(
key="test-key10", value="0/s", data_type=Configuration.DataType.RATE
)
self.assertEqual(config10.get_value(), "")
config11 = Configuration.objects.create(
key="test-key11", value="-5/m", data_type=Configuration.DataType.RATE
)
self.assertEqual(config11.get_value(), "")
# Empty value
config12 = Configuration.objects.create(
key="test-key12", value="", data_type=Configuration.DataType.RATE
)
self.assertEqual(config12.get_value(), "")
================================================
FILE: configuration/tests/test_signals.py
================================================
from django.core.cache import caches
from django.test import TestCase
from configuration.models import Configuration
class TestConfigurationSignal(TestCase):
def setUp(self):
caches["configuration_cache"].clear()
def test_signal_caches_valid_value(self):
Configuration.objects.create(
key="signal-key",
value="42",
data_type=Configuration.DataType.NUMBER,
)
self.assertEqual(caches["configuration_cache"].get("config_signal-key"), 42)
def test_signal_does_not_cache_invalid_json(self):
Configuration.objects.create(
key="signal-json-invalid",
value="not valid json",
data_type=Configuration.DataType.JSON,
)
# Should not raise, but value should not be cached
self.assertIsNone(
caches["configuration_cache"].get("config_signal-json-invalid")
)
================================================
FILE: configuration/tests/test_templatetags.py
================================================
from django.test import TestCase
from configuration.models import Configuration
from configuration.templatetags.configuration_tags import configuration_value
class TestTemplatetags(TestCase):
def test_configuration_value(self):
Configuration.objects.create(
key="test-key", value="Test value", data_type=Configuration.DataType.TEXT
)
self.assertEqual(configuration_value("test-key"), "Test value")
Configuration.objects.create(
key="test-key2", value="100", data_type=Configuration.DataType.NUMBER
)
self.assertEqual(configuration_value("test-key2"), 100)
Configuration.objects.create(
key="test-key3", value="TrUe", data_type=Configuration.DataType.BOOLEAN
)
self.assertEqual(configuration_value("test-key3"), True)
Configuration.objects.create(
key="test-key4", value="Test value", data_type=Configuration.DataType.JSON
)
# This raises an exception due to invalid JSON, but the template tag should
# catch and return a blank string instead
self.assertEqual(configuration_value("test-key4"), "")
Configuration.objects.create(
key="test-key5",
value='{"key" : "value"}',
data_type=Configuration.DataType.JSON,
)
self.assertEqual(configuration_value("test-key5"), {"key": "value"})
Configuration.objects.create(
key="test-key6",
value="{% configuration_value 'test-key' %}
",
data_type=Configuration.DataType.HTML,
)
self.assertEqual(configuration_value("test-key6"), "Test value
")
================================================
FILE: configuration/tests/test_utils.py
================================================
import json
from django.core.cache import caches
from django.test import TestCase
from configuration.models import Configuration
from configuration.utils import (
CONFIGURATION_KEY_PREFIX,
cache_configuration_value,
configuration_value,
)
class TestConfigurationUtils(TestCase):
def setUp(self):
self.cache = caches["configuration_cache"]
self.cache.clear()
def test_configuration_value(self):
Configuration.objects.create(
key="test-key", value="Test value", data_type=Configuration.DataType.TEXT
)
self.assertEqual(configuration_value("test-key"), "Test value")
Configuration.objects.create(
key="test-key2", value="100", data_type=Configuration.DataType.NUMBER
)
self.assertEqual(configuration_value("test-key2"), 100)
Configuration.objects.create(
key="test-key3", value="TrUe", data_type=Configuration.DataType.BOOLEAN
)
self.assertEqual(configuration_value("test-key3"), True)
Configuration.objects.create(
key="test-key4", value="", data_type=Configuration.DataType.JSON
)
self.assertRaises(
json.decoder.JSONDecodeError, configuration_value, "test-key4"
)
Configuration.objects.create(
key="test-key5",
value='{"key" : "value"}',
data_type=Configuration.DataType.JSON,
)
self.assertEqual(configuration_value("test-key5"), {"key": "value"})
Configuration.objects.create(
key="test-key6",
value="{% configuration_value 'test-key' %}
",
data_type=Configuration.DataType.HTML,
)
self.assertEqual(configuration_value("test-key6"), "Test value
")
class TestCacheConfigurationValue(TestCase):
def setUp(self):
self.cache = caches["configuration_cache"]
self.cache.clear()
def test_explicit_value_is_cached(self):
key = "explicit-key"
cached = cache_configuration_value(key, 123)
self.assertEqual(cached, 123)
self.assertEqual(
self.cache.get(f"{CONFIGURATION_KEY_PREFIX}_{key}"),
123,
)
def test_value_is_fetched_from_model_if_not_supplied(self):
Configuration.objects.create(
key="fetched-key", value="true", data_type=Configuration.DataType.BOOLEAN
)
cached = cache_configuration_value("fetched-key")
self.assertEqual(cached, True)
self.assertEqual(
self.cache.get(f"{CONFIGURATION_KEY_PREFIX}_fetched-key"),
True,
)
================================================
FILE: configuration/tests/test_validation.py
================================================
from django.core.exceptions import ValidationError
from django.test import TestCase
from configuration.validation import validate_rate
class TestValidation(TestCase):
def test_valid_rates(self):
self.assertEqual(validate_rate("1/s"), "1/s")
self.assertEqual(validate_rate("10/m"), "10/m")
self.assertEqual(validate_rate("100/h"), "100/h")
self.assertEqual(validate_rate("1000/d"), "1000/d")
def test_rate_stripping_whitespace(self):
# Leading/trailing spaces
self.assertEqual(validate_rate(" 10/m "), "10/m")
self.assertEqual(validate_rate("\t10/m"), "10/m")
self.assertEqual(validate_rate("10/m\t"), "10/m")
self.assertEqual(validate_rate("\n10/m\n"), "10/m")
self.assertEqual(validate_rate(" \n\t10/m\t\n "), "10/m")
# Internal whitespace is not allowed and should still raise
with self.assertRaises(ValidationError):
validate_rate("10 /m")
with self.assertRaises(ValidationError):
validate_rate("10/ m")
with self.assertRaises(ValidationError):
validate_rate("10 / m")
def test_non_string_input(self):
with self.assertRaises(ValidationError):
validate_rate(10)
with self.assertRaises(ValidationError):
validate_rate(None)
with self.assertRaises(ValidationError):
validate_rate(["5/m"])
def test_invalid_format(self):
with self.assertRaises(ValidationError):
validate_rate("10") # no unit
with self.assertRaises(ValidationError):
validate_rate("10/min") # full word
with self.assertRaises(ValidationError):
validate_rate("ten/m") # non-numeric
with self.assertRaises(ValidationError):
validate_rate("10/") # missing unit
with self.assertRaises(ValidationError):
validate_rate("/m") # missing number
# This is now valid due to stripping
self.assertEqual(validate_rate("10/m\n"), "10/m")
with self.assertRaises(ValidationError):
validate_rate("10/m/extra") # too many parts
def test_zero_or_negative_values(self):
with self.assertRaises(ValidationError):
validate_rate("0/s")
with self.assertRaises(ValidationError):
validate_rate("-5/m")
def test_invalid_unit(self):
with self.assertRaises(ValidationError):
validate_rate("10/w") # unsupported unit
with self.assertRaises(ValidationError):
validate_rate("10/ms") # unsupported unit
with self.assertRaises(ValidationError):
validate_rate("10/seconds") # full unit
================================================
FILE: configuration/utils.py
================================================
from typing import Any
from django.conf import settings
from django.core.cache import caches
from configuration.models import Configuration
CONFIGURATION_KEY_PREFIX = "config"
def configuration_value(key: str) -> Any:
"""
Retrieve a configuration value by key with caching and type casting.
Behavior:
- Look up the value in the ``configuration_cache`` using a namespaced
cache key.
- If the value is missing, delegate to
``cache_configuration_value(key)`` to fetch, cast, cache, and return
the value.
- Casting is performed by ``Configuration.get_value()`` based on the
instance's ``data_type``.
Caching:
- Values are stored in the cache alias ``configuration_cache``.
- Cache entries expire according to
``settings.CONFIGURATION_CACHE_TIMEOUT``.
Args:
key (str): The configuration key to resolve.
Returns:
Any: The resolved and type-cast configuration value.
Raises:
Configuration.DoesNotExist: If the key is not present in the database
when attempting to populate the cache.
"""
config_cache = caches["configuration_cache"]
cache_key = f"{CONFIGURATION_KEY_PREFIX}_{key}"
value = config_cache.get(cache_key)
if value is None:
value = cache_configuration_value(key)
return value
def cache_configuration_value(key: str, value: Any | None = None) -> Any:
"""
Populate or refresh the cached value for a configuration key.
Behavior:
- If ``value`` is ``None``, fetch the ``Configuration`` by ``key``,
cast it via ``get_value()``, and cache the result.
- If ``value`` is provided, cache that value directly.
- Always write to the ``configuration_cache`` using the configured
``settings.CONFIGURATION_CACHE_TIMEOUT``.
Args:
key (str): The configuration key to cache.
value (Any | None): An explicit value to cache. If ``None``, the value
is loaded from the database and cast via ``get_value()``.
Returns:
Any: The value that was stored in the cache.
Raises:
Configuration.DoesNotExist: If ``value`` is ``None`` and there is no
``Configuration`` row with the given key.
"""
config_cache = caches["configuration_cache"]
cache_key = f"{CONFIGURATION_KEY_PREFIX}_{key}"
if value is None:
config = Configuration.objects.get(key=key)
value = config.get_value()
config_cache.set(cache_key, value, timeout=settings.CONFIGURATION_CACHE_TIMEOUT)
return value
================================================
FILE: configuration/validation.py
================================================
import re
from django.core.exceptions import ValidationError
RATE_LIMIT_PATTERN = re.compile(r"^\d+/(s|m|h|d)$")
def validate_rate(rate: str) -> str:
"""
Validate that a rate string matches the expected pattern like '10/m'.
Behavior:
- Strip leading and trailing whitespace.
- Require the format '/' where unit is one of
's', 'm', 'h', or 'd' (seconds, minutes, hours, days).
- Return the cleaned string unchanged if valid.
Args:
rate (str): The candidate rate string to validate.
Returns:
str: The cleaned rate string if valid.
Raises:
ValidationError: If the input is not a string, if the format does not
match the required pattern, or if the integer portion is less than
or equal to zero.
"""
if not isinstance(rate, str):
raise ValidationError("Rate limit must be a string.")
rate = rate.strip()
if not RATE_LIMIT_PATTERN.match(rate):
raise ValidationError("Invalid rate limit format. Use '/'.")
count, unit = rate.split("/")
if int(count) <= 0:
raise ValidationError("Rate limit count must be greater than 0.")
return rate
================================================
FILE: configuration/views.py
================================================
# Create your views here.
================================================
FILE: db_scripts/Dockerfile
================================================
# Base layer with all tools
FROM public.ecr.aws/amazonlinux/amazonlinux:2023-minimal AS base
# Trusted CA for proxy
RUN curl -fsO --output-dir /etc/pki/ca-trust/source/anchors/ http://crl.loc.gov/LOC-ROOT-CA-1.crt \
&& update-ca-trust
# Install tools once
RUN dnf -y upgrade && \
dnf -y install postgresql15.x86_64 awscli-2.noarch && \
dnf -y clean all
# Logic for Dump
FROM base AS dump
COPY dump.sh .
RUN chmod +x dump.sh
CMD ["./dump.sh"]
# Logic for Restore
FROM base AS restore
COPY restore.sh .
RUN chmod +x restore.sh
CMD ["./restore.sh"]
================================================
FILE: db_scripts/dump.sh
================================================
#!/bin/bash
set -eu -o pipefail
if [[ -z "${ENV_NAME}" ]]; then
echo "ENV_NAME must be set prior to running this script."
exit 1
fi
if [ $ENV_NAME != "prod" ]; then
echo "This script should only be run in the production environment."
exit 1
fi
TODAY=$(date +%Y%m%d)
if [[ "$TODAY" =~ (0101|0401|0701|1001)$ ]]; then
TAGVALUE="true"
else
TAGVALUE="false"
fi
POSTGRESQL_PW="$(aws secretsmanager get-secret-value --region us-east-1 --secret-id crowd/${ENV_NAME}/DB/MasterUserPassword | python3 -c 'import json,sys;Secret=json.load(sys.stdin);SecretString=json.loads(Secret["SecretString"]);print(SecretString["password"])')"
POSTGRESQL_HOST="$(aws ssm get-parameter --region us-east-1 --name /concordia/${ENV_NAME}/db.url | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput["Parameter"];print(Parameter["Value"])')"
DUMP_FILE=concordia.dmp
echo "${POSTGRESQL_HOST}:5432:*:concordia:${POSTGRESQL_PW}" > ~/.pgpass
chmod 600 ~/.pgpass
pg_dump -Fc --no-acl -U concordia -h "${POSTGRESQL_HOST}" concordia -f "${DUMP_FILE}"
if [ -s $DUMP_FILE ]; then
aws s3 cp "${DUMP_FILE}" "s3://crowd-deployment/database-dumps/concordia.${TODAY}.dmp"
aws s3 cp "${DUMP_FILE}" s3://crowd-deployment/database-dumps/concordia.latest.dmp
aws s3api put-object-tagging --bucket 'crowd-deployment' --key database-dumps/concordia.${TODAY}.dmp --tagging '{"TagSet": [{ "Key": "first-dmp-of-quarter", "Value": "'${TAGVALUE}'" }]}'
aws s3api put-object-tagging --bucket 'crowd-deployment' --key database-dumps/concordia.latest.dmp --tagging '{"TagSet": [{ "Key": "first-dmp-of-quarter", "Value": "'${TAGVALUE}'" }]}'
fi
echo $?
================================================
FILE: db_scripts/restore.sh
================================================
#!/bin/bash
set -eu -o pipefail
export PATH=$HOME/.local/bin:$PATH
if [[ -z "${ENV_NAME}" ]]; then
echo "ENV_NAME must be set prior to running this script."
exit 1
fi
if [ $ENV_NAME = "prod" ]; then
echo "This script should not be run in the production environment."
exit 1
fi
POSTGRESQL_PW="$(aws secretsmanager get-secret-value --region us-east-1 --secret-id crowd/${ENV_NAME}/DB/MasterUserPassword | python3 -c 'import json,sys;Secret=json.load(sys.stdin);SecretString=json.loads(Secret["SecretString"]);print(SecretString["password"])')"
POSTGRESQL_HOST="$(aws ssm get-parameter --region us-east-1 --name /concordia/${ENV_NAME}/db.url | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput["Parameter"];print(Parameter["Value"])')"
DUMP_FILE=concordia.dmp
aws s3 cp s3://crowd-deployment/database-dumps/concordia.latest.dmp ${DUMP_FILE}
echo "${POSTGRESQL_HOST}:5432:*:concordia:${POSTGRESQL_PW}" > ~/.pgpass
chmod 600 ~/.pgpass
aws s3 sync s3://crowd-content s3://crowd-${ENV_NAME}-content --delete
psql -U concordia -h "$POSTGRESQL_HOST" -d postgres -c "select pg_terminate_backend(pid) from pg_stat_activity where datname='concordia';"
psql -U concordia -h "$POSTGRESQL_HOST" -d postgres -c "drop database concordia with (force);"
pg_restore --create -U concordia -h "${POSTGRESQL_HOST}" -Fc --dbname=postgres --no-owner --no-acl "${DUMP_FILE}"
RETURNCODE=$?
echo $RETURNCODE
if [ $RETURNCODE = 0 ] && [ $ENV_NAME = "test" ]; then
ECS_SERVICE="$(aws ecs list-services --region us-east-1 --cluster crowd-${ENV_NAME} | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput["serviceArns"];print(Parameter[0].split("/")[2])')"
# If a feature branch env is running the number of services in the test cluster increases.
NUMBER_OF_SERVICES="$(aws ecs list-services --region us-east-1 --cluster crowd-test | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput["serviceArns"];print(len(Parameter))')"
if [ $NUMBER_OF_SERVICES = 3 ];then
# Normal
ECS_SERVICE_2="$(aws ecs list-services --region us-east-1 --cluster crowd-${ENV_NAME} | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput["serviceArns"];print(Parameter[2].split("/")[2])')"
else
# Feature branch env exists.
ECS_SERVICE_2="$(aws ecs list-services --region us-east-1 --cluster crowd-${ENV_NAME} | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput["serviceArns"];print(Parameter[3].split("/")[2])')"
fi
aws ecs update-service --region us-east-1 --force-new-deployment --cluster crowd-${ENV_NAME} --service ${ECS_SERVICE}
aws ecs update-service --region us-east-1 --force-new-deployment --cluster crowd-${ENV_NAME} --service ${ECS_SERVICE_2}
elif [ $RETURNCODE = 0 ]; then
ECS_SERVICE="$(aws ecs list-services --region us-east-1 --cluster crowd-${ENV_NAME} | python3 -c 'import json,sys;ParameterInput=json.load(sys.stdin);Parameter=ParameterInput["serviceArns"];print(Parameter[0].split("/")[2])')"
aws ecs update-service --region us-east-1 --force-new-deployment --cluster crowd-${ENV_NAME} --service ${ECS_SERVICE}
fi
================================================
FILE: development/Containerfile
================================================
FROM python:3.12-slim-bookworm
# Major Node.js version to install (e.g., 20, 22). This is used to select the
# NodeSource APT repository "node_.x".
ARG NODE_MAJOR=20
# Define build-time arguments for UID and GID
ARG USERNAME
ARG UID
ARG GID
ENV DEBIAN_FRONTEND="noninteractive"
# Create the group and user with specified UID/GID
RUN groupadd -g $GID $USERNAME && \
useradd -m -u $UID -g $GID -s /bin/bash $USERNAME
# Bootstrap minimal tooling needed later in the build:
# - curl: download files/keys
# - ca-certificates: validate HTTPS endpoints
# - gnupg: import and dearmor APT repository signing keys
RUN apt-get update -qy && apt-get install -qy curl ca-certificates gnupg
# Ensure that the Library's certificate authority is trusted so the tampering
# proxy will not break TLS validation. See
# https://staff.loc.gov/wikis/display/SE/Configuring+HTTPS+clients+for+the+HTTPS+tampering+proxy.
RUN curl -fso /etc/ssl/certs/LOC-ROOT-CA-1.crt http://crl.loc.gov/LOC-ROOT-CA-1.crt && openssl x509 -inform der -in /etc/ssl/certs/LOC-ROOT-CA-1.crt -outform pem -out /etc/ssl/certs/LOC-ROOT-CA-1.pem && c_rehash
# Install Node.js via the NodeSource APT repository (manual setup; no setup
# script). Debian bookworm ships Node 18; adding this repo allows installing a
# newer major version (e.g., Node 20) via apt.
#
# This step:
# - creates a dedicated keyring directory under /etc/apt/keyrings
# - downloads and installs the NodeSource signing key into a keyring file
# - registers the NodeSource repository for the selected Node.js major line
#
# Note: When installing Node.js from NodeSource, the `nodejs` package includes
# npm (and npm comes with node-gyp), so there is no separate `npm` or
# `node-gyp` APT package to install here.
#
# References: NodeSource "Repository Manual Installation" guide. https://github.com/nodesource/distributions/wiki/Repository-Manual-Installation
RUN \
# Create a dedicated directory for third-party APT keyrings.
mkdir -p /etc/apt/keyrings && \
# Download the NodeSource repository signing key and store it as a keyring
# file that apt can use to verify NodeSource packages.
curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
| gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
# Register the NodeSource repository for the selected Node.js major version.
# The "signed-by=" option scopes trust to just this repository entry.
echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" \
> /etc/apt/sources.list.d/nodesource.list
RUN apt-get update -qy && apt-get dist-upgrade -qy && apt-get install -o Dpkg::Options::='--force-confnew' -qy \
build-essential \
git \
libmemcached-dev \
# Pillow/Imaging: https://pillow.readthedocs.io/en/latest/installation.html#external-libraries
libz-dev libfreetype6-dev \
libtiff-dev libjpeg-dev libopenjp2-7-dev libwebp-dev zlib1g-dev \
# Postgres client library to build psycopg
libpq-dev \
locales \
# Weasyprint requirements
libpango-1.0-0 libharfbuzz0b libpangoft2-1.0-0 \
# Tesseract
tesseract-ocr tesseract-ocr-all \
# Selenium/Chrome/chromedriver requirements
libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libxdamage1 libxkbcommon0 libasound2 libatspi2.0-0 \
# Additional tools for development
nano bash-completion \
nodejs && apt-get -qy autoremove && apt-get -qy autoclean
RUN locale-gen en_US.UTF-8
ENV LC_ALL=en_US.UTF-8
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US.UTF-8
ENV PYTHONUNBUFFERED 1
ENV PYTHONPATH /workspace
ENV DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-concordia.settings_docker}
RUN pip install --upgrade pip
RUN pip install --no-cache-dir pre-commit
RUN pip install --no-cache-dir pipenv
RUN npm install --silent --global npm@10
# Set the working directory and permissions
WORKDIR /workspace
COPY . /workspace
RUN chown -R $USERNAME:$USERNAME /workspace
# Switch to the new user
USER $USERNAME
# Set user's path to include local bin, which is where Python libraries will be installed
ENV PATH "/home/${USERNAME}/.local/bin:$PATH"
RUN mkdir -p /workspace/logs
RUN touch /workspace/logs/concordia.log
# Front-end build and asset pipeline:
# - update npm to a known major version
# - install JS dependencies (production-only)
RUN npm install --silent
# Additional JS build step for Vite - compile scss, bundle, hash, and compress js files.
RUN npm run build
RUN pipenv install --system --dev --deploy && rm -rf ~/.cache/
RUN pre-commit install-hooks
EXPOSE 80
CMD ["bash"]
================================================
FILE: development/README.md
================================================
# Concordia Development Containers
The files in this directory, `compose.yml` and `Containerfile`, have been created to better facilitate developing Concordia in a containerized environment. The files are compatible with both Docker and Podman, using docker-compose or podman-compose.
Though newer versions of docker-compose and podman-compose support combining compose files and compose file overriding, the versions of these tools available on some current distributions (such as Red Hat Enterprise Linux 9) do not, so a singular compose file (`compose.yml`) with all the necessary settings is provided here.
## Purpose
The intention of these files is to provide a usable development environment purely in containers.
The default configuration (`../docker-compose.yml`) creates a container environment that's not suitable for development. Primarily, it creates an container for Elasticsearch, which is often used except in production, and the `app` container is not configured well for development. It runs as root, which causes file permission issues, and runs the daphne asgi server, which can't be restarted without restarting the entire container (manually killing and starting daphne does not work, either, because that causes the container to shutdown).
In addition, the default container file for the `app` container (`../Dockerfile`) runs `../entrypoint.sh`, which does several things that are undesirable in a development environment that involves restarting containers regularly. It automatically generates and applies migrations, runs collectstatic and launches daphne. The development Containerfile instead simply launches a bash shell.
## Configuration
In order to use `compose.yml` with your compose CLI tool of choice, you'll need to pass in the path, either through an environment variable or a command-line switch.
```bash
podman-compose -f development/compose.yml
```
```dotenv
COMPOSE_FILE=development/compose.yml
```
Your CLI tool should be executed in the concordia directory (`..`).
### .env caveat
Note that some versions of the tools (notably, podman-compose<=1.0.6) do not use .env files. If you wish to use environment variables, you'll need to inject those variables manually, such as by using a script that sources your .env file before executing podman-compose.
## Development
Configuring for your environment when using containers follows the process in the [For Developers](../docs/for-developers.md) page, except that most of the work is done for you by `compose.yml` and `Containerfile`. Notably, you do not need to install any dependencies (except git and your compose CLI) on your host. You will still need to configure your .env file, but otherwise simply running `podman-compose up -d` or `docker-compose up -d`, with the proper COMPOSE_FILE configuration (and other additions to .env, see below), will create an app container with everything you need for development.
### Additions to .env
There are a few additions to your .env file required to properly use the provided `compose.yml` and `Containerfile`:
```dotenv
COMPOSE_FILE=development/compose.yml
HOME_DIR=/home//
AWS_SHARED_CREDENTIALS_FILE=/home//.aws/credentials
CONTAINER_UID=
CONTAINER_GID=
CONTAINER_USERNAME=
```
These values should be for the user account that you'll be doing development with, the same one that owns your local repository. This information is used to mount various necessary directories in the container, as well as configure the user account inside the container (to avoid running as root).
The last three settings can automatically be added to your .env file with the following scripts (executed in the directory with the .env):
```bash
#!/bin/bash
ENV_FILE=".env"
BACKUP_FILE=".env.bak"
# Create the .env file if it doesn't exist
touch "$ENV_FILE"
# Backup the original .env once
cp "$ENV_FILE" "$BACKUP_FILE"
# Set the values
NEW_UID=$(id -u)
NEW_GID=$(id -g)
NEW_USERNAME=$(whoami)
# Function to add or update a key in the .env file
update_env_var() {
local key="$1"
local value="$2"
if grep -qE "^${key}=" "$ENV_FILE"; then
sed -i "s/^${key}=.*/${key}=${value}/" "$ENV_FILE"
else
echo "${key}=${value}" >> "$ENV_FILE"
fi
}
# Update the values
update_env_var "CONTAINER_UID" "$NEW_UID"
update_env_var "CONTAINER_GID" "$NEW_GID"
update_env_var "CONTAINER_USERNAME" "$NEW_USERNAME"
echo "Backup saved as $BACKUP_FILE"
echo ".env updated with CONTAINER_UID=$NEW_UID, CONTAINER_GID=$NEW_GID, CONTAINER_USERNAME=$NEW_USERNAME"
```
### Attaching to the app container
Once you've launched your containers, you can attach to a shell in the app container to perform development. Your compose CLI tool should provide a method for doing this.
```bash
sudo docker-compose exec -it app bash
```
Note that older versions of podman-compose do not properly pass switches to the underlying command, meaning the above won't work with those versions of podmon-compose. You can instead run it without the switches:
```bash
sudo podman-compose exec app bash
```
However, this has the disadvantage of not creating an interactive shell shell, which can cause issues with bash functionality. If you have a version of podman-compose with this limitation, the workaround is to use podman directly instead:
```bash
sudo podman exec -it concordia_app bash
```
You have to use the full container name because compose.yml is not referenced when using podman directly.
### Configuring for development
You will need to manually collect the static files before running the development server. This only needs to be done once after building the app container (and when changing static files in the future).
```bash
npx vite build
python manage.py collectstatic --no-post-process
```
### Launch the development server
Launching the development server is identical to launching it outside a container. The app container is configured to map port 8000 in the container to port 80 on the host:
```bash
python manage.py runserver 0.0.0.0:8000
```
### Committing changes
Git and the Concordia precommit hooks are included in the app container. You can simply use git commands as normal inside /workspace in the container.
================================================
FILE: development/compose.yml
================================================
version: '3.6'
services:
redis:
container_name: concordia_redis
restart: unless-stopped
image: redis:latest
hostname: redis
ports:
- 63791:6379
volumes:
- redis_volume:/data
db:
container_name: concordia_db
restart: unless-stopped
image: postgres:15
environment:
POSTGRES_PASSWORD: ${POSTGRESQL_PW}
POSTGRES_USER: concordia
POSTGRES_MULTIPLE_DATABASES: test_concordia
ports:
- 54323:5432
volumes:
- ../postgresql:/docker-entrypoint-initdb.d:z
- db_volume:/var/lib/postgresql/data/
app:
container_name: concordia_app
hostname: app
restart: unless-stopped
build:
context: ..
dockerfile: development/Containerfile
args:
UID: ${CONTAINER_UID}
GID: ${CONTAINER_GID}
USERNAME: ${CONTAINER_USERNAME}
environment: &django-environment
WAIT_HOSTS: db:5432, redis:6379
POSTGRESQL_HOST: db
POSTGRESQL_PW: ${POSTGRESQL_PW}
CONCORDIA_ENVIRONMENT: development
DJANGO_SETTINGS_MODULE: ${DJANGO_SETTINGS_MODULE:-concordia.settings_template}
DEBUG: ${DEBUG:-}
REDIS_ADDRESS: redis
REDIS_PORT: 6379
AWS_PROFILE: ${AWS_PROFILE}
AWS_SHARED_CREDENTIALS_FILE: ${AWS_SHARED_CREDENTIALS_FILE}
TURNSTILE_SITEKEY: ${TURNSTILE_SITEKEY:-1x00000000000000000000AA}
TURNSTILE_SECRET: ${TURNSTILE_SECRET:-1x0000000000000000000000000000000AA}
depends_on:
- redis
- db
volumes:
- ..:/workspace:z
- ${HOME_DIR}/.aws:${HOME_DIR}/.aws:z
- ${HOME_DIR}/.gitconfig:${HOME_DIR}/.gitconfig:z
- ${HOME_DIR}/.ssh:${HOME_DIR}/.ssh:z
- images_volume:/concordia_images
networks:
- default
ports:
- 80:8000
stdin_open: true
tty: true
importer:
container_name: concordia_importer
hostname: importer
restart: unless-stopped
build:
context: ..
dockerfile: importer/Dockerfile
environment: *django-environment
depends_on:
- redis
- db
networks:
- default
volumes:
- ..:/app:z
- ${HOME_DIR}/.aws:/root/.aws:z
- images_volume:/concordia_images
celerybeat:
container_name: concordia_celerybeat
hostname: celerybeat
restart: unless-stopped
build:
context: ..
dockerfile: celerybeat/Dockerfile
environment: *django-environment
depends_on:
- redis
- db
networks:
- default
volumes:
db_volume:
images_volume:
redis_volume:
networks:
default:
================================================
FILE: docker-compose.yml
================================================
version: '3.6'
services:
opensearch-node:
image: public.ecr.aws/opensearchproject/opensearch:1
container_name: opensearch-node
environment:
- cluster.name=opensearch-cluster
- node.name=opensearch-node
- discovery.type=single-node
- bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
- OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
- 'DISABLE_INSTALL_DEMO_CONFIG=true' # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch
- 'DISABLE_SECURITY_PLUGIN=true' # initial local setup - Disables security plugin
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
hard: 65536
volumes:
- opensearch-data:/usr/share/opensearch/data
ports:
- 9200:9200 # REST API
- 9600:9600 # Performance Analyzer
networks:
- default
opensearch-dashboards:
image: public.ecr.aws/opensearchproject/opensearch-dashboards:1
container_name: opensearch-dashboards
ports:
- 5601:5601
expose:
- '5601'
environment:
- 'OPENSEARCH_HOSTS=http://opensearch-node:9200'
- 'DISABLE_SECURITY_DASHBOARDS_PLUGIN=true' # disables security dashboards plugin in OpenSearch Dashboards
networks:
- default
redis:
restart: unless-stopped
image: redis:latest
hostname: redis
ports:
- 6379:6379
volumes:
- redis_volume:/data
db:
restart: unless-stopped
image: postgres:15
environment:
POSTGRES_PASSWORD: ${POSTGRESQL_PW}
POSTGRES_USER: concordia
POSTGRES_MULTIPLE_DATABASES: test_concordia
ports:
- 5432:5432
volumes:
- ./postgresql:/docker-entrypoint-initdb.d
- db_volume:/var/lib/postgresql/data/
app:
restart: unless-stopped
build: .
env_file:
- .env
environment: &django-environment
WAIT_HOSTS: db:5432, redis:6379
POSTGRESQL_HOST: db
POSTGRESQL_PW: ${POSTGRESQL_PW}
CONCORDIA_ENVIRONMENT: development
DJANGO_SETTINGS_MODULE: ${DJANGO_SETTINGS_MODULE:-concordia.settings_docker}
DEBUG: ${DEBUG:-}
REDIS_ADDRESS: redis
REDIS_PORT: 6379
depends_on:
- redis
- db
volumes:
- .:/app
- images_volume:/concordia_images
networks:
- default
ports: # if running locally use 80:80, if running in local container use 8000:80
- 80:80
importer:
restart: unless-stopped
build:
context: .
dockerfile: importer/Dockerfile
environment: *django-environment
depends_on:
- redis
- db
networks:
- default
volumes:
- images_volume:/concordia_images
celerybeat:
restart: unless-stopped
build:
context: .
dockerfile: celerybeat/Dockerfile
environment: *django-environment
depends_on:
- redis
- db
networks:
- default
volumes:
db_volume:
images_volume:
redis_volume:
opensearch-data:
networks:
default:
================================================
FILE: docs/accessibility-goals.md
================================================
# Accessibility Goals
## Background Information
- Microsoft has a site with a number of resources detailing their “Inclusive Design” strategy: https://www.microsoft.com/design/inclusive/
Of note is the [“Inclusive 101” PDF](https://download.microsoft.com/download/b/0/d/b0d4bf87-09ce-4417-8f28-d60703d672ed/inclusive_toolkit_manual_final.pdf) covering the range of factors which designs should take into account and common strategies.
[Kill Your Personas: How persona spectrums champion real user needs](https://medium.com/microsoft-design/kill-your-personas-1c332d4908cc)
has a longer discussion about avoiding binary thinking when designing
## Desired Outcomes by Activity
We have desired goals for each category of activity on the site
### Transcription
- Users with mobility restrictions should be able to complete the process
- Users with color vision impairments should be able to complete the process except for specific pages where the content is inherently inaccessible until https://github.com/LibraryOfCongress/concordia/issues/666 is complete
- Users with significant vision impairments should be able to learn the nature of the task and why it is difficult to continue
### Review
- Users with mobility restrictions should be able to complete the process
- Users with color vision impairments should be able to complete the process except for specific pages where the content is inherently inaccessible until https://github.com/LibraryOfCongress/concordia/issues/666 is complete
- Users with significant vision impairments should be able to learn the nature of the task and why it is difficult to continue
### Final results
- The transcribed text should be accessible to everyone
### Project Information
- Information about the project should be accessible to everyone
- Users should be able to easily learn the level of difficulty for their participation in particular activities
================================================
FILE: docs/accessibility-techniques.md
================================================
# Accessibility Techniques
## Background Information
- 18F has a detailed U.S. government-focused guide: https://accessibility.18f.gov/
- The BBC has their comprehensive guide: https://bbc.github.io/accessibility-news-and-you/
## Assistive Technology Categories
### Screen Readers
#### Background information
- https://webaim.org/techniques/screenreader/
- The BBC has some guides for testing: https://bbc.github.io/accessibility-news-and-you/accessibility-and-testing-with-assistive-technology
#### Resources for common screen readers
##### JAWS
- Freedom Scientific has a detailed [JAWS HTML and ARIA support matrix](https://freedomscientific.github.io/VFO-standards-support/) and [a downloable version](https://www.freedomscientific.com/Downloads/JAWS) which may be used for up to 40 minutes without a license
- https://bbc.github.io/accessibility-news-and-you/accessibility-and-testing-with-jaws
##### Windows Narrator
- https://support.microsoft.com/en-us/help/17173/windows-10-hear-text-read-aloud
##### Apple (macOS and iOS) VoiceOver
- https://help.apple.com/voiceover/info/guide/
- https://bbc.github.io/accessibility-news-and-you/accessibility-and-testing-with-voiceover-ios
- https://bbc.github.io/accessibility-news-and-you/accessibility-and-testing-with-voiceover-os
================================================
FILE: docs/design-principles.md
================================================
## Design Principles for System Features & Capabilities
We are collaborating across the Library of Congress to support this vision: _creating new pathways of engagement, scholarship, and serving the public—while making history with people, staff, collections and systems—by inviting public participation in a transformative project that improves discovery of the treasures of the Library of Congress._
Two principles guide these proposed features and the ways we can launch, manage, and sustain a crowdsourced transcription and tagging platform at the Library of Congress:
**Trust** and **Approachability**
The system features and capabilities envisioned below will support a user-centered crowdsourcing initiative through functionality and a program of engagement.
## **ENGAGE**
### Participation by many audiences in the same place, at the same time
We have a unique opportunity to support many audiences including, but not limited to, a curious public, researchers, teachers, students, those seeking volunteer opportunities, and staff. This crowdsourcing platform presents a digital space in which audiences with different, but possibly overlapping interests, skill levels, and familiarity with the Library of Congress may gather together. This vision builds on the work Library of Congress staff have invested in crafting programs that serve tanging audiences based on their specific needs. We are likely to have a dynamic and constantly changing atmosphere of participants and collections; here are some essential approaches and features to accommodate many participants in the same place, at the same time.
#### Lowest barrier to entry: No accounts necessary (but possible)
To ensure that every visitor can contribute and begin their Library of Congress crowdsourcing experience immediately, it would be best that accounts are not required to participate. Accounts may still be possible but not set as a prerequisite to contribute. Entries by "anonymous" users can be assessed separately or in a wider pool of contributions. As discussed below, it will be important to understand activity on the site; therefore, care should be taken to inform even anonymous users about the ways their behavior is captured. It is essential that these details are explained with clear language about the need for and use of that information.
#### Accessibility
Library of Congress audiences and visitors have a range of needs. This project should at all times meet the needs of all audiences. Examples of ways to create a welcoming and accommodating space include making it clear how to customize font sizes, interface adjustments, font styles and color choices, and plain language for instructions and project contexts. There are opportunities to invite participation in the system and use of the collections in many ways including reviewing and integrating voice to text capabilities, particularly for tagging. By meeting and thoughtfully surpassing compliance standards, the Library of Congress can create pathways for increased accessibility within the transcription workflow - such as the simple addition of a field to create alt text for the images. Captioned video content to support collections contexts, blog posts about discoveries and activity updates, and other featured content would also foreground that all audiences, even those who are not directly transcribing or tagging, are invited.
#### Multiple entry points
We should anticipate visitors entering from many different paths: whether a visitor is routed through the homepage into a collection and then selects an individual page to transcribe and tag; or referred by a post on a social network; or uncovered in internet search engine results; or introduced via [https://loc.gov](https://loc.gov) or simply word of mouth; or returning to transcribe and tag again via their own account page. It should always be clear how a visitor would navigate to the instructions, to an overview of the crowdsourcing program, to move to one's account pages (if one has created an account), and to navigate "up" to the project or campaign page(s), if entering on an individual transcription or tagging page from persistent unique URL.
#### Many Paths through Digital Collections
Heterogeneous subjects, forms, stories, and tasks are key to sustaining a long-term and representative crowdsourcing project. It is likely that many visitors will follow the sequential presentation of images, just as they were reading a diary or letter, or following other documentation as it unfolds. However, other visitors may wish to explore pages across collections with minimal reorientation to the home or collection-level pages. Still other volunteers may wish to change tasks between transcribing and tagging, or performing review passes, while participating. Community managers may also wish to lead visitors to targeted activity. There are unique opportunities to use the tool with prepared and simultaneous programming around the collections and campaigns. It may be helpful to have "tracks" for the multiple audiences interacting in this dynamic space.
#### Make Sense Quickly: Invitations to Contribute through Information Architecture
Upon entering the platform, a visitor should understand what tasks are requested and the goals of the program. It should be easy to explore featured collections by tags identified by community managers and curators; it should also be possible to explore collections based on community or self-created tags. Some tag categories that might connect to audience interests include location, era, and event. It should also be easy to understand the historical contexts of the collections, including difficult topics and subject matter.
#### Make it Easy to Participate
From entry into the site, volunteers should feel welcomed and within only a few steps of getting started with contributing. Clear instructions that are accessible from many different points that incorporate illustrative examples; perhaps even instruction or participation support in varying formats, such as video. Helpful tips about material culture and practice, such as handwriting or paleography resources may further help. Technical features that support accuracy and ease of contribution include annotation tools and editors. Providing a mechanism for volunteers to elect to receive timely feedback on their efforts and provide similar to others are just some community management approaches that can support accuracy and ease of contribution.
#### Serving Completed Data in Multiple Formats
Text and tags that are created by volunteers as they participate will be used to improve the ways collections can be found and connected. These forms of data should also be made available as project level JSON, CSV, and XML files, as well as a full corpus of completed transcription text. Ideally, these forms or data would be presented within the site on a "data" or research page. It should also be made clear the ways in which visitors might use the loc.gov interface to download individual images; and the loc.gov JSON API to download images and data from digital collections.
#### Responsibly Share Code
Thoroughly documenting and sharing the design decisions that inform the crafting of the codebase will ensure that other libraries, cultural heritage organizations, and educational bodies can best decide if this tool is appropriate for their needs and matches the skills and resources they have to apply to a web-application based participatory project. We should appropriately document and make available the source code for the tool, as well as other technical considerations and design decisions. Furthermore, lessons learned in the process of developing and improving the underlying tool, as well as the program(s) of engagement made possible by the affordances of the tool, should be shared openly as supporting documentation and stand-alone considerations.
#### Foregrounding Collaboration
There are many ways to signal respect for the contributions of volunteers while making the experience of participating most rewarding. One way is to maximize the extent of collaboration by better integrating opportunities to "positively compound" participation within the process of transcription, tagging, and review. Most existing tools use a blend of asynchronous transcription and either algorithmic review (matching) or volunteer review. Showcasing interpretation and activity by other volunteers can help participants craft a shared, agreed upon, and quality version of transcribed text. Displaying tags as categorization in the form of arranged and connected knowledge allows participants to crosswalk understanding, opening possibilities of new discovery and connection to the project mission and one another.
## **UNDERSTAND**
### Analyze activity & assess participant motivations to improve experiences
It will be imperative to responsibly gather, analyze, and share information about activity in the crowdsourcing platform. This information will be used to best communicate, improve upon existing capabilities, and extend future possibilities of the tool and program of engagement. This section discusses features and capabilities to support better experiences for all participants, from volunteers to staff.
#### Privacy & Ethical Use of Data
It is imperative that we responsibly and carefully define the need to collect data - whether that is about activity, location, or other forms of information - and connect that explicitly to intended uses of that data to understand audience needs and improve service. However, it should not be required or possible to track users through their visits. Furthermore, it should be possible for users to have their accounts deleted and account information purged from the site, while retaining their anonymized contributions.
#### No Accounts Necessary... but also creating an Account is possible
As noted above, it should be as easy as possible for visitors to become volunteers. However, creating an account within the system will allow participants to recall their activity, customize their experience, and fulfill needs for reporting should their motivations relate to formal volunteering or school assignments. We should explain to volunteers that creating an account affords additional possibilities including estimating participation. Accounts would also allow volunteers to experience other benefits: invitations to webinars with curators, alerts about new or related collections, and notifications of completion of projects or subjects of interest perhaps based on tags and other self-selection.
#### Understanding for Community Management
For the health and future of the crowdsourcing initiative, gathering information about the efforts and communication of volunteers imperative. Presenting a clear snapshot of activity, recent discussion, possible roadblocks, and upcoming campaigns or communication would aid community management through quick assessment of the ecosystem's state. Easy access to "live data," rapidly gathering the pulse on high traffic projects or energetic discussion, and performance of ongoing campaigns are key needs for community managers. This scope of activity information would also be useful for community managers in discreetly and sparingly offering volunteers opportunities to re-engage with new collections.
#### Motivations meet Behavior
We can help identify needed capabilities or features by surfacing the reasons people wish to participate and mapping patterns of activity in the system to these goals. The experience of participating in this project will be dynamic. Our visitors will not cleanly map onto single personas because their reasons for participating may change from visit to visit and over time. Common motivations include pursuing personal learning objectives, contributing to something greater and access to open knowledge, and to fulfill course or volunteer requirements. In 2014, researchers and project managers from the Zooniverse described the benefits of designing digital citizen science projects for participants with limited time and commitment; Eveleigh et al termed this approach "designing for dabblers." Crafting workflows that support bite-sized or small targeted tasks can meet the needs of this type of volunteer who may fit in participation in the crowdsourcing platform amid their other interests and activities. Other volunteers may seek more immersive experiences. Furthermore, students and other volunteers may have participation targets to meet during their time on site.
#### Enabling Immersion
From user interfaces to tracks, there are opportunities to design a site that enables immersive, deep engagement with stories in the collections, and to find flow in serialized tasks; yet also makes it possible to step out of the workflow quickly and with confidence that one's contributions will be retained and valued. Success may not always mean pace of transcription but rather the attainment of a visitor's goals, whether to learn, fulfill volunteering hours, or make a meaningful contribution to the Library of Congress.
#### Volunteers with accounts should be able to understand their own activity
Participants in the crowdsourcing program who have created accounts should be able to quickly access, understand, filter or scope, print or download summaries of their activity. This information should be presented to them in a visually dynamic and customizable (or filtered) manner. It should also represent their cumulative activity as well as recent tasks, allow them to pick up where they left off, share or engage others with their own contributions, and perhaps suggest related content via tags or campaigns.
#### Ability to gather feedback from participants about their needs
Responses, questions, feedback, and other means of communication between participants and community managers will be essential to the health of the project. Beyond a feedback button or comment form, community managers will want to be able to assess participation activity data as feedback on complexities of the collections, spot barriers and drop offs, identify stickiness, and sight opportunities in content and behavior.
#### Tell a Public Story by Displaying Activity
Presenting information about system-wide activity, as well as individual activity, creates opportunities for shared understanding of the progress toward collective goals, as well as individualized approaches. This information can also be presented on an About or Homepage to quickly convey a sense of participation--via dynamic and perhaps interactive charts, torque maps, timelines, and more--and encourage visitors to join the activity.
#### Reporting for Staff and Organization
Being able to provide reporting that integrates with or corresponds to staff workflows will help this project become more closely connected to regular activity in the organization. Information gathered about project performance might include the number of pages per collection, the number of volunteers that contributed, the number of views and engagement with the object in loc.gov/Project One, and details about visits including generalized time on site and return visits that engage with collections they've shared. It should be possible to query this data for custom reports, and include filters or other bounding options for specific time frames. Careful consideration should be taken around visitor information; never achieving tracking of users, and at all times generalized and disconnected from any cross-walkable search.
## **CONNECT**
### Manage & Match Collections to Tasks to Data and more
These activities focus on engaging audiences with Library of Congress collections. Simultaneously, we strive to design a series of tasks and support that result in data that may be applied to improve access to those Library collections. This section recommends tasks, design decisions, and processes to achieve these goals.
#### Tasks: Transcription AND Tagging
Of a range of possible tasks, we've identified transcription and tagging to be most relevant to our goals to improve search and identification within Library of Congress collections. These two tasks can be mapped to the digital content lifecycle in the description phase. These tasks are envisioned as a means of creating asset or page level text that can enhance discovery of and access to the Library of Congress digital collections.
#### Transcription
Based on the goals of engaging audiences and creating useful text for discovery, legibility, and access, the recommended transcription process is one that facilitates asynchronous but rapid access transcription; asks for minimal interaction with the collection asset image; situates the transcription window adjacent to the collection asset image; offers an adjustable and immersive presentation, if desired; "positively compounds" the efforts of participants; honors the time and contributions of participants; visually supports identification of possible errors or areas needing additional work; and connects to the context of the object in focus, whether smooth transition between pages before and after or to the catalog information and project description.
#### Tagging
The goals of tagging are to classify or categorize the content in such a way as to make it discoverable for future use. It is recommended that the tagging feature be built to accommodate set(s) of collection level and platform-wide controlled vocabularies, as well as to accept crowd-generated tags. The former set of tags would allow the platform to leverage existing subject headings, known and popular loc.gov search terms, and other forms of metadata already associated with the collections. The latter tagging capability would allow volunteers to customize their experiences with collections once these tags were made available to search or surface content across the platform; it would also allow community managers to organize campaigns at the point of import and feature this content both on the homepage and highlighted throughout the experience of transcription and tagging. Finally, a visual coding of the tags would offer an opportunity for a subtle contextualization of staff and Library of Congress generated tags and those created by participants.
#### Task Ecosystem: Tasks in the Workflow
Projects and pilots including [Beyond Words](https://labs.loc.gov/experiments/beyond-words/) have demonstrated that the transcription and tagging tasks may successful be presented together in the crowdsourced workflow; however it should be possible for participants to elect to only complete one task, if they prefer. It is also possible that these tasks might be best presented in separate interfaces or as distinct workflows. The tasks should also be introduced to participants by describing the goals of the information that is being created, examples of how and where it will be discoverable after volunteers have contributed. Specifically, care should be taken to communicate the ways that the information created will be used—whether for search, for research, for publication and display, or to improve the features of the system—as these details will shape the ways the tasks are undertaken by volunteers.
#### Reaching Agreement
Agreement around the final version of transcription text will be a process that should be achieved through displaying the work of volunteers and facilitating a process in which they can negotiate to reach consensus of the completeness and quality of the text. The collections we will ask the public to transcribe will be of varying format, even changing in format from asset image to asset image. Therefore, there are likely to many points at which interpretation will occur. As a result, workflows that allow participants to work together and create a shared understanding of the asset image (at object/collection level) are most likely to be successful when the opportunity to exchange interpretations are presented throughout the transcription and tagging processes. Features in support of negotiated consensus include an ability to see, mark, report, and/or correct errors in the text; to discuss an asset image or page in a forum; and generalized discussion at the collection level. Another way to permit negotiated consensus is to allow for a peer-review workflow.
#### Complete the Cycle
It is imperative that the crowd-generated transcription text be returned and served with the [loc.gov](https://loc.gov) presentation of the object and at the asset image level. It may be possible to marry this information as metadata or perhaps as data supporting the collection or object. This urgency and responsibility in completing the cycle connects to a user-centered design; building trust with public participants by meeting motivations that relate to contributing to greater access to knowledge, while honoring their time and contribution.
#### Connect Activity to Collections
At all times it should be possible to navigate from the crowdsourcing project, campaign, or transcription and tagging page to the source object. In the volunteer's account view, it should be possible to navigate from the project to which they contributed to the record of that collection in [loc.gov](https://loc.gov). Furthermore, there are opportunities to connect volunteers to the [Ask a Librarian](https://www.loc.gov/rr/askalib/) service from the crowdsourcing platform, such as in the discussion section, the project and collection pages, the homepage, and even in the footer.
#### Managing Projects
These key features and approaches should be available for the long-term sustainability of the project:
- Continuity with existing staff workflows, including work within CTS (content transfer system)
- A pipeline for the queue of projects, as well as a monthly mechanism for proposing, identifying, and planning new collections for transcription including a recurring forum for nominating projects - perhaps even a sandbox or test space in which details of nominated collections may be stored or prepared
- Collections management in bulk to support creating an extensive queue; storing offline in advance of campaign, collection, or other programming needs
- Enabling Community Managers to post, launch and queue projects and campaigns via an administrative interface. Over time, this functionality can be expanded with a permissions-based self-service mechanism for curators and collections staff to identify, flag, or queue projects for transcription. This will likely require coherence with Library of Congress web services, Design & Development, as well as the CTS workflow
#### Applying our APIs
As with other features recommended here, it would be best to seize and map to existing workflows and technologies. A goal for the platform should be to identify and deliver collections via [loc.gov](https://loc.gov) API to become transcription projects. Furthermore, allowing the transcription results to be queried via the [loc.gov JSON API](https://libraryofcongress.github.io/data-exploration/) once associated with the source images or record would be valuable.
#### Transparent Development
The greatest opportunity exists to develop this tool and platform in the open, as open source with appropriate licensing. Documenting development in a repository like GitHub is advisable.
#### Identify, Articulate, Acknowledge Content Created by Volunteers
Identify content created by participants within loc.gov as volunteer-created. Allow volunteers with an account to retain a pointer to the work they created, with a persistent URL or URI. Actively acknowledge the contributions of volunteers in publications, presentations, social media, and communications outreach. License for content created in the transcription and tagging workflow should be public domain or CC0; it should also be explicitly stated and incorporated as metadata when presented as a dataset and in loc.gov. These contexts should be clearly communicated to volunteers contributing the content as well as researchers engaging with the content.
#### Data Available for Download
To compliment research and exploration at the object and collection level, providing transcription and tagging results as a bulk data set would allow researchers in a range of disciplines to explore patterns and connections across collections. The transcription text and tags should be licensed as openly as possible and volunteers should be kept informed of the ways their efforts support discovery in the Library of Congress systems, as well as the role their work plays in other scholarly inquiry.
#### Navigating options to get started
Visitors and volunteers should be able to navigate the available collections in several ways. For example, presenting available projects as a list or set of tiles and incorporating filtering and sorting capability would allow volunteers to swiftly make sense of the active opportunities. Incorporating existing metadata offers other ways to orient volunteers to opportunities. For example, offering selection based on the object time period or era, location metadata, or subject heading. Finally, as participation increases, it may be possible to estimate time to complete a project based on participation data. This approach serves the needs of visitors who wish to dedicate a specific amount of time to their visit to the project.
### **GROW**
### Sustaining and Improving through Workflows and Spaces of Participation
Once the system has been designed, developed, and launched and once visitors become volunteers, there are endless opportunities to improve and sustain the platform and the program based on the ways people use it, their needs and challenges, and information gathered in the process of participation.
#### Show the Work!
Displaying the work of others in the system allows participants, whether new or seasoned, to more quickly understand that others are contributing to shared goals. The visibility of content & contribution also signals active collaboration. Furthermore, visual indications of collaboration, whether intended or even in conflict, can be reflected in an interface that shows the efforts of editing. Allowing content to be editable, then marking out those edits—with font, size or other indicator—allows individuals to better understand who and what has come before them on the page.
#### Create an Atmosphere
Creating a shared understanding of expectations around behavior and communication can help shape expectations of respect and civility in this space. Inviting participants to acknowledge and commit to a Code of Conduct provides them the opportunity to reflect on how they will engage with others, as well as offers them support for framing communication they receive. Examples of public community code of conduct include the Coral Project and Airbnb; the former articulates acceptable and unacceptable behavior and expectations, while the latter asks customers (guests) to sign a code of conduct pledge as part of the booking process.
#### Clear Examples
As participants create transcription and tagged content and more collections are added, plenty of examples to best represent what is needed when encountering decision-making moments will emerge. In the interim, creating clear and annotated examples may be useful to those who are just getting started with transcription and tagging.
#### Collate and Connect Extraneous Knowledge
As volunteers participate in the crowdsourcing initiative, they will acquire, recall, and perhaps seek to share information and knowledge with others; perhaps to be helpful to other volunteers and, at other times, as an expression of their interest and engagement with Library of Congress collections. They may wish to share examples and further non-Library of Congress resources. A discussion board or forum can help volunteers achieve their motivations of learning, contributing to wider knowledge (a greater good), and to build community or a sense of shared purpose through discussion. It may also be useful to create an asset level discussion for specific or nuanced questions about the asset image or page that is receiving transcription and tagging activity. Impressive examples of discussion spaces in crowdsourcing projects include the Zooniverse talk pages and the Discourse implementation in use by In the Spotlight at the British Library.
#### Sustain Interest and Increase the Capability of Participants
As described above, visitors may have many different motivations when they first arrive at the crowdsourcing platform; they may also have different motivations each time they arrive on site. Their movement to becoming volunteers may be catalyzed by offering a range of tasks, heterogeneous collections, and oscillating levels of complexity. Furthermore, the system may be built with tracks that prompt a participant to continue in the next step, or asks them to contribute to a new task. It is also possible to blend these approaches and particularly helpful if these are cyclical or distributed opportunities - either naturally based on variety in the collections or designed by community management approaches. Furthermore, offering opportunities to problem-solve, support other volunteers in other roles, level up in transcription and tagging tasks, or self-select for more complex (or pilot) tasks can create a dynamic experience that is rewarding to volunteers and the program ecosystem.
#### Build Knowledge for Outcomes
It is recommended that Community Managers and staff with collections expertise work together closely to build programming and engagement in relation to the transcription and tagging tasks. This programming could take the shape of essays, webinars, edited video content, chats hosted in discussion pages or social networks, and in-person events. In these activities, persistent URLs for the page, collection, and campaign (or collection of tags) would be required. In the previous section, recommendations were shared for connecting activity to collections. It is also possible to connect collections to catalyze activity; for example, a prompt or pathway from the collection or object Project One page to the crowdsourcing platform.
#### Practice
One way to encourage activity is to offer a low risk point of entry in which a volunteer might practice and keep notes, such as in a sandbox. This space may also make it possible for a volunteer to elect to receive feedback that can increase their confidence, coherence and consistency in transcription and tagging, and connection between their goals and those of the program. Another means of training could be a practice page, potentially offered after a volunteer has made a few contributions or after they have created an account.
#### Piloting
The Library of Congress Labs team is well-situated to partner with staff in the Office of the Chief Information Officer and Library Services to continue to run crowdsourcing experiments focused on tasks, cataloging and metadata, and machine learning. Examples include working with Optical Character Recognition (OCR) and Handwritten Text Recognition (HTR) outputs to assess quality of transcription & assistance with tagging; parsing task workflows; repeated passes on the same material for thematic, semantic, or other tagging; named entity recognition and more. Furthermore, there are opportunities to integrate other collections and materials into a crowdsourced workflow including audio-visual and time-based media objects. Creating an extensible tool or system into which features that emerge from experiments could be added would support the evolution of the project, as well as create opportunities for volunteers to grow their skills and knowledge.
================================================
FILE: docs/for-developers.md
================================================
# For Developers
## Prerequisites
This application can run on a single Docker host using docker-compose.
(recommended for development environments). See the [development Readme](../development/README.md) for more information. Note that the instructions below assume you'll be developing on your host rather than in a container. The development Readme provides instructions on performing development in a purely containerized environment, without installing any dependencies (outside of git and your container tool of choice) on the host.
For production, see the
[cloudformation](https://github.com/LibraryOfCongress/concordia/tree/master/cloudformation) directory for AWS Elastic Container Service
stack templates.
## Running Concordia
### Docker Compose
```bash
git clone https://github.com/LibraryOfCongress/concordia.git
```
If you're intending to edit static resources, templates, etc. and would like to
enable Django's DEBUG mode ensure that your environment has `DEBUG=true` set
before you run `docker-compose up` for the `app` container. The easiest way to
do this permanently is to add it to the `.env` file:
```bash
echo DEBUG=true >> .env
```
##### Install the application virtual environment
These steps only need to be performed the first time you setup a fresh
virtualenv environment:
1. Ensure that you have the necessary C library dependencies available:
- `libmemcached`
- `postgresql`
- `node` & `npm` for the front-end tools
1. Ensure that you have Python 3.8 or later installed
1. Install [pipenv](https://docs.pipenv.org/) either using a tool like
[Homebrew](https://brew.sh) (`brew install pipenv`) or using `pip`:
```bash
pip3 install pipenv
```
1. If you encounter errors installing psycopg, you may need to set LDFLAGS in your environment variables.
1. Let Pipenv create the virtual environment and install all of the packages,
including our developer tools:
```bash
pipenv install --dev
```
n.b. if `libmemcached` is installed using Homebrew you will need to [set the CFLAGS long enough to build it](https://stackoverflow.com/questions/14803310/error-when-install-pylibmc-using-pip#comment94853072_19432949):
```bash
CFLAGS=$(pkg-config --cflags libmemcached) LDFLAGS=$(pkg-config --libs libmemcached) pipenv install --dev
```
Once it has been installed you will not need to repeat this process unless
you upgrade the version of libmemcached or Python installed on your system.
1. Configure the Django settings module in the `.env` file which Pipenv will use
to automatically populate the environment for every command it runs:
```bash
echo DJANGO_SETTINGS_MODULE="concordia.settings_dev" >> .env
```
You can use this to set any other values you want to customize, such as
`POSTGRESQL_PW` or `POSTGRESQL_HOST`.
n.b to allow a local server to connect to the dockerized db set `POSTGRESQL_PORT=54323` - the db containers external postgres port.
1. Make sure that [redis](https://redis.io/docs/getting-started/) is installed and
running.
1. Configure Turnstile in your `.env` file. Unless specifically testing Turnstile,
you'll probably want the following settings:
```bash
echo TURNSTILE_SITEKEY=1x00000000000000000000BB >> .env
echo TURNSTILE_SECRET=1x0000000000000000000000000000000AA >> .env
```
Those two settings ensure all Turnstile tests pass. See [Turnstile Testing](https://developers.cloudflare.com/turnstile/troubleshooting/testing/) for other options.
### Local Development Environment
You will likely want to run the Django development server on your localhost
instead of within a Docker container if you are working on the backend. This is
best done using the same `pipenv`-based toolchain as the Docker deployments:
#### Python Dependencies
Python dependencies and virtual environment creation are handled by
[pipenv](https://docs.pipenv.org/).
If you want to add a new Python package requirement to the application
environment, it must be added to the Pipfile and the Pipfile.lock file.
This can be done with the command:
```bash
pipenv install
```
If the dependency you are installing is only of use for developers, mark it as
such using `--dev` so it will not be deployed to servers — for example:
```bash
pipenv install --dev django-debug-toolbar
```
Both the `Pipfile` and the `Pipfile.lock` files must be committed to the source
code repository any time you change them to ensure that all testing uses the
same package versions which you used during development.
#### Launching the environnment
In order to successfully launch the environment, the environment variables
`POSTGRESQL_PW` and `DJANGO_SETTINGS_MODULE` must be set. `POSTGRESQL_PW`
may be set to any value (which will become the database password for the
environment), but `DJANGO_SETTINGS_MODULE` should be set to
`concordia.settings_dev` to use the development settings file.
```bash
export POSTGRESQL_PW=password
export DJANGO_SETTINGS_MODULE=concordia.settings_dev
```
```bash
cd concordia
docker-compose up
```
Browse to [localhost](http://localhost)
#### Setting up a local development server
##### See section - [Ensuring your work follows the Library's coding standards](https://github.com/LibraryOfCongress/concordia/blob/master/docs/how-we-work.md#ensuring-your-work-follows-the-librarys-coding-standards) in How We Work
##### Start the support services
Instead of doing `docker-compose up` as above, instead start everything except the app:
```bash
docker-compose up -d db redis importer celerybeat
```
This will run the database in a container to ensure that it always matches the
expected version and configuration. If you want to reset the database, simply
delete the local container so it will be rebuilt the next time you run
`docker-compose up`: `docker-compose rm --stop db`.
##### Install front end
1. Install Node 20. If you're on MacOS, you can install it using brew:
```bash
brew install node@12
```
1. Use NPM to install our development tools:
```bash
npm install
```
1. In another terminal, start Vite to watch for changes to the SCSS files and
compile them to CSS, and changes in the bundled, hased and compressed js files:
```bash
npx vite
```
If you only want to build, bundle, and compress them a single time without live updates:
```bash
npx vite build
```
1) You may need to manually create a logs directory.
```bash
mkdir logs
```
1) Collect Django static files:
```bash
pipenv run ./manage.py collectstatic --no-post-process
```
##### Start the application server
1. Apply any database migrations:
```bash
pipenv run ./manage.py migrate
```
1. Start the development server:
```bash
pipenv run ./manage.py runserver
```
#### Running the unit tests
Use the `settings_local_test` Django settings in your environment. Your `.env` file should look something like:
```bash
POSTGRESQL_PW=password
DJANGO_SETTINGS_MODULE=concordia.settings_local_test
```
Bring up the docker database and redis servers:
```bash
docker-compose up -d db redis
```
Then execute the tests:
```bash
pipenv run ./manage.py test
```
#### Import Data
Once the database, redis service, importer and the application
are running, you're ready to import data.
First, [create a Django admin user](https://docs.djangoproject.com/en/2.1/intro/tutorial02/#creating-an-admin-user)
and log in as that user.
Then, go to the Admin area (under Account) and click "Bulk Import Items".
Upload a spreadsheet populated according to the instructions. Once all the import
jobs are complete, publish the Campaigns, Projects, Items and Assets that you
wish to make available.
#### Data Model Graph
To generate a model graph, make sure that you have [GraphViz](https://graphviz.org/doc/info/command.html) installed (e.g.
`brew install graphviz` or `apt-get install graphviz`) and use the
[django-extensions `graph_models`](https://django-extensions.readthedocs.io/en/latest/graph_models.html) command:
```bash
dot -Tsvg <(pipenv run ./manage.py graph_models concordia importer) -o concordia.svg
```
## Other Front-End Tools
### Public-facing URLs
1. If you need a list of public-facing URLs for testing, there's a management
command which may be helpful:
```bash
pipenv run ./manage.py print_frontend_test_urls
```
### Accessibility testing using aXe
Automated tools such as [aXe](https://www.deque.com/axe/) are useful for
catching low-hanging fruit and regressions. You run aXe against a development
server by giving it one or more URLs:
```bash
npx axe-cli --show-errors http://localhost:8000/
pipenv run ./manage.py print_frontend_test_urls | xargs npx axe-cli --show-errors
```
### Static Image Compression
When you update any of the files under `concordia/static/img`, please use an
optimizer such as [ImageOptim](https://imageoptim.com) or [Caesium](https://caesium.app/)
to **losslessly** compress JPEG, PNG, SVG, etc. files.
```bash
brew cask install imageoptim
open -a ImageOptim concordia/static/img/
```
================================================
FILE: docs/how-we-work.md
================================================
# How We Work
## Principles
Our basic principles are:
- We produce open source software, shared in repositories where it may be inspected by the public who places their trust in it and copied for use by other agencies or institutions.
- We adhere to the basic practices of agile software development, using the Scrum development framework.
- We practice human-centered design. Everything that we produce is highly accessible, per [WCAG 2.1](https://www.w3.org/TR/WCAG21/).
- Finally, we believe in having the relevant decision-makers at the table during all meetings, to maximize efficiency and maintain momentum.
## Product Team
This is a cross functional product team for Concordia made up of members across the Library who are working together. This product team will be comprised of the following roles:
- Product owner
- Product manager (Scrum master)
- Technical lead
- User Experience designer
- Developers (Front-end, Back-end, Full-stack)
- QA Tester
- Community Managers (content writers, administrators)
This team participates in stand ups, product alignment, backlog grooming and retrospectives in service of prioritizing, defining and delivering value to the department and the public it serves.
## Sprint Organization and Meetings
Each sprint is three weeks long. We have a sprint kick off the first day of the new sprint. There are five basic meeting rhythms:
- **Backlog grooming and Sprint Planning, every 2 weeks**
- Structure: tickets in the backlog are sorted by priority, the team adds acceptance criteria, estimates size, and assigns the tasks to a team member.
- **Demo and retrospectives, every 2 weeks**
- At the end of each sprint, Developers or content writers demo completed work in the sprint for the larger library stakeholders. During demo, we will confirm if the user acceptance criteria is met and moved to be tested. Following the demo, the team will go through a retrospective. These are held back-to-back, on the same day
## Definition of Done
So that we can work more efficiently and be confident in the quality of the work we are delivering, we have a clear definition of what it means for a user story to be done, or production-ready.
**For delivering a user story to the product team:**
- Story needs to be written in a way that is clear from both a development and a testing standpoint. Stories will need to be reviewed by the product team during creation.
- Acceptance criteria should include the relevant tests needed (unit, security, performance, acceptance, etc)
- Acceptance criteria should include the objective of the story, for use in approval by PO or tech team or both
- The delivered functionality should match the acceptance criteria of the user story
**for product team to accept the user story and ship it:**
- The functionality meets the acceptance criteria
- The product team has verified the functionality in staging
- All tests must pass in the the stage environment (unit, integration, feature)
- The delivered functionality should be 508 compliant
- Security requirements must be met
- All documentation must be up to date (diagrams, training documentation, API documentation, help text, etc)
- The delivered functionality should be compatible with the latest versions of Firefox, Chrome and Safari
## Processes
### Testing Strategy
We practice testing at three levels: unit tests, integration tests, and feature tests. For details about how we create and maintain unit, integration and feature tests.
- Unit - Unit tests must be created for all new code, during the sprint in which the code is written, with coverage of at least 90%.
- Integration - Code must include tests that verify that interfaces are functioning as designed.
- Feature - New features must have functional definitions of the thing that they are to perform, and a description of human-performable actions to verify that they perform that thing.
#### Testing new code
Each ticket will include acceptance tests, especially for new user facing functionality. When the developer signals that the ticket is ready for test, they will move it to the `Needs test` column and pings the tester in the comment of the ticket.
Testers will identify issues - HIGH, LOW, or NONE. Here are the criteria for each of the levels:
- FAIL: Does not meet the acceptace criteria and can not complete the acceptance test.
- PASS: Does meet both acceptace criteria and and acceptance tests. When a ticket passes but there are noticable opportunities for improvements or enhancements, close the ticket and create a new ticket and add to the backlog.
If all user acceptance criteria has been met, the ticket will be closed and moved to Done column.
Final step: Technical lead will create a release with all tickets that are done.
**How to provide feedback**
If feature testers find issues to address:
- FAIL: comment and @ the developer in the feature ticket
- Enhancements or Improvements: open a new FEATURE ticket. This ticket will be added to the backlog and up for priortization in alignment meeting and backlog grooming. Link to related ticket by adding the issue #.
If a FAIL issue needs to be addressed, tester should expect to be available to respond to developers questions and retest until acceptance criteria is met.
### Ticket Movement in a Sprint
**For a new feature ticket:**
1. Ticket is generated as an issue and placed in the backlog
2. Product owner will place priority tickets in a sprint
3. Product Manager will ensure all acceptance criteria has been articulated and assign developer
4. Developer will move to In Progress when ticket is being worked on
5. When Developer has completed initial code will move code into crowd-test.loc.gov
6. Developer moves ticket into test assigned Product Owner to review feature needs
7. Product Owner test the feature and test if Acceptance Criteria is met, provides feedback to devs if needed
8. Product Owner approves feature then assigns to QA
9. QA will affirm AC and test broader functionality and accessibility. Will pass/fail ticket. Outcome of testing will be written in comments.
- If pass, QA will close ticket and move to Done.
- If fail, QA will move ticket back into In Progress and assigned back to developer
10. If additional issues are found in testing that are not related to the feature of functionality, new tickets will be written by QA and added to the backlog. Ticket will be assigned to PO for further ticket development and grooming.
**For system wide upgrades**
1. Ticket is generated as an issue and placed in the backlog in GitHub
2. Technical Lead will place priority tickets in a sprint
3. Product Manager will ensure all acceptance criteria has been articualted and assign developer
4. Developer will move to In Progress when ticket is being worked on
5. Unit testing incorporated into the code pipeline
- Manually run unit tests
- CI/CD integration
6. Move ticket into test and assign peer review testing by developer
- If fail, developer provides feedback, moves ticket back to In Progress and assigns to the original dev
- If pass, assign to QA for regression testing
7. Tester will copy the [Regression Testing Checklist](https://staff.loc.gov/wikis/x/UomCBQ) as a comment in the ticket
8. Tester will go through all the functionality described in the checklist
- If fail, provides feedback in the comments, moves ticket back to In Progress and assigns to dev
- If pass, checks boxes to show that all main functionality are working and moves ticket to done
### Branch strategy and Pull Request Process
#### Git branching strategy
We have two long-lived git branches, `master` and `release`.
The `master` branch continuously deploys to our development environment.
The `release` branch continuously deploys to our staging environment.
Our development and staging environments are on AWS and only accessible through the Library's network.
##### Starting new work
When someone begins new work, they cut a new branch from `master` and name it after their work, perhaps `feature1`. New changes are pushed to the feature branch origin.
##### Merging to `master`
When new work is complete, we set up a Pull Request (PR) from `feature1` to `master`. Discussion about, and approval of changes by either the Technical Lead, Product Owner or both happens in the PR interface in GitHub.
Once this new work is approved we merge the code, which closes the PR.
From here, our CI pipeline will build the new changes on the `master` branch. Next, our CD pipeline will deploy the new work to our development environment.
##### Merging to `release`
Once the development work on a sprint is completed, we set up a PR from `master` to `release`.
This constitutes a new release candidate. Any last-minute discussion, as well as approval happens in the PR interface. Once approved by the Technical Lead, Product Owner or both and merged, CI runs for `release` branch to the staging environment.
##### Tagging and deploying to production
When the `release` branch has been fully tested in the staging environment, we create a GitHub release with a tag on the `release` branch.
Either trigger a Jenkins build manually or wait for continuous integration for the `release` branch to kick in. This will build a cleanly tagged versioned release candidate and upload the docker images to Amazon Elastic Container Registry.
To deploy to production, create a new task revision for the `concordia-prod` task which updates the version numbers of the docker containers to match the recently built cleanly tagged release candidate. Update the production service to use the new task definition revision. Monitor the health check endpoint to ensure the service is updated to the proper version.
##### Patching production mid-sprint
If a problem is identified in production that needs a quick fix, we code the fix to production in a new branch cut from `release`, maybe called `prod_fix`. We set up a PR against `release` for review and discussion.
Any QA or manual testing will take place in the staging environment deployed from the `release` branch. Once the release is tagged and deployed to production, we have to bring those new changes in release back into master. We use rebase again: `git rebase master release`.
### Code quality and review process
Code reviews are about keeping code clean and limiting technical debt. We will look for things that increase technical debt or create an environment where technical debt can be introduced easily later. Each pull request will be reviewed by the technical lead or assigned reviewer. As a reviewer, they will look closely for untested code, if there are tests that they are testing what they're supposed to, that they are following the Library's code standards.
### Ensuring your work follows the Library's coding standards
The project extends the standard Django settings model for project configuration and the Django test framework for unit tests.
#### Configuring your virtual env
The easiest way to install the site is using [Pipenv](https://pipenv.readthedocs.io/) to manage the virtual environment and install dependencies.
#### Configure your local checkout with code-quality hooks
1. Install [pre-commit](https://pre-commit.com/)
1. Run `pre-commit install`
Now every time you make a commit in Git the various tools listed in the next
section will automatically run and report any problems.
n.b. Each time you check out a new copy of this Git repository, run `pre-commit install`.
#### Configure your editor with helpful tools:
[setup.cfg](https://github.com/LibraryOfCongress/concordia/blob/master/setup.cfg) contains configuration for [pycodestyle](https://pypi.org/project/pycodestyle/), [isort](https://pypi.org/project/isort/) and [flake8](https://pypi.org/project/flake8/).
Configure your editor to run black and isort on each file at save time.
1. Install [black](https://pypi.org/project/black/) and integrate it with your editor of choice.
2. Run [flake8](http://flake8.pycqa.org/en/latest/) to ensure you don't increase the warning count or introduce errors with your commits.
3. This project uses [EditorConfig](https://editorconfig.org) for code consistency.
If you can't modify your editor, here is how to run the code quality
tools manually:
```
$ black .
$ isort --recursive
```
Black should be run prior to isort. It's recommended to commit your code
before running black, after running black, and after running isort so
the changes from each step are visible.
## Tools we use
- GitHub - We use our GitHub organization for storing both software and collaboratively-maintained text.
- Slack - We use the Slack for communication that falls outside of the structure of Jira or GitHub, but that doesn’t rise to the level of email, or for communication that it’s helpful for everybody else to be able to observe.
- WebEx - We use WebEx for video conferencing in all our meetings
================================================
FILE: entrypoint.sh
================================================
#!/bin/bash
set -e -u # Exit immediately for unhandled errors or undefined variables
mkdir -p /app/logs
touch /app/logs/concordia.log
echo "Running makemigrations"
./manage.py makemigrations --merge --noinput
echo "Running migrations"
./manage.py migrate
echo "Ensuring our base configuration is present in the database"
./manage.py ensure_initial_site_configuration
if [ -v SENTRY_BACKEND_DSN ]; then
echo "Testing Sentry configuration"
echo "from sentry_sdk import capture_message;capture_message('This is a test event');" | ./manage.py shell
fi
echo "Running Django ASGI server"
daphne -b 0.0.0.0 -p 80 concordia.asgi:application
================================================
FILE: exporter/__init__.py
================================================
================================================
FILE: exporter/admin.py
================================================
# Register your models here.
================================================
FILE: exporter/apps.py
================================================
from django.apps import AppConfig
class ExporterConfig(AppConfig):
name = "exporter"
================================================
FILE: exporter/exceptions.py
================================================
from typing import List, Tuple
class UnacceptableCharacterError(ValueError):
"""
Raised when unacceptable characters are discovered in text to be exported.
Each violation is stored so that callers can inspect which line / column held
the character.
Args:
violations: A list of `(line, column, character)` triples representing
every disallowed character found. Line and column numbers are both
**1-based** so they can be reported directly to users.
"""
def __init__(self, violations: List[Tuple[int, int, str]]):
self.violations: List[Tuple[int, int, str]] = violations
details = ", ".join(
f"line {ln} col {col} -> {ch!r}" for ln, col, ch in violations
)
super().__init__(f"Unacceptable characters found: {details}")
================================================
FILE: exporter/migrations/__init__.py
================================================
================================================
FILE: exporter/models.py
================================================
# Create your models here.
================================================
FILE: exporter/tabular_export/admin.py
================================================
# encoding: utf-8
"""
Helpers for exporting Django admin querysets as Excel or CSV files.
Usage in a ModelAdmin:
actions = (export_to_excel_action, export_to_csv_action)
These actions take the current queryset and export it using the same field
selection you would get from `values()` by default. The download filename is
derived from the `ModelAdmin.model._meta.verbose_name_plural` unless a custom
filename is passed.
These helpers are adapted from the original django-tabular-export implementation:
https://github.com/LibraryOfCongress/django-tabular-export/blob/master/tabular_export/admin.py
"""
from functools import wraps
from typing import Any, Callable, Iterable
from django.contrib.admin import ModelAdmin
from django.db.models import QuerySet
from django.http import HttpRequest, HttpResponse
from django.utils.encoding import force_str as force_text
from django.utils.translation import gettext_lazy as _
from .core import (
export_to_csv_response,
export_to_excel_response,
flatten_queryset,
)
def ensure_filename(suffix: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
"""
Decorator factory to ensure a default filename for export admin actions.
If the wrapped action is called with ``filename=None``, the filename is
built from ``modeladmin.model._meta.verbose_name_plural`` plus the given
suffix.
Args:
suffix (str): File extension to append (for example, ``"csv"`` or
``"xlsx"``).
Returns:
Callable[[Callable[..., Any]], Callable[..., Any]]: A decorator that
wraps an admin action and injects a default filename when needed.
"""
def outer(f: Callable[..., Any]) -> Callable[..., Any]:
@wraps(f)
def inner(
modeladmin: ModelAdmin,
request: HttpRequest,
queryset: QuerySet[Any],
filename: str | None = None,
*args: Any,
**kwargs: Any,
) -> HttpResponse:
if filename is None:
filename = "%s.%s" % (
force_text(modeladmin.model._meta.verbose_name_plural),
suffix,
)
return f(
modeladmin,
request,
queryset,
*args,
filename=filename,
**kwargs,
)
return inner
return outer
@ensure_filename("xlsx")
def export_to_excel_action(
modeladmin: ModelAdmin,
request: HttpRequest,
queryset: QuerySet[Any],
filename: str | None = None,
field_names: Iterable[str] | None = None,
extra_verbose_names: dict[str, str] | None = None,
) -> HttpResponse:
"""
Django admin action that exports selected records as an Excel XLSX download.
The queryset is first flattened via :func:`flatten_queryset`, optionally
restricted to the provided ``field_names`` and ``extra_verbose_names``,
then returned as an XLSX file response.
Args:
modeladmin (ModelAdmin): The Django admin class that owns this action.
request (HttpRequest): The current admin request.
queryset (QuerySet[Any]): The selected objects to export.
filename (str | None): Optional download filename. When omitted, a
name is generated from the model's ``verbose_name_plural`` and the
``"xlsx"`` suffix.
field_names (Iterable[str] | None): Optional iterable of field names to
include in the export. When omitted, the default flattening logic
is used.
extra_verbose_names (dict[str, str] | None): Optional mapping of field
names to custom column headers.
Returns:
HttpResponse: A response containing the XLSX file.
"""
headers, rows = flatten_queryset(
queryset,
field_names=field_names,
extra_verbose_names=extra_verbose_names,
)
return export_to_excel_response(filename, headers, rows)
export_to_excel_action.short_description = _("Export to Excel")
@ensure_filename("csv")
def export_to_csv_action(
modeladmin: ModelAdmin,
request: HttpRequest,
queryset: QuerySet[Any],
filename: str | None = None,
field_names: Iterable[str] | None = None,
extra_verbose_names: dict[str, str] | None = None,
) -> HttpResponse:
"""
Django admin action that exports selected records as a CSV download.
The queryset is first flattened via :func:`flatten_queryset`, optionally
restricted to the provided ``field_names`` and ``extra_verbose_names``,
then returned as a CSV file response.
Args:
modeladmin (ModelAdmin): The Django admin class that owns this action.
request (HttpRequest): The current admin request.
queryset (QuerySet[Any]): The selected objects to export.
filename (str | None): Optional download filename. When omitted, a
name is generated from the model's ``verbose_name_plural`` and the
``"csv"`` suffix.
field_names (Iterable[str] | None): Optional iterable of field names to
include in the export. When omitted, the default flattening logic
is used.
extra_verbose_names (dict[str, str] | None): Optional mapping of field
names to custom column headers.
Returns:
HttpResponse: A response containing the CSV file.
"""
headers, rows = flatten_queryset(
queryset,
field_names=field_names,
extra_verbose_names=extra_verbose_names,
)
return export_to_csv_response(filename, headers, rows)
export_to_csv_action.short_description = _("Export to CSV")
================================================
FILE: exporter/tabular_export/core.py
================================================
# encoding: utf-8
"""Exports to tabular (2D) formats
This module contains functions which take (headers, rows) pairs and return
HttpResponses with either XLSX or CSV downloads
The ``export_to_FORMAT_response`` functions accept a ``filename``, and
``headers`` and ``rows``. This allows full control over the data using
non-database data-sources, the Django ORM's various aggregations and
optimization methods, generators for large responses, control over the
column names, or post-processing using methods like ``get_FOO_display()``
to format the data for display.
The ``flatten_queryset`` utility used to generate lists from QuerySets
intentionally does not attempt to handle foreign-key fields to avoid
performance issues. If you need to include such data, prepare it in advance
using whatever optimizations are possible and pass the data in directly.
If your Django settings module sets ``TABULAR_RESPONSE_DEBUG`` to ``True``
the data will be dumped as an HTML table and will not be delivered as a
download.
Originally from
https://github.com/LibraryOfCongress/django-tabular-export/blob/master/tabular_export/core.py
"""
import csv
import datetime
from functools import wraps
from itertools import chain
from typing import Any, Callable, Iterable, Mapping, Sequence
from urllib.parse import quote
import xlsxwriter
from django.conf import settings
from django.db.models import QuerySet
from django.http import HttpResponse, StreamingHttpResponse
from django.utils.encoding import force_str
ResponseType = HttpResponse | StreamingHttpResponse
def get_field_names_from_queryset(qs: QuerySet[Any]) -> list[str]:
"""
Derive field names from a queryset, including extra and aggregate columns.
The queryset is first coerced to a ``values()`` queryset so that extra
selects and annotations appear with the same names Django would use for
``values()`` results.
Args:
qs: QuerySet to introspect.
Returns:
List of field and annotation names in the order they will appear in
``qs.values()``.
"""
# We'll set the queryset to include all fields including calculated
# aggregates using the same names which a values() queryset would return:
v_qs = qs.values()
field_names: list[str] = []
field_names.extend(i.target.name for i in v_qs.query.select)
field_names.extend(v_qs.query.extra_select.keys())
field_names.extend(v_qs.query.annotation_select.keys())
return field_names
def flatten_queryset(
qs: QuerySet[Any],
field_names: Iterable[str] | None = None,
extra_verbose_names: Mapping[str, str] | None = None,
) -> tuple[list[str], Iterable[Sequence[Any]]]:
"""
Convert a queryset into headers and row tuples for tabular export.
By default the column headers are derived from the queryset's field
names (as returned by ``get_field_names_from_queryset``) and the rows
use ``values_list()`` for efficient iteration.
If ``field_names`` is provided, only those fields are included and they
are used to order both headers and row values.
The ``extra_verbose_names`` mapping can override the verbose names for
specific fields, including related lookups or calculated values.
Args:
qs: Base queryset to flatten.
field_names: Optional explicit list of field names to include.
extra_verbose_names: Optional mapping of field names to friendly
column labels. This can be used to provide proper names for
related lookups (for example,
``{"institution__title": "Institution"}``) or calculated values
(for example, ``{"items__count": "Item Count"}``).
Returns:
A 2-tuple of ``(headers, rows)`` where ``headers`` is a list of
column labels and ``rows`` is an iterable of sequences of values.
"""
if field_names is None:
field_names = get_field_names_from_queryset(qs)
# Headers will use the verbose names where available and fall back to the
# field name if not (e.g. custom aggregate or extra fields):
verbose_names = {i.name: i.verbose_name for i in qs.model._meta.fields}
if extra_verbose_names is not None:
verbose_names.update(extra_verbose_names)
headers = [verbose_names.get(i, i) for i in field_names]
return headers, qs.values_list(*field_names)
def convert_value_to_unicode(v: Any) -> str:
"""
Convert a value to a display-safe string for tabular export.
``None`` is rendered as an empty string. ``date`` and ``datetime``
instances are converted using ``isoformat()``. All other values are
coerced via ``force_str``.
Args:
v: Value to convert.
Returns:
String representation suitable for CSV, HTML, or XLSX output.
"""
if v is None:
return ""
elif hasattr(v, "isoformat"):
return v.isoformat()
else:
return force_str(v)
def set_content_disposition(
f: Callable[..., ResponseType],
) -> Callable[..., ResponseType]:
"""
Decorator that applies a Content-Disposition header using the filename.
The wrapped function must accept ``filename`` as its first positional
argument and return an ``HttpResponse`` (or subclass). The decorator
sets the ``Content-Disposition`` header using RFC 5987 encoding for the
provided filename.
Args:
f: Callable that builds the HTTP response for a given filename.
Returns:
Wrapped callable that always sets ``Content-Disposition`` on the
response.
"""
@wraps(f)
def inner(filename: str, *args: Any, **kwargs: Any) -> ResponseType:
response = f(filename, *args, **kwargs)
# See RFC 5987 for the filename* spec:
response["Content-Disposition"] = "attachment; filename*=UTF-8''%s" % quote(
filename
)
return response
return inner
def return_debug_reponse(
f: Callable[..., ResponseType],
) -> Callable[..., ResponseType]:
"""
Decorator to swap export responses for an HTML debug table.
When the ``TABULAR_RESPONSE_DEBUG`` setting is truthy, the wrapped
function is not called. Instead ``export_to_debug_html_response`` is
used and the ``Content-Disposition`` header is removed so the browser
renders the table inline.
Args:
f: Export callable to wrap.
Returns:
Wrapped callable that either returns the original export response or
an HTML debug response, depending on settings.
"""
@wraps(f)
def inner(filename: str, *args: Any, **kwargs: Any) -> ResponseType:
if not getattr(settings, "TABULAR_RESPONSE_DEBUG", False):
return f(filename, *args, **kwargs)
else:
resp = export_to_debug_html_response(filename, *args, **kwargs)
del resp["Content-Disposition"] # Don't trigger a download
return resp
return inner
def export_to_debug_html_response(
filename: str,
headers: Iterable[Any],
rows: Iterable[Sequence[Any]],
) -> StreamingHttpResponse:
"""
Build an HTML table response for inspection of tabular export data.
This is used when ``TABULAR_RESPONSE_DEBUG`` is enabled. It renders the
headers and rows into a simple Bootstrap-styled HTML table and returns a
``StreamingHttpResponse``.
Args:
filename: Suggested filename for the export (kept for API parity).
headers: Iterable of header labels.
rows: Iterable of row sequences.
Returns:
StreamingHttpResponse streaming the HTML document.
"""
def output_generator():
# Note the use of bytestrings to avoid unnecessary Unicode-bytes cycles:
yield b""
yield b'TABULAR DEBUG '
yield b'' # noqa
yield b""
yield b" ".join(
convert_value_to_unicode(i).encode("utf-8") for i in headers
)
yield b" "
for row in rows:
values = map(convert_value_to_unicode, row)
values = [i.encode("utf-8").replace(b"\n", b"%s ".join(values)
yield b" "
yield b"
Unacceptable Characters Report
{% if errors %}
Asset
Violations (line, column, char)
{% for entry in errors %}
{{ entry.asset }}
{% for v in entry.violations %}
Line {{ v.0 }}, Col {{ v.1 }}:
{{ v.2|reprchar }}
{% endfor %}
{% endfor %}
{% else %}
No unacceptable characters were found.
{% endif %}
{% endblock content %}
================================================
FILE: exporter/tests/__init__.py
================================================
================================================
FILE: exporter/tests/test_exceptions.py
================================================
from django.test import TestCase
from exporter.exceptions import UnacceptableCharacterError
class UnacceptableCharacterErrorTests(TestCase):
def test_violations_are_stored(self):
"""The `violations` list passed to `__init__` is stored unmodified."""
violations = [(2, 3, "\u200b"), (4, 1, "\x00")]
err = UnacceptableCharacterError(violations)
self.assertEqual(err.violations, violations)
def test_message_contains_formatted_details(self):
"""The exception message should embed a human-readable summary."""
violations = [(1, 1, "\x00")]
err = UnacceptableCharacterError(violations)
msg = str(err)
self.assertIn("line 1 col 1", msg)
# The backslash in "\\x00" is escaped once by repr() and once in the
# string literal, so we search for the double-escaped form.
self.assertIn("\\x00", msg)
================================================
FILE: exporter/tests/test_tabular_export.py
================================================
import datetime
from unittest.mock import Mock
from django.db import models
from django.http import HttpResponse, StreamingHttpResponse
from django.test import TestCase, override_settings
from exporter.tabular_export.admin import (
export_to_csv_action,
export_to_excel_action,
)
from exporter.tabular_export.core import (
Echo,
convert_value_to_unicode,
export_to_csv_response,
export_to_debug_html_response,
export_to_excel_response,
flatten_queryset,
force_utf8_encoding,
get_field_names_from_queryset,
set_content_disposition,
)
class DummyModel(models.Model):
name = models.CharField(max_length=255, verbose_name="Name")
created = models.DateField(null=True, verbose_name="Created")
class Meta:
app_label = "tests"
class DummyQuerySet:
def __init__(self, data, field_names):
self._data = data
self._field_names = field_names
self.model = DummyModel
def values_list(self, *args):
return self._data
def values(self):
return self
@property
def query(self):
class Query:
select = [
type("Field", (), {"target": type("Target", (), {"name": fn})()})
for fn in self._field_names
]
extra_select = {"extra": "value"}
annotation_select = {"annotate": "value"}
return Query()
class CoreTests(TestCase):
def test_convert_value_to_unicode(self):
self.assertEqual(convert_value_to_unicode(None), "")
self.assertEqual(convert_value_to_unicode("abc"), "abc")
dt = datetime.datetime(2020, 1, 1, 12, 0)
self.assertEqual(convert_value_to_unicode(dt), "2020-01-01T12:00:00")
d = datetime.date(2020, 1, 1)
self.assertEqual(convert_value_to_unicode(d), "2020-01-01")
def test_echo_write(self):
echo = Echo()
self.assertEqual(echo.write("abc"), "abc")
def test_get_field_names_from_queryset(self):
qs = DummyQuerySet([], ["name", "created"])
self.assertEqual(
get_field_names_from_queryset(qs),
["name", "created", "extra", "annotate"],
)
def test_flatten_queryset_defaults(self):
qs = DummyQuerySet([("abc", datetime.date(2020, 1, 1))], ["name", "created"])
headers, rows = flatten_queryset(qs)
self.assertEqual(headers, ["Name", "Created", "extra", "annotate"])
self.assertEqual(list(rows), [("abc", datetime.date(2020, 1, 1))])
def test_flatten_queryset_with_custom_headers(self):
qs = DummyQuerySet([("abc",)], ["name"])
headers, rows = flatten_queryset(
qs, field_names=["name"], extra_verbose_names={"name": "Full Name"}
)
self.assertEqual(headers, ["Full Name"])
self.assertEqual(list(rows), [("abc",)])
def test_force_utf8_encoding(self):
def rows():
yield ["ü", "æ"]
out = list(force_utf8_encoding(rows)())
self.assertEqual(out, [[b"\xc3\xbc", b"\xc3\xa6"]])
def test_set_content_disposition(self):
@set_content_disposition
def dummy(filename):
return StreamingHttpResponse()
resp = dummy("test.csv")
self.assertIn(
"attachment; filename*=UTF-8''test.csv", resp["Content-Disposition"]
)
def test_export_to_debug_html_response(self):
headers = ["h1", "h2"]
rows = [["val1", "val2"], ["val3", "val4"]]
resp = export_to_debug_html_response("test.html", headers, rows)
self.assertIsInstance(resp, StreamingHttpResponse)
content = b"".join(resp.streaming_content)
self.assertIn(b" invalid
submitted=timezone.now(),
)
assets = get_latest_transcription_data(
Asset.objects.filter(pk__in=[self.asset.pk, bad_asset.pk])
)
with tempfile.TemporaryDirectory() as tmpdir:
errors = do_bagit_export(assets, tmpdir, "bad-bagit-no-request")
# helper should return a list, not an HttpResponse
self.assertIsInstance(errors, list)
self.assertEqual(len(errors), 1)
self.assertEqual(errors[0]["asset"], bad_asset)
# directory should have been removed
mock_rmtree.assert_called_once_with(Path(tmpdir), ignore_errors=True)
@override_settings(EXPORT_S3_BUCKET_NAME=None)
@patch("exporter.views.shutil.rmtree")
@patch(
"pathlib.Path.exists", return_value=False
) # force the exists() check to fail
def test_do_bagit_export_validation_errors_no_export_dir(
self,
mock_exists,
mock_rmtree,
):
"""
If validation fails and the export directory no longer exists,
do_bagit_export should NOT attempt to call shutil.rmtree.
"""
bad_asset = create_asset(
item=self.item,
sequence=77,
title="BadAssetNoDir",
download_url=DOWNLOAD_URL,
resource_url=RESOURCE_URL,
)
Transcription.objects.create(
asset=bad_asset,
user=self.user,
text="Oops\u200b", # zero-width space -> invalid
submitted=timezone.now(),
)
assets = get_latest_transcription_data(
Asset.objects.filter(pk__in=[self.asset.pk, bad_asset.pk])
)
with tempfile.TemporaryDirectory() as tmpdir:
# invoke without request -> should return raw errors list
errors = do_bagit_export(assets, tmpdir, "bad-bagit-no-dir")
# exists() forced to False -> rmtree must not be called
mock_rmtree.assert_not_called()
self.assertIsInstance(errors, list)
self.assertEqual(len(errors), 1)
================================================
FILE: exporter/utils.py
================================================
"""
Utility helpers for validating outgoing text.
The primary public entry-point is `exporter.utils.validate_text_for_export`,
which raises an `exporter.exceptions.UnacceptableCharacterError` when any
non-printable Unicode character is detected. Validation is performed per
line so the caller receives the exact location of every problem character.
"""
from typing import List, Tuple
from exporter.exceptions import UnacceptableCharacterError
_WHITELIST = [
"\t", # Tab
"\xa0", # Non-breaking space
"\u3000", # Ideographic space; used in Chinese/Japanese/Korean
"\u2003", # em space (space the width of the 'm' character)
]
def is_acceptable_character(character: str) -> bool:
"""
Return `True` when `character` is considered printable.
The function simply wraps `str.isprintable()` so behaviour stays in sync
with the official Unicode definition of *printable*.
Args:
character: A single Unicode character.
Returns:
`True` if the character is printable; otherwise `False`.
"""
return character.isprintable() or character in _WHITELIST
def find_unacceptable_characters(text: str) -> List[Tuple[int, int, str]]:
"""
Locate every non-printable character in *text*.
The scan is performed line by line so that the exact position (line and
column) of each offending character can be fed back to the caller.
Args:
text: The string to validate.
Returns:
A list of `(line_number, column_number, character)` triples. The list
may contain duplicates because each instance of an invalid character is
recorded individually.
"""
violations: List[Tuple[int, int, str]] = []
for line_no, line in enumerate(text.splitlines(), start=1):
for col_no, ch in enumerate(line, start=1):
if not is_acceptable_character(ch):
violations.append((line_no, col_no, ch))
return violations
def validate_text_for_export(text: str) -> bool:
"""
Validate `text` and raise if it contains any unacceptable characters.
Args:
text: The text destined for export.
Returns:
`True` if the text is valid for export
Raises:
UnacceptableCharacterError: If at least one non-printable character is
found.
"""
violations = find_unacceptable_characters(text)
if violations:
raise UnacceptableCharacterError(violations)
return True
def remove_unacceptable_characters(text: str) -> str:
"""
Produce a copy of `text` with all non-printable characters removed.
The removal uses the same acceptability rules as validation, ensuring the
behaviour stays consistent with `is_acceptable_character()` and the shared
whitelist. Standard line breaks are preserved. Characters considered
unacceptable (i.e., not printable and not in the whitelist) are omitted
from the result.
Args:
text: The input string to sanitize.
Returns:
A new string with all unacceptable characters removed.
Notes:
The scan mirrors `find_unacceptable_characters()` by operating
line-by-line. Unlike validation, there is no error raised; the
offending characters are dropped from the output. Newline characters
(``\\n`` and ``\\r``) are preserved so the original line structure is
maintained.
"""
cleaned_parts: List[str] = []
for line in text.splitlines(keepends=True):
# Keepends means any trailing '\n'/'\r\n' is part of `line`.
out_line_chars: List[str] = []
for ch in line:
# Preserve standard line breaks exactly as seen.
if ch == "\n" or ch == "\r":
out_line_chars.append(ch)
continue
if is_acceptable_character(ch):
out_line_chars.append(ch)
cleaned_parts.append("".join(out_line_chars))
return "".join(cleaned_parts)
================================================
FILE: exporter/views.py
================================================
import os
import re
import shutil
import tempfile
from collections.abc import Iterable
from logging import getLogger
from pathlib import Path
from typing import Any, List
import bagit
import boto3
from django.conf import settings
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.postgres.aggregates.general import StringAgg
from django.db.models import OuterRef, Subquery, TextField, Value
from django.db.models.functions import Coalesce
from django.db.models.query import QuerySet
from django.http import (
HttpRequest,
HttpResponse,
HttpResponseRedirect,
)
from django.shortcuts import render
from django.utils.decorators import method_decorator
from django.views.generic import TemplateView
from concordia.models import (
Asset,
Campaign,
Item,
Project,
Transcription,
TranscriptionStatus,
)
from exporter.exceptions import UnacceptableCharacterError
from exporter.tabular_export.core import export_to_csv_response, flatten_queryset
from exporter.utils import validate_text_for_export
logger = getLogger(__name__)
def get_latest_transcription_data(
asset_qs: QuerySet[Asset],
) -> QuerySet[Asset]:
"""
Annotate each asset with its latest transcription text.
The annotation is named ``latest_transcription`` and is derived from the
most recent ``Transcription`` by primary key.
Args:
asset_qs:
QuerySet[Asset] to annotate.
Returns:
QuerySet[Asset]: The input queryset annotated with
``latest_transcription``.
"""
latest_trans_subquery = (
Transcription.objects.filter(asset=OuterRef("pk"))
.order_by("-pk")
.values("text")
)
assets = asset_qs.annotate(
latest_transcription=Coalesce(
Subquery(latest_trans_subquery[:1]),
Value("", output_field=TextField()),
)
)
return assets
def get_tag_values(asset_qs: QuerySet[Asset]) -> QuerySet[Asset]:
"""
Annotate each asset with a semicolon-joined string of tag values.
The annotation is named ``tag_values`` and aggregates related tag text from
``userassettagcollection__tags__value``.
Args:
asset_qs:
QuerySet[Asset] to annotate.
Returns:
QuerySet[Asset]: The input queryset annotated with ``tag_values``.
"""
assets = asset_qs.annotate(
tag_values=StringAgg(
"userassettagcollection__tags__value",
"; ",
default=Value("", output_field=TextField()),
)
)
return assets
def remove_incomplete_items(item_qs: QuerySet[Item]) -> QuerySet[Asset]:
"""
Filter out items that are not fully completed and return their assets.
An item is considered incomplete if any of its assets have a status of
NOT_STARTED, IN_PROGRESS or SUBMITTED.
Args:
item_qs:
QuerySet[Item] to check for completeness.
Returns:
QuerySet[Asset]: Assets belonging to completed items, ordered by
project, item and sequence.
"""
incomplete_item_assets = Asset.objects.filter(
item__in=item_qs,
transcription_status__in=(
TranscriptionStatus.NOT_STARTED,
TranscriptionStatus.IN_PROGRESS,
TranscriptionStatus.SUBMITTED,
),
)
item_qs = item_qs.exclude(asset__in=incomplete_item_assets)
asset_qs = Asset.objects.filter(item__in=item_qs).order_by(
"item__project", "item", "sequence"
)
return asset_qs
def get_original_asset_id(download_url: str) -> str:
"""
Derive a stable external asset identifier from a LOC download URL.
For ``tile.loc.gov`` URLs a best-effort pattern is used to extract the
identifier. Non-matching URLs are returned unchanged.
Args:
download_url:
The asset's download URL.
Returns:
str: Identifier suitable for naming files inside the BagIt payload.
Raises:
ValueError: If the URL looks like ``tile.loc.gov`` but no ID is found.
"""
download_url = download_url.replace("https", "http")
if download_url.startswith("http://tile.loc.gov/"):
pattern = (
r"/service:([A-Za-z0-9:.\-\_]+)/"
+ r"|/master/([A-Za-z0-9/]+)([0-9.]+)"
+ r"|/public:[A-Za-z0-9]+:([A-Za-z0-9:.\-_]+?)/"
)
asset_id = re.search(pattern, download_url)
if not asset_id:
logger.error(
"Could not find a matching asset ID in download URL %s",
download_url,
)
raise ValueError(
f"Could not find a matching asset ID in download URL {download_url}"
)
matching_asset_id = next((group for group in asset_id.groups() if group))
logger.debug(
"Found asset ID %s in download URL %s",
matching_asset_id,
download_url,
)
return matching_asset_id
logger.warning(
"Download URL does not start with tile.loc.gov: %s",
download_url,
)
return download_url
def write_distinct_asset_resource_file(
assets: Iterable[Any], export_base_dir: str | Path
) -> None:
"""
Write a unique list of resource URLs for the provided assets.
The file is named ``item-resource-urls.txt`` and written at the export
root. Each line contains one distinct ``Asset.resource_url``.
Args:
assets:
Iterable of asset identifiers or a QuerySet[Asset]. Passed to
``Asset.objects.filter(pk__in=assets)``.
export_base_dir:
Directory where the file should be created.
Raises:
AssertionError: If an asset has no ``resource_url``.
"""
asset_resource_file = os.path.join(export_base_dir, "item-resource-urls.txt")
with open(asset_resource_file, "a") as f:
distinct_resource_urls = (
Asset.objects.filter(pk__in=assets)
.order_by("resource_url")
.values_list("resource_url", "title")
.distinct("resource_url")
)
for url, title in distinct_resource_urls:
if url:
f.write(url)
f.write("\n")
else:
logger.error("No resource URL found for asset %s", title)
raise AssertionError
def do_bagit_export(
assets: Iterable[Asset] | QuerySet[Asset],
export_base_dir: str | Path,
export_filename_base: str,
request: HttpRequest | None = None,
) -> HttpResponse | HttpResponseRedirect | List[dict[str, Any]]:
"""
Build and deliver a BagIt package for ``assets`` or report invalid chars.
The function validates every asset's ``latest_transcription``. For each
unacceptable character it records:
- the asset ID
- the 1-based line number
- the 1-based column number
- the offending character
Behaviour:
1. Validation pass: files are written only after a transcription passes
validation.
2. Failure(s): any files already written are removed. If ``request`` is
supplied a template is rendered, otherwise the raw error list is
returned.
3. All clear: a BagIt structure is created, zipped and either returned as a
download or uploaded to S3.
Args:
assets:
Iterable or QuerySet of ``Asset`` to export. Each must have
``download_url`` and ``latest_transcription``.
export_base_dir:
Temporary directory into which the BagIt hierarchy is built.
export_filename_base:
Base name (without ``.zip``) for the archive.
request:
Current request. If provided, an error template will be rendered
when validation fails.
Returns:
HttpResponse | HttpResponseRedirect | list[dict[str, Any]]:
- a download response when packaging locally
- a redirect to the uploaded archive when S3 is configured
- a list of validation errors when ``request`` is ``None`` and errors
were found
"""
export_base_dir = Path(export_base_dir)
errors: List[dict[str, Any]] = []
# Validate every transcription before writing anything to disk
for asset in assets:
asset_id = get_original_asset_id(asset.download_url)
asset_id = asset_id.replace(":", "/") # BagIt-safe path fragment
transcription = asset.latest_transcription or ""
try:
validate_text_for_export(transcription)
except UnacceptableCharacterError as err:
errors.append({"asset": asset, "violations": err.violations})
continue
# Passed validation -> write the transcription file
asset_path, asset_filename = os.path.split(asset_id)
asset_dest_path = export_base_dir / asset_path
asset_dest_path.mkdir(parents=True, exist_ok=True)
if transcription:
with open(asset_dest_path / f"{asset_filename}.txt", "w") as fp:
fp.write(transcription)
# If any errors -> cleanup and report
if errors:
if export_base_dir.exists():
shutil.rmtree(export_base_dir, ignore_errors=True)
if request is not None:
return render(
request,
"admin/exporter/unacceptable_character_report.html",
{"errors": errors},
)
return errors
# All assets valid -> create BagIt, zip, upload / return
write_distinct_asset_resource_file(assets, export_base_dir)
bagit.make_bag(
export_base_dir,
{
"Content-Access": "web",
"Content-Custodian": "dcms",
"Content-Process": "crowdsourced",
"Content-Type": "textual",
"LC-Bag-Id": export_filename_base,
"LC-Items": f"{len(assets)} transcriptions",
"LC-Project": "gdccrowd",
"License-Information": "Public domain",
},
)
archive_name = shutil.make_archive(
str(export_base_dir), "zip", root_dir=export_base_dir
)
export_filename = f"{export_filename_base}.zip"
s3_bucket = getattr(settings, "EXPORT_S3_BUCKET_NAME", None)
if s3_bucket:
logger.debug("Uploading exported bag to S3 bucket %s", s3_bucket)
s3 = boto3.resource("s3")
s3.Bucket(s3_bucket).upload_file(archive_name, export_filename)
return HttpResponseRedirect(
f"https://{s3_bucket}.s3.amazonaws.com/{export_filename}"
)
# Local download
with open(archive_name, "rb") as zip_file:
response = HttpResponse(zip_file, content_type="application/zip")
response["Content-Disposition"] = f"attachment; filename={export_filename}"
return response
class ExportCampaignToCSV(TemplateView):
"""
Stream a CSV of the most recent transcription for each asset in a campaign.
Only the latest transcription text per asset is included. Tag values
are aggregated into a semicolon-delimited string.
"""
@method_decorator(staff_member_required)
def get(self, request: HttpRequest, *args, **kwargs) -> HttpResponse:
"""
Return a CSV response for the requested campaign.
Args:
request: Current HTTP request.
Returns:
HttpResponse: CSV content for the campaign.
"""
asset_qs: QuerySet[Asset] = Asset.objects.filter(
item__project__campaign__slug=self.kwargs["campaign_slug"]
)
assets: QuerySet[Asset] = get_latest_transcription_data(asset_qs)
assets = get_tag_values(assets)
headers, data = flatten_queryset(
assets,
field_names=[
"item__project__campaign__title",
"item__project__title",
"item__title",
"item__item_id",
"title",
"id",
"transcription_status",
"download_url",
"latest_transcription",
"tag_values",
],
extra_verbose_names={
"item__project__campaign__title": "Campaign",
"item__project__title": "Project",
"item__title": "Item",
"item__item_id": "ItemId",
"item_id": "ItemId",
"title": "Asset",
"id": "AssetId",
"transcription_status": "AssetStatus",
"download_url": "DownloadUrl",
"latest_transcription": "Transcription",
"tag_values": "Tags",
},
)
logger.info("Exporting %s to csv", self.kwargs["campaign_slug"])
return export_to_csv_response(
"%s.csv" % self.kwargs["campaign_slug"], headers, data
)
class ExportItemToBagIt(TemplateView):
"""
Build a BagIt archive for a single item consisting of completed assets.
Only assets with ``TranscriptionStatus.COMPLETED`` are included.
"""
@method_decorator(staff_member_required)
def get(
self, request: HttpRequest, *args, **kwargs
) -> HttpResponse | HttpResponseRedirect:
"""
Create and return a BagIt archive for the requested item.
Args:
request: Current HTTP request.
Returns:
HttpResponse | HttpResponseRedirect: Local zip download or redirect
to S3.
"""
campaign_slug = self.kwargs["campaign_slug"]
project_slug = self.kwargs["project_slug"]
item_id = self.kwargs["item_id"]
asset_qs: QuerySet[Asset] = Asset.objects.filter(
item__project__campaign__slug=campaign_slug,
item__project__slug=project_slug,
item__item_id=item_id,
transcription_status=TranscriptionStatus.COMPLETED,
).order_by("sequence")
assets: QuerySet[Asset] = get_latest_transcription_data(asset_qs)
campaign = Campaign.objects.get(slug__exact=campaign_slug)
campaign_slug_dbv = campaign.slug
project = Project.objects.get(campaign=campaign, slug__exact=project_slug)
project_slug_dbv = project.slug
item_id_dbv = Item.objects.get(item_id__exact=item_id).item_id
export_filename_base = "%s-%s-%s" % (
campaign_slug_dbv,
project_slug_dbv,
item_id_dbv,
)
with tempfile.TemporaryDirectory(
prefix=export_filename_base
) as export_base_dir:
return do_bagit_export(
assets, export_base_dir, export_filename_base, request
)
class ExportProjectToBagIt(TemplateView):
"""
Build a BagIt archive for a project consisting of completed items only.
"""
@method_decorator(staff_member_required)
def get(
self, request: HttpRequest, *args, **kwargs
) -> HttpResponse | HttpResponseRedirect:
"""
Create and return a BagIt archive for the requested project.
Args:
request: Current HTTP request.
Returns:
HttpResponse | HttpResponseRedirect: Local zip download or redirect
to S3.
"""
campaign_slug = self.kwargs["campaign_slug"]
project_slug = self.kwargs["project_slug"]
item_qs: QuerySet[Item] = Item.objects.filter(
project__campaign__slug=campaign_slug, project__slug=project_slug
)
asset_qs: QuerySet[Asset] = remove_incomplete_items(item_qs)
assets: QuerySet[Asset] = get_latest_transcription_data(asset_qs)
campaign = Campaign.objects.get(slug__exact=campaign_slug)
campaign_slug_dbv = campaign.slug
project = Project.objects.get(campaign=campaign, slug__exact=project_slug)
project_slug_dbv = project.slug
export_filename_base = "%s-%s" % (campaign_slug_dbv, project_slug_dbv)
with tempfile.TemporaryDirectory(
prefix=export_filename_base
) as export_base_dir:
return do_bagit_export(
assets, export_base_dir, export_filename_base, request
)
class ExportCampaignToBagIt(TemplateView):
"""
Build a BagIt archive for a campaign consisting of completed items only.
"""
@method_decorator(staff_member_required)
def get(
self, request: HttpRequest, *args, **kwargs
) -> HttpResponse | HttpResponseRedirect:
"""
Create and return a BagIt archive for the requested campaign.
Args:
request: Current HTTP request.
Returns:
HttpResponse | HttpResponseRedirect: Local zip download or redirect
to S3.
"""
campaign_slug = self.kwargs["campaign_slug"]
item_qs: QuerySet[Item] = Item.objects.filter(
project__campaign__slug=campaign_slug
)
asset_qs: QuerySet[Asset] = remove_incomplete_items(item_qs)
assets: QuerySet[Asset] = get_latest_transcription_data(asset_qs)
campaign_slug_dbv = Campaign.objects.get(slug__exact=campaign_slug).slug
export_filename_base = "%s" % (campaign_slug_dbv,)
with tempfile.TemporaryDirectory(
prefix=export_filename_base
) as export_base_dir:
return do_bagit_export(
assets, export_base_dir, export_filename_base, request
)
class ExportProjectToCSV(TemplateView):
"""
Stream a CSV of the most recent transcription for each asset in a project.
Only the latest transcription text per asset is included. Tag values
are aggregated into a semicolon-delimited string.
"""
@method_decorator(staff_member_required)
def get(self, request: HttpRequest, *args, **kwargs) -> HttpResponse:
"""
Return a CSV response for the requested project.
Args:
request: Current HTTP request.
Returns:
HttpResponse: CSV content for the project.
"""
campaign_slug = self.kwargs["campaign_slug"]
project_slug = self.kwargs["project_slug"]
campaign = Campaign.objects.get(slug__exact=campaign_slug)
project = Project.objects.get(campaign=campaign, slug__exact=project_slug)
asset_qs: QuerySet[Asset] = Asset.objects.filter(item__project=project)
assets: QuerySet[Asset] = get_latest_transcription_data(asset_qs)
assets = get_tag_values(assets)
headers, data = flatten_queryset(
assets,
field_names=[
"item__project__campaign__title",
"item__project__title",
"item__title",
"item__item_id",
"title",
"id",
"transcription_status",
"download_url",
"latest_transcription",
"tag_values",
],
extra_verbose_names={
"item__project__campaign__title": "Campaign",
"item__project__title": "Project",
"item__title": "Item",
"item__item_id": "ItemId",
"item_id": "ItemId",
"title": "Asset",
"id": "AssetId",
"transcription_status": "AssetStatus",
"download_url": "DownloadUrl",
"latest_transcription": "Transcription",
"tag_values": "Tags",
},
)
logger.info("Exporting %s to csv", self.kwargs["project_slug"])
return export_to_csv_response(
f"{campaign.slug}-{project.slug}.csv", headers, data
)
================================================
FILE: fixtures/original-static-pages.json
================================================
[
{
"fields": {
"body": "crowd.loc.gov provides students of all ages with opportunities to explore unique historical documents from the collections of the Library of Congress.\n\nDeciphering and transcribing these documents can build students\u2019 skills in close reading, examining historical context, and building interpretive consensus.\n\n## Teaching Ideas\n\n- As students transcribe a document, urge them to pay close attention to the language used in the document. What does that language tell them about who the audience for the document was? Was it a close friend or family member? A powerful person? A complete stranger? If the document is a letter, what clues can they find in the greeting and closing?\n\n- Prompt students to speculate about what the document\u2019s author was trying to accomplish. What strategies, persuasive or otherwise, did the author use to accomplish their goal?\n\n- Challenge students to recreate the document in their own words, using today\u2019s communication tools. What did your students say differently? What did they say that was the same?\n\n- Ask students to identify clues about the time and place in which the document was created. What do they know about what was going on at the time? How can they find out more? \n\n- Ask students to research and describe what happened as a result of this document being created. If they can\u2019t identify what happened, ask them to speculate about what might have happened as a result of the document.\n\n- As students complete the transcription-reconciliation process, encourage them to take note of the ways they compared the differences between different transcriptions of the same document. What similarities and differences can they find between this process and the process of evaluating sources for a class project or research paper?\n\n## Primary Source Analysis\n\nThe documents in crowd.loc.gov are _primary sources_ \u2014 the raw materials of history and culture. Analyzing primary sources like these can give students a powerful sense of the complexity of the past, and can guide them toward higher-order thinking and better critical thinking skills.\n\nUse the Library\u2019s [primary source analysis tool and teacher\u2019s guide to analyzing manuscripts](http://www.loc.gov/teachers/usingprimarysources/guides.html) to guide your students through an analysis of any of the documents in crowd.loc.gov.\n\n**A note about content**: The historical documents in the Library\u2019s collections may include language or topics that aren\u2019t appropriate for your students, or that your students might find especially difficult to engage with. You may want to review documents before assigning them, or to use some of the strategies explored here.\n\n## Campaign Specific Resources\n\n### Letters to Lincoln (Featured Challenge for 2018!)\n\n- [\"I Do Solemnly Swear...\" Presidential Inaugurations](https://www.loc.gov/rr/program/bib/inaugurations/index.html)\n- [The Library of Congress Celebrates the Songs of America](https://www.loc.gov/collections/songs-of-america/about-this-collection/)\n- [Additional Primary Materials](http://www.loc.gov/teachers/classroommaterials/primarysourcesets/lincoln/)\n\n### Branch Rickey: Changing the Game\n\n- Lesson Plan: [Baseball, Race and Ethnicity: Rounding the Bases](http://www.loc.gov/teachers/classroommaterials/lessons/bases/)\n- Lesson Plan: [Baseball, Race Relations and Jackie Robinson](http://www.loc.gov/teachers/classroommaterials/lessons/robinson/)\n- Primary Source Set: [Baseball Across a Divided Society](http://www.loc.gov/teachers/classroommaterials/primarysourcesets/baseball/)\n\n### Civil War Soldiers: \"Disabled but not disheartened\"\n\n- Lesson plan: [The Civil War Through a Child's Eye](http://www.loc.gov/teachers/classroommaterials/lessons/childs-eye/)\n- Teacher's guide: [The Civil War: The Nation Moves Towards War, 1850-61](http://www.loc.gov/teachers/classroommaterials/primarysourcesets/civil-war-approach/)\n\n### Clara Barton: \"Angel of the Battlefield\"\n\n- [Clara Barton Missing Soldiers Office Museum External additional resources](http://www.clarabartonmuseum.org/learn/studentresearch/)\n\n### Mary Church Terrell: Advocate for African Americans and Women\n\n- Lesson plan: [Suffrage Strategies: Voices for Votes](http://www.loc.gov/teachers/classroommaterials/lessons/suffrage/)\n- [http://www.loc.gov/teachers/classroommaterials/lessons/strivings/](http://www.loc.gov/teachers/classroommaterials/lessons/strivings/)\n- [Votes for Women: Selections from the National American Woman Suffrage Association Collection, 1848 to 1921](https://www.loc.gov/teachers/classroommaterials/connections/votes-women/)\n- [Civil Rights, information for Students](http://www.loc.gov/teachers/classroommaterials/themes/civil-rights/students.html)\n- [From Slavery to Civil Rights: A Timeline of African American History](https://www.loc.gov/teachers/classroommaterials/presentationsandactivities/presentations/civil-rights/) \n- [Segregation: From Jim Crow to Linda Brown](http://www.loc.gov/teachers/classroommaterials/lessons/jimcrow/)",
"created_on": "2018-11-26T22:00:43.688Z",
"path": "/for-educators/",
"title": "Resources for Educators",
"updated_on": "2018-11-26T22:00:43.696Z"
},
"model": "concordia.simplepage",
"pk": 1
},
{
"fields": {
"body": "## Join us at the Library for the 155th Anniversary of the Gettysburg Address\n\n---\n\nOn November 19, 1863, Abraham Lincoln delivered the Gettysburg Address in Gettysburg, Pennsylvania. The Library of Congress will mark the 155th anniversary of this historic speech with a one-day celebration, featuring an exhibition of the earliest known draft of the speech, and a Letters to Lincoln Challenge transcribe-a-thon for volunteers on and off site!\n\n### Schedule\n\n- 08:30am Jefferson Building doors open!\n- 10:00am Welcome remarks by Dr. Carla Hayden, 14th Librarian of Congress\n- 10:05am Special talk about the background of the Gettysburg Address by historian and curator Michelle Krowl\n- 10:15am Gettysburg Address delivered by student orator Christian Melgar\n- 10:25am Details of the Letters to Lincoln Challenge for onsite and remote participants\n- 10:30am Nicolay copy of the Gettysburg Address unveiled \n- 10:35am Letters to Lincoln Challenge transcribe-a-thon begins \n- 10:35am-1:30pm Letters to Lincoln Challenge Transcribe-a-thon\n- 4:30pm Jefferson Building doors close for the day \n\n#### In the area on November 19th?\n\nJoin us in the Great Hall of the Jefferson Building (Library of Congress) to have the rare opportunity to see the Nicolay copy of the Gettysburg Address in person. Dr. Hayden will kick things off at 10am, followed by a reading of the Address by a student orator, and a special talk about Lincoln and the Gettysburg Address, by historian and curator Michelle Krowl. After the Gettysburg Address is revealed at 10:30 various hands on transcription and learning opportunities will be available. \n\n#### Off site but online?\n\nWe\u2019ll livestream the Librarian and curator\u2019s talks, and the reading of the Gettysburg address from 10:00-10:30 [via this link](https://www.youtube.com/loc). After that people at the Library and online can participate in the #LettersToLincoln challenge right here on [crowd.loc.gov](/campaigns/letters-to-lincoln/)! Choose an item from the Letters to Lincoln Campaign to transcribe, review, and/or tag. So far we've made material from the 1830s through 1850s available, but for our transcribe-a-thon we'll release materials from 1860, 1861, and 1862, spanning part of the Civil War and important moments in Lincoln's career and the history of the nation. Join in the discussion on [History Hub](https://historyhub.history.gov/community/crowd-loc) and [Twitter](https://twitter.com/Crowd_LOC)! \n\nYou can [check in live](https://historyhub.history.gov/community/crowd-loc/blog/2018/11/14/transcribe-with-us-nov-19-the-155th-anniversary-of-the-gettysburg-address) with a Community Manager on History Hub throughout the transcribe-a-thon from 10:30am to 1:30pm EST. \n\n#### Opportunities for students near and far\n\nTune in at 10:00am for the livestream or join us in person to see the Gettysburg address, hear talks, and do some hands on activities and online transcriptions. Students onsite or in their own classrooms will be invited to transcribe, tag, and review documents received by Abraham Lincoln throughout his career. You can transcribe as a group or challenge your students to transcribe on their own or in pairs. All transcriptions are reviewed by at least one other volunteer, so don't be shy to try! If you or your students are finding it hard to read something, try finding something to review first. This is a great way to \"get your eye in\" and learn from others how to read these original documents.\n\nTo confirm participation for your class or students, please e-mail the Community Managers at [crowd@loc.gov](mailto:crowd@loc.gov) for further instructions. [Visit History Hub](https://historyhub.history.gov/community/crowd-loc/blog/2018/11/14/transcribe-with-us-nov-19-the-155th-anniversary-of-the-gettysburg-address) for your transcribe-a-thon pack including details on how to take part. \n\n## Letters to Lincoln Challenge\n\n---\n\n\n\n### Help us transcribe 10,000+ items from the Abraham Lincoln Papers by the end of 2018!\n\n#### A grand challenge: why we're asking you to join us\n\nAround half of the digitized Abraham Lincoln Papers, primarily materials written by Lincoln, have been transcribed by other volunteers at Knox College and elsewhere, and are already keyword searchable at loc.gov. However, there remain 10,000+ items including letters and other materials sent to him that are not yet keyword searchable. Completing the [Letters to Lincoln Challenge](https://crowd.loc.gov/campaigns/letters-to-lincoln/) will make all of the digital Lincoln Papers word-searchable and accessible to future readers. Just imagine the possibilities--from new research to local connections--that will be possible once we've achieved this goal. Thank you in advance for sharing your time with us. Your Community Managers, reference librarians and curatorial staff here at the Library of Congress will be cheering you on with bonus historical context and resources all along the way, as well as some special rewards for goals met!\n\n#### So, what’s the challenge? Our first milestone was completion of all the material in the first three Campaign projects: \"1830-1839, first forays in politics and law,\" \"1840-1849, marriage, election to Congress,\" and \"1850-1857: death and birth of children, and re-entry to politics\" by November 1st. Our next *updated* challenge to you is to transcribe and review all 646 pages in the \"1858-1859 Presidential Nomination\" project by midnight on November 6th, election day!\n\nCan you transcribe even just one letter and share the challenge with one friend to help push toward our goal? When the project completes we’ll move onto the next exciting decade of Lincoln's life, the 1850s when he returned to politics.\n\n#### What are the Letters to Lincoln?\n\nYou might guess that the Abraham Lincoln Papers include materials written in his own hand, but did you know the collection contains correspondence sent to Abraham Lincoln throughout his life and political career? Here's a taste of what you'll find: a range of materials by writers ranging from friends and associates from Lincoln’s Springfield days, well-known political figures and reformers, constituents writing to their President, and even the occasional document in Lincoln's own hand. Read the concerns and requests of nineteenth-century Americans and international correspondents.\n\n## Library of Congress News\n\n---\n\nEnabling Discovery of unique treasures at the Library of Congress [Press release](https://www.loc.gov/item/prn-18-134/crowdsourcing-tool-enables-discovery-of-unique-treasures-at-the-library-of-congress/2018-10-24/)\n\nAnnouncing crowd.loc.gov: [Here we go!](https://blogs.loc.gov/thesignal/2018/10/lets-go-explore-transcribe-and-tag-at-crowd-loc-gov/)\n\n[Connecting crowdsourcing](https://blogs.loc.gov/thesignal/2018/10/new-strategy-new-crowd-new-team/) to the new Library of Congress 2019-2023 Strategic Plan and Digital Strategy\n\n## Press coverage\n\n---\n\nMental Floss - 28 October 2018 - [The Library of Congress Needs Help Transcribing Lincoln's Letters and Other Historic Documents](http://mentalfloss.com/article/561842/library-congress-needs-help-transcribing-lincolns-letters-and-other-historic)",
"created_on": "2018-11-26T22:00:43.700Z",
"path": "/latest/",
"title": "Latest News",
"updated_on": "2018-11-26T22:00:43.702Z"
},
"model": "concordia.simplepage",
"pk": 2
},
{
"fields": {
"body": "\n
\n
\n
\n
\n
Welcome to crowd.loc.gov \n
Here you will find a guide to help you get started. \n
Learn more » \n
\n
\n
\n
\n
\n \n\n
\n
\n
\n crowd.loc.gov is an online transcription platform where anyone with an internet connection can transcribe documents from the Library of Congress\u2019 digitized collections. We welcome anyone interested in making non-machine readable resources fully word searchable to contribute.\n
\n
\n Crowdsourcing invites members of the public, non-specialists and specialists alike, to help make data more usable and discoverable. Crowdsourcing at the Library of Congress invites unpaid volunteers to explore collections while gaining new skills, for example, learning to read older forms of handwriting such as cursive.\n
\n
\n
\n\n
\n
\n \n
\n
\n Anyone who wants to help the Library make its collections more discoverable online. Anyone who is interested in history, cultural heritage, literature, languages, art, sciences, and much more. Anyone who wants to be a virtual volunteer, exploring collections and transcribing at their own pace and at times that are convenient for them. Students of all ages who want to help the Library and learn new skills.\n\n
\n
\n\n
\n
\n \n
\n
\n You'll collaborate with other volunteers to transcribe and review collections. We ask for at least one person to transcribe and a different person to review each transcription. If you find that a transcription needs a few corrections while you're reviewing, you can edit that page. Another person will then review your new edits. Sometimes more than one person will contribute to transcribe an image; such as if you find an image with a transcription that needs more work. We think of this negotiated editing process as a way to get the best version of a transcription and help solve different challenges for each image.\n\n
\n
\n\n
\n\n
\n \n
\n
\n Materials in crowd.loc.gov represent the diversity of the Library\u2019s collections and are selected from across the Library of Congress\u2019s curatorial divisions. You\u2019ll encounter presidential papers, materials from the women's suffrage, abolition and other movements, the work of American poets, such as Walt Whitman, and much more. We will add new content regularly:
sign up for our newsletter to hear about new Campaigns and Challenges.\n\n
\n
\n\n
\n\n
\n \n
\n
\n Follow the transcribe, review, or tag links at the top of the page. Our goal is to make transcriptions that are readable to computers and humans, with minimal markup, not attempting to recreate the layout of the original images. Quick tips are available within the transcription interface.\n\n
\n
\n\n
\n
\n \n
\n
\n \nRegistering for an account is optional, but gives you access to the tagging and reviewing features of crowd.loc.gov. Go to Register at the top of the homepage to make an account. Create a username, which will be visible to other volunteers and users of the site. Enter your email address in the \u201cEmail\u201d field (this will not be visible to or shared with other users), and in the password field create a unique password with a combination of capital and lowercase letters, numbers and special characters such as #, $, !, or %.\n\nYou can also register for a separate account on the
History Hub discussion forum , where you can discuss the material you are transcribing or your experience of encountering primary sources. Feel free to raise questions or concerns, especially if you think other volunteers might be able to respond or benefit from your post. History Hub is a moderated forum. The Community Managers will check in regularly to approve comments and engage in discussion, and will try to answer questions about the project or the collections within 3-5 business days. Reference Librarians and curators will also answer some questions.\n\n
\n
\n\n
\n
\n \n
\n
\n \nIf you've forgotten your password click Login, then \"Forgot my password\". You will receive an email with a link asking you to reset your password. You may also change your password within your profile.\n
\n
\n
\n
\n \n
\n
\n \nYour email address gives us the ability to support you. Community managers are here to help with account administration like changes to your profile, troubleshooting any issues with contributing to a transcription, or to answer general questions. We will never share your information with other institutions or individuals. At registration you can opt in to receive email updates on crowd.loc.gov campaigns and features \u2013 you can also
register for emails from us here .\n
\n
\n
\n
\n \n
\n
\n \nThe data contributed by volunteers like you can be used in many different ways. We are giving back to our community by making this data public. All contributions to this application are released into the public domain. Anyone is free to use this data set in any way they want. The data produced by volunteers is also free to reuse. If you need help accessing the data or want to share news of your research with the crowd.loc.gov community, please contact the Community Managers at
crowd@loc.gov . The transcriptions produced on crowd.loc.gov will typically be published in the Library catalog on loc.gov within a year of a Campaign's completion. \n
\n
\n
\n
\n \n
\n
\n Tags are an experimental feature. Tags can be used to identify people, places or things in documents that are not already identified in the page or asset\u2019s metadata on
loc.gov . We want to understand how volunteers like to use tags. We also want to understand whether tags can someday be included in the metadata on the Library catalog to make items discoverable through search terms that are not represented in the existing metadata or the transcriptions we will produce on crowd.loc.gov.\n\n
\n
\n\n
\n
\n \n
\n
\nThe language and terminology used in the historical materials on this site reflect the context and culture of their creators, and may include words, phrases, and attitudes that would now be deemed insensitive, inappropriate or factually inaccurate, or may not be appropriate for all ages. Views expressed in historical documents do not reflect the views of the Library of Congress. Because the purpose of crowd.loc.gov is to make the Library\u2019s collections searchable, we ask that all original content be transcribed as it appears in the original material. If you find some material offensive or upsetting, please choose something else to transcribe. If you have questions or comments regarding the material you encounter during your participation here, please contact a Community Manager via
crowd.loc.gov@loc.gov , the
Contact Us or join and start a new conversation on the
History Hub discussion forum .\n\n
\n
\n\n
\n\n
\n \n
\n
\nBecause crowd.loc.gov invites you to transcribe documents, it is best experienced on a device with a large or full sized keyboard. A desktop computer or laptop is best; a tablet with keyboard should work. Unfortunately, phones are not yet supported. We recommend an external mouse for most precise zoom. We support the two most recent versions of major browsers. You\u2019ll have the best experience if you use Chrome, Firefox, Edge, and Safari browsers. The site will not work as designed on the Internet Explorer browser.\n\n
\n
\n\n
\n
\n \n
\n
\nCrowd.loc.gov runs on Concordia, new open source software developed by the Library of Congress to power crowdsourced transcription projects. The code is visible and free to reuse:
Visit our Github repository for more information. The platform was built utilizing user-centered design principles based around building trust and approachability. This project is a partnership between the Library and a growing community of volunteers who help us to iteratively improve the platform. Everyone is welcome to take part in transcription and tagging and to give feedback about how we can improve the code base and the project itself. Be in touch!\n\n
\n
\n\n
\n
\n \n
\n
\n
\nA detailed explanation of the Library\u2019s Privacy Policy including what kinds of data we collect and store, and what we use to track your session while you are on a Library website is available at this link, and in the footer of this page under the \u201cLegal\u201d link button.\n
\n
\nYou do not have to register an account on crowd.loc.gov in order to transcribe, but you can register if you would like to review or tag. In order to make sure a transcription is submitted by a real human, anonymous users will be prompted to fill in a captcha before their first submission will be accepted. The Library\u2019s captcha is an image of a few letters and numbers that you need to transcribe into the box below the image.\n
\n
\nA session cookie will be used in your browser while you are transcribing so that you do not need to enter a captcha every time you work on a page. Session cookies for anonymous users are limited to 24 hours, so you will only be prompted to enter a captcha once a session.\n
\n
\nSession cookies are used for registered users too, so that your contributions can be saved to your account. Check out your user profile to see how many pages you have transcribed, tagged and reviewed. Registered user session cookies last two weeks.\n
\n
\n
\n\n
\n
\n \n
\n
\nThe Library of Congress has long invested in building digitized collections and making them searchable. The Library\u2019s first attempt recruiting members of the public to increase findability on our website began in 2008 when the Photography and Prints Division published thousands of photographs on
Flickr Commons . This long-running project invites visitors to help identify people and places in the photographs and, once verified, this rich information is used to enhance the online catalog and improve access for all users. Two additional crowdsourcing efforts within the Library include
Roll the Credits and
Beyond Words , projects that invited people to transcribe credit captions from television programs, and identify cartoons and photographs in the Library's historic newspaper collections respectively.\n\n
\n
\n\n
\n
\n \n
\n
\nIn addition to the Library of Congress\u2019 own history of varied participatory projects, other cultural heritage institutions with established transcription programs have paved the way for crowd.loc.gov. Projects at the National Archives and Records Administration (NARA), the Smithsonian, the New York Public Library, Zooniverse.org, From the Page, and others have developed workflows and user engagement strategies that this platform leverages and builds upon. Concordia deploys a different architecture to these existing models of crowdsourced transcription, and aims to provide simple data structures and easier project implementation for cultural heritage institutions and other people who want to set up their own crowdsourced transcription projects.\n
\n
\n
\n
\n
\n
\n
\n
\n
Find guides to help you tag, transcribe, and review. \n
\n
\n
\n
\n
\n
\n
\n
Join the conversation on History Hub. We'll take you there in a new tab. \n
\n
\n
\n
\n
\n
\n
\n
Have a problem with a page or an image? Want to see a new feature? Send us a message. \n
\n
\n
\n
\n
\n
\n\n
\nIn addition to transcribing, you can review and edit transcriptions created by your fellow volunteers. A reviewer\u2019s task is to read the entire transcription and carefully compare it against the image of the document. If you want to become a reviewer, please register an account.\n
\n\n
A good transcription: \n\n
\nWhen a transcription is accurate and you do not need to make any changes, click the \"Approve\" button to mark the page as complete. The page will no longer be available for transcription, but you and other volunteers can still read the document and add tags.\n
\n\n
\nA transcription that needs more work:\n \n
\nWhile reviewing, you can change a transcription to fix errors or add missing material. Remember, do not edit the spelling and grammar of the original document, but do correct any spelling errors or misreadings created by the transcriber. When you’re done, click the \"Save\" and \"Submit\" buttons. Another volunteer will then need to review the page. A page is complete when a reviewer clicks \"Accept\" without making changes.\n
\n
\n
\n
\n
\n
\nYou can use tags however you like, so long as you avoid using offensive, degrading or hurtful language about other individuals or groups of people. Read the Library\u2019s commenting policy here . Here are some ways you might consider tagging:\n
\n
\nIf you transcribe an important word in a document, such as somebody\u2019s name, and the original author spelled the name incorrectly, you can add a tag to provide the correct name using the \u201cTag\u201d button. \n\nSometimes writers use nicknames or code words. If you know or can correctly identify the full name or subject using contextual information from the larger document or collection, please tag this information using the \u201cTag\u201d button.\n \n Are you interested in documents mentioning cats? Use the \u201cTag\u201d button to tag all pages that mention cats. Other examples include \u201cCivil War\u201d, \u201cCooking\u201d, \u201cSports\u201d. You can apply whatever tags you like. \n Keep tags as short as you can and use whole words instead of abbreviations. This will make it easier for other people to understand your tags and to reuse them elsewhere in the collection. \n Tagging is an experimental feature. Tags might one day go back into the Library website, but for now they will only be used as a method to explore items within crowd.loc.gov.\n \n \n
\n
\n
\n
\n
\nWelcome to crowd.loc.gov! Our goal is to make documents word-searchable in the Library of Congress catalog. Help us transcribe original items, many of which have never been transcribed before. Anyone can transcribe, you don\u2019t need an account, but registered volunteers can access the review and tag features. You can choose which tasks you want to do, and what pages you want to work on. To create an account, go to our registration page . Let's get started!\n
\n
\nChoose what to transcribe. You can either click the \"Let's Go\" button on the homepage or explore our various campaigns .\n \n\nTranscribe what you can into the white box on the right. Transcribe lines in the order they appear and preserve line breaks. This will make the transcriptions useful for researchers. If you see multiple pages, such as an open notebook with two pages, just transcribe the content in the order it appears.\n \n\nDon\u2019t reproduce features such as underlining, bold text, line breaks or indentation in your transcription. Only words will be searchable in these transcriptions--not formatting. See the How to transcribe section for more information.\n \n\nClick \u201cSave\u201d as you go to save your work in progress. Always click \u201cSave\u201d before you move from transcribing to tagging or moving to another page.\n \n\nClick \"Save\" and \u201cSubmit for review\u201d when you are done transcribing and are ready to move on to the next page or to reviewing or tagging. Anonymous users will be prompted to submit a captcha.\n \n\nYou can explore collections by clicking the Campaigns link at the top of the screen.\n \n\nRegistered volunteers can review other people\u2019s transcriptions. To start reviewing pages, select an item and click an image titled \u201cReview\u201d or filter the images by \"Review\". Read a page carefully and decide if the transcription looks correct. When a transcription is accurate and you do not need to make any changes, click the \"Approve\" button. The page will now be marked as complete. If you need to make changes start editing the page by clicking \"Edit\". Be sure to click \u201cSave\u201d periodically, and then click \u201cFinish\u201d. Changes you submit will be reviewed by another volunteer. You cannot review your own work. This process continues until someone Completes a transcription by clicking \u201cApprove\u201d without making changes.\n \n\nIf you have a question or comment about how crowd.loc.gov works, or about a Campaign, item or page, connect with our team and other volunteers on History Hub , a public online forum where everyone can join in the discussion. If you would prefer to email a Community Manager send us a message on our Contact Us page. \n \n \n
\n
\n
\n
\nWe transcribe to improve search functionality. Our goal is to make documents word-searchable in the Library of Congress catalog, which means typing transcriptions that can be read by that computer system, as well as by humans. Most handwriting and some typed text cannot be automatically and accurately translated into machine-readable text using current technologies -- that\u2019s why we need your help!\n
\n
\nWe ask you to transcribe a document roughly as it appears on the page. Preserve line breaks, except in cases where words are broken over two lines. Our main goal is to capture all of the words on these pages, so broken words are not helpful for search. Using line breaks to roughly capture the layout of the page helps reviewers (other volunteers) check transcriptions. These will be viewable beside the original images in the catalog, so anyone who is interested in the physical layout of the original document will be able to see it.\n
\n\n
All in order: \n
\nTranscribe text in the order it appears.\n
\n
\nSpelling:\n \n
\nPreserve original spelling unless the author seems to have made a minor error, such as writing \u201cteh\u201d instead of \u201cthe\u201d. If a misspelling will impact the searchability of the document, use a tag to add the correct spelling. Example:\n
\n An author wrote \u201cWilla Kather\u201d instead of \u201cWilla Cather\u201d. Transcribe Willa Kather, and tag \u201cWilla Cather\u201d. \n \n\n\n
Insertions: \n
\nWhen text has been inserted or added later, but should be read as part of a sentence, bring it down into the line and type it in the order you would read it aloud.\n
\n
\nLine-breaks: \n
\nDo not reproduce words broken between two lines. Write library rather than li-brary, kitten rather than kit-ten. Otherwise hit \"enter\" or \"return\" at the end of a line, to roughly mirror the layout of the original document. This will help reviewers in crowd.loc.gov to easily compare your transcription with the image.\n
\n
\nEmphasis:\n \n
\nThe Library catalog cannot search for bold, italic, underlined or superscript text, so even when you see these features please transcribe the words without any styling.\n
\n
\nIllegible:\n \n
\nIllegible text is anything you can\u2019t read because a page is damaged, text is heavily crossed out or you can\u2019t tell what the author has written. If there is a word or a string of words you cannot read use a pair of square brackets around an empty space [ ]. Example:\n
\n
\n \"I have [ ] loved coffee ice cream\" \n \n
\nIf you can read any letters or parts of words transcribe what you can and use question marks for the remaining letters or words. Examples:\n
\n
\n \"I have [a?????] loved coffee ice cream\" \n \"I have [a?] loved coffee ice cream\" \n \n
\nIf you cannot read a word or phrase that\u2019s ok. Another volunteer may be able to identify the missing letters and update your transcription. If there is a lot of text you cannot read consider looking for another page that you can decipher better.\n
\n
Deletions: \n
\nIf you can read crossed out or otherwise deleted text, transcribe the deleted words within a pair of square brackets. Example:\n
\n
\u201cI have always loved [vanilla] coffee ice cream.\u201d \n\n
Marginalia: \n\n
\nMarginalia is text written in the space around the main block of text. It is usually a comment on the main body text but can also be unrelated. It is different from an insertion, because it cannot be directly inserted into the main text and still make sense when read aloud. Use a pair of square brackets and asterisk [*] around marginalia text and order it within the transcription where it makes the most sense (or at the end of the transcription if it appears unrelated). Example:\n
\n
\n I have always loved coffee ice cream. Last summer I made my own using a recipe from the 1970s. It was the creamiest coffee ice cream I ever ate. No one else in my family likes that flavor. Oh well, more for me! [*Brazil was the largest coffee producing country in the world in 2017*] \n\n\n
Printed and typed text: \n
\nSome material in crowd.loc.gov was created on a typewriter or printed. If we have included it here it is because the text is not machine-readable. A computer using Optical Character Recognition (OCR) technology cannot create an accurate word-searchable transcription. Examples include the scouting reports of Branch Rickey, which are typed on thin paper and are often too fuzzy for successful OCR. Similarly, mixed materials containing manuscript and print have not been run through OCR, so please transcribe letterhead and any other printed features that will shed light on where a document was created.\n
\n\n
When not to transcribe printed text: \n
\nSome calendars and diaries contain many pages of pre-printed almanacs or other text that is probably machine-readable and therefore should not be transcribed as part of this project. It might be interesting to copy the first page from a repeating template in a diary or journal, but this is not the core text we are aiming to capture. However, if you want to transcribe it, feel free. Alternatively, click \"Nothing to transcribe\". This button should also be used for archival folders, blank pages, and pictorial images.\n
\n \n
Tables, graphics, images: \n
\nSome documents will contain tables. Transcribe these in a way that will make them relatively easy for a reviewer to check over, but don't try to capture the exact layout of the data. The material will go back into the catalog without styling. Don't include notes or descriptions of visual features. If you would like to describe images, watermarks, stamps, or any other non-text features, feel free to use the tagging function. Register for an account if you want to add tags!\n
\n
\n
Vite + React
================================================
FILE: frontend/package.json
================================================
{
"name": "frontend",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build",
"lint": "eslint .",
"preview": "vite preview"
},
"dependencies": {
"@reduxjs/toolkit": "^2.8.2",
"lodash.debounce": "^4.0.8",
"openseadragon": "^5.0.1",
"openseadragon-filters": "^2.2.0",
"react": "^19.1.0",
"react-bootstrap": "^2.10.10",
"react-dom": "^19.1.0",
"react-redux": "^9.2.0",
"react-router-dom": "^6.30.3",
"screenfull": "^6.0.2",
"split.js": "^1.6.5"
},
"devDependencies": {
"@eslint/js": "^9.25.0",
"@types/react": "^19.1.2",
"@types/react-dom": "^19.1.2",
"@vitejs/plugin-react": "^4.4.1",
"eslint": "^9.25.0",
"eslint-plugin-react-hooks": "^5.2.0",
"eslint-plugin-react-refresh": "^0.4.19",
"globals": "^16.0.0",
"vite": "^7.3.2",
"vite-plugin-static-copy": "^3.1.2"
}
}
================================================
FILE: frontend/src/App.jsx
================================================
import React, {useEffect, useState} from 'react';
import {HashRouter, Routes, Route, Link, useParams} from 'react-router-dom';
import ViewerSplit from './ViewerSplit';
/**
* Fetches a JSON endpoint and displays the response.
*
* Useful as a temporary inspector while APIs and UI are evolving.
*
* @param {Object} props
* @param {string} props.endpoint
* Relative or absolute API URL to request.
* @param {string} [props.method="GET"]
* HTTP method to use.
* @returns {JSX.Element}
*/
function FetchAndDisplay({endpoint, method = 'GET'}) {
const [data, setData] = useState(null);
const [error, setError] = useState(null);
useEffect(() => {
fetch(endpoint, {
method,
headers: {'Content-Type': 'application/json'},
})
.then((response) => {
if (!response.ok) throw new Error(response.statusText);
return response.json();
})
.then(setData)
.catch((err) => setError(err.toString()));
}, [endpoint, method]);
return (
{method} {endpoint}
{error &&
Error: {error}
}
{JSON.stringify(data, null, 2)}
);
}
/**
* Loads asset JSON by id or by slugs, then renders children with the data.
*
* Route params are read from the current URL. If params describe either
* `/assets/:assetId` or the slug form, the component fetches the asset
* from the API and passes results to a render prop.
*
* Children receive an object with:
* - `assetData`: the latest asset payload
* - `handleTranscriptionUpdate`: callback to merge a server response from
* a transcription action back into `assetData`
*
* @param {Object} props
* @param {Function} props.children
* Render prop called as `children({ assetData, handleTranscriptionUpdate })`.
* @returns {JSX.Element}
*/
function AssetLoader({children}) {
const params = useParams();
const [assetData, setAssetData] = useState(null);
const [error, setError] = useState(null);
useEffect(() => {
let endpoint;
if (params.assetId) {
endpoint = `/api/assets/${params.assetId}`;
} else if (
params.campaignSlug &&
params.projectSlug &&
params.itemId &&
params.assetSlug
) {
endpoint = `/api/assets/${params.campaignSlug}/${params.projectSlug}/${params.itemId}/${params.assetSlug}/`;
} else {
setError('Missing asset parameters');
return;
}
fetch(endpoint, {
method: 'GET',
headers: {'Content-Type': 'application/json'},
})
.then((response) => {
if (!response.ok) throw new Error(response.statusText);
return response.json();
})
.then(setAssetData)
.catch((err) => setError(err.toString()));
}, [params]);
/**
* Merge a transcription API response back into local asset state.
*
* Expects the server to return `{ id, text, ..., asset: }`.
* If the response is missing `asset`, the update is skipped.
*
* @param {Object} updatedTranscription
*/
const handleTranscriptionUpdate = (updatedTranscription) => {
if (!updatedTranscription?.asset) {
console.error(
'Missing asset on updatedTranscription:',
updatedTranscription,
);
return;
}
setAssetData({
...updatedTranscription.asset,
transcription: updatedTranscription,
transcriptionStatus: updatedTranscription.asset.transcriptionStatus,
});
};
if (error) return Error: {error}
;
if (!assetData) return Loading asset data...
;
return children({assetData, handleTranscriptionUpdate});
}
/**
* Defines nested routes for a single asset view.
*
* Renders the split viewer by default and wires routes for supporting
* actions like OCR, rollback, rollforward, submit and review.
*
* @param {Object} props
* @param {Object} props.assetData
* @param {Function} props.handleTranscriptionUpdate
* @returns {JSX.Element}
*/
function AssetRoutes({assetData, handleTranscriptionUpdate}) {
return (
<>
);
}
/**
* Renders navigation links for the current asset and optional
* links for a specific transcription.
*
* @param {Object} props
* @param {Object} props.assetData
* @returns {JSX.Element|null}
*/
function NavLinks({assetData}) {
if (!assetData) return null;
const currentAssetId = assetData.id;
const transcriptionId = assetData.transcription?.id;
return (
);
}
/**
* Fallback route for unknown paths.
*
* @returns {JSX.Element}
*/
function NotFound() {
return 404 Not Found ;
}
/**
* Debug route: show transcriptions list payload for an asset.
*
* @returns {JSX.Element}
*/
function Transcriptions() {
const {assetId} = useParams();
const endpoint = `/api/assets/${assetId}/transcriptions`;
return
{({assetData, handleTranscriptionUpdate}) => (
{({assetData, handleTranscriptionUpdate}) => (
);
}
================================================
FILE: frontend/src/ViewerSplit.jsx
================================================
import React, {useLayoutEffect, useRef, useState} from 'react';
import Split from 'split.js';
import Editor from './editor/Editor';
import Viewer from './viewer/Viewer';
import OcrSection from './ocr/Section';
/**
* @typedef {Object} AssetData
* @property {number} id
* @property {string} imageUrl
* @property {Object} [transcription]
* Latest transcription object for the asset, or null if none.
* @property {string} transcriptionStatus
* One of the server statuses used to choose UI state.
* @property {number} registeredContributors
* @property {Array<[string,string]>} languages
* Array of [isoCode, languageName] pairs used by OCR.
* @property {boolean} undoAvailable
* @property {boolean} redoAvailable
*/
/**
* Split-pane layout for the transcription UI.
*
* Renders the image viewer on one side and the editor on the other,
* with a draggable gutter. The split direction and pane sizes persist
* to localStorage. When the direction changes, the viewer is nudged to
* re-fit the image.
*
* Local storage keys:
* - "transcription-split-sizes-vertical" for vertical sizes
* - "transcription-split-sizes-horizontal" for horizontal sizes
* - "transcription-split-direction" for the active direction
*
* @param {{assetData: AssetData, onTranscriptionUpdate?: (t: Object) => void}} props
* assetData: Data for the current asset.
* onTranscriptionUpdate: Callback when a new transcription is saved
* or loaded.
*/
export default function ViewerSplit({assetData, onTranscriptionUpdate}) {
const contributeContainerRef = useRef(null);
const editorColumnRef = useRef(null);
const verticalKey = 'transcription-split-sizes-vertical';
const horizontalKey = 'transcription-split-sizes-horizontal';
const directionKey = 'transcription-split-direction';
const [splitDirection, setSplitDirection] = useState(
JSON.parse(localStorage.getItem(directionKey)) || 'h',
);
const [transcription, setTranscription] = useState(assetData.transcription);
/**
* Handle an updated transcription payload from child components.
* Updates local state and forwards to the optional parent callback.
*
* @param {Object} updated
*/
const handleTranscriptionUpdate = (updated) => {
if (!updated?.text) {
console.warn(
'handleTranscriptionUpdate called with malformed object:',
updated,
);
return;
}
setTranscription(updated);
if (onTranscriptionUpdate) onTranscriptionUpdate(updated);
};
/**
* Update only the transcription text field as the user types.
*
* @param {string} newText
*/
const handleTranscriptionTextChange = (newText) => {
setTranscription((prev) => ({
...prev,
text: newText,
}));
};
/**
* Read persisted Split.js sizes or return the provided defaults.
*
* @param {string} key
* @param {number[]} defaultSizes
* @returns {number[]}
*/
const getSizes = (key, defaultSizes) => {
const sizes = localStorage.getItem(key);
return sizes ? JSON.parse(sizes) : defaultSizes;
};
/**
* Persist pane sizes for the current direction.
*
* @param {number[]} sizes
*/
const saveSizes = (sizes) => {
const key = splitDirection === 'h' ? horizontalKey : verticalKey;
localStorage.setItem(key, JSON.stringify(sizes));
};
/**
* Persist the active split direction.
*
* @param {'h'|'v'} dir
*/
const saveDirection = (dir) => {
localStorage.setItem(directionKey, JSON.stringify(dir));
};
/**
* Create or recreate the Split.js instance whenever direction changes.
* Cleans up on unmount. Uses flex-basis so panes respect the gutter size.
*/
useLayoutEffect(() => {
const sizes =
splitDirection === 'h'
? getSizes(horizontalKey, [50, 50])
: getSizes(verticalKey, [50, 50]);
const splitInstance = Split(['#viewer-column', '#editor-column'], {
sizes,
minSize: 100,
gutterSize: 8,
direction: splitDirection === 'h' ? 'horizontal' : 'vertical',
elementStyle: (dimension, size, gutterSize) => ({
flexBasis: `calc(${size}% - ${gutterSize}px)`,
}),
gutterStyle: (dimension, gutterSize) => ({
flexBasis: `${gutterSize}px`,
}),
onDragEnd: saveSizes,
});
return () => {
splitInstance.destroy();
};
}, [splitDirection]);
/**
* Toggle between horizontal and vertical layouts.
* Saves direction then requests the OpenSeadragon viewer to re-fit.
*
* @param {'h'|'v'} dir
*/
const handleToggle = (dir) => {
if (dir !== splitDirection) {
setSplitDirection(dir);
saveDirection(dir);
setTimeout(() => {
if (window.seadragonViewer?.viewport) {
window.seadragonViewer.viewport.zoomTo(1, undefined, true);
}
}, 10);
}
};
return (
handleToggle('h')}
onLayoutVertical={() => handleToggle('v')}
/>
);
}
================================================
FILE: frontend/src/config.js
================================================
/**
* Viewer configuration helpers for the React transcription UI.
*
* Reads optional values from a DOM element with id "viewer-data" and exposes
* resolved settings as named exports. Falls back to safe defaults when the
* element is missing or its dataset values are empty.
*
* Expected dataset attributes on #viewer-data:
* - data-prefix-url: string URL prefix for OpenSeadragon control images
* - data-contact-url: string URL for the "contact us" link
*
* No runtime side effects beyond a DOM lookup and a console warning when the
* element is not present.
*/
/** Default OpenSeadragon image prefix if none is provided via #viewer-data. */
const DEFAULT_PREFIX_URL = '/static/openseadragon-images/';
/** Default contact URL if none is provided via #viewer-data. */
const DEFAULT_CONTACT_URL = 'https://ask.loc.gov/crowd';
/**
* Shape of the viewer configuration object.
*
* @typedef {Object} ViewerConfig
* @property {string} prefixUrl
* Base URL where OpenSeadragon looks for its control images.
* @property {string} contactUrl
* Absolute URL used by "contact us" or help links.
*/
/**
* Resolve viewer configuration from the DOM with fallbacks.
*
* Looks for an element with id "viewer-data". If found, reads the
* `data-prefix-url` and `data-contact-url` attributes. Empty strings are
* treated as missing and replaced by defaults.
*
* @returns {ViewerConfig}
*/
function getViewerConfig() {
const viewerDataElement = document.getElementById('viewer-data');
if (!viewerDataElement) {
console.warn('viewer-data element not found');
return {
prefixUrl: DEFAULT_PREFIX_URL,
contactUrl: DEFAULT_CONTACT_URL,
};
}
const {prefixUrl, contactUrl} = viewerDataElement.dataset;
return {
prefixUrl: prefixUrl || DEFAULT_PREFIX_URL,
contactUrl: contactUrl || DEFAULT_CONTACT_URL,
};
}
/**
* Resolved configuration values for consumers.
*
* `prefixUrl` is used by OpenSeadragon for control image paths.
* `contactUrl` is used by UI links that route users to support.
*
* @type {string}
* @name prefixUrl
*
* @type {string}
* @name contactUrl
*/
export const {prefixUrl, contactUrl} = getViewerConfig();
================================================
FILE: frontend/src/editor/Buttons.jsx
================================================
import React from 'react';
import EditableButtons from './buttons/Editable';
import SubmitButton from './buttons/Submit';
import ReviewButton from './buttons/Review';
/**
* Render the editor button row.
*
* Shows:
* - when `isEditable` is true
* - when `submitVisible` is true
* - when `inReview` is true
*
* If none of the sections are visible, the component returns null.
*
* Layout: a centered flex container with wrap to handle narrow viewports.
*
* @param {Object} props
* @param {boolean} props.isEditable
* Whether the draft editing controls should be shown.
* @param {boolean} props.submitVisible
* Whether the submit control should be shown.
* @param {boolean} props.inReview
* Whether accept and reject controls should be shown.
* @param {boolean} props.undoAvailable
* Whether undo is available for the current asset.
* @param {boolean} props.redoAvailable
* Whether redo is available for the current asset.
* @param {string} props.text
* Current transcription text, passed to .
* @param {boolean} props.isSaving
* True while a save is in flight.
* @param {boolean} props.isSubmitting
* True while a submit is in flight.
* @param {boolean} props.isReviewing
* True while a review action is in flight.
* @param {boolean} props.submitEnabled
* Whether the submit button should be enabled.
* @param {() => void} props.onSave
* Handler for saving a draft transcription.
* @param {() => void} props.onSubmit
* Handler for submitting a transcription for review.
* @param {() => void} props.onAccept
* Handler for accepting a submitted transcription.
* @param {() => void} props.onReject
* Handler for rejecting a submitted transcription.
* @param {() => void} props.onUndo
* Handler to trigger an undo action.
* @param {() => void} props.onRedo
* Handler to trigger a redo action.
*/
export default function EditorButtons({
isEditable,
submitVisible,
inReview,
undoAvailable,
redoAvailable,
text,
isSaving,
isSubmitting,
isReviewing,
submitEnabled,
onSave,
onSubmit,
onAccept,
onReject,
onUndo,
onRedo,
}) {
if (!isEditable && !submitVisible && !inReview) return null;
return (
{isEditable && (
);
}
================================================
FILE: frontend/src/editor/Editor.jsx
================================================
import React from 'react';
import EditorHeader from './Header';
import TranscriptionTextarea from './TranscriptionTextarea';
import EditorStatusMessages from './StatusMessages';
import EditorButtons from './Buttons';
/**
* Editor panel for the React transcription page.
*
* Renders the header, textarea and action buttons. Manages save, submit,
* accept, reject, undo and redo flows against the API, then emits updates
* to the parent via `onTranscriptionUpdate`.
*
* Status mapping:
* - "not_started" or "in_progress" -> editable with submit option visible
* - "submitted" -> review controls visible
*
* This code is functional but not final. The API surface and UX may change.
*/
/**
* Submit a draft transcription for review.
*
* @param {number} transcriptionId
* @returns {Promise} JSON payload from the API
* @throws {Error} when the response is not OK
*/
async function submitTranscription(transcriptionId) {
const response = await fetch(
`/api/transcriptions/${transcriptionId}/submit`,
{
method: 'POST',
headers: {'Content-Type': 'application/json'},
},
);
if (!response.ok) {
const error = await response.json();
throw new Error(error.detail || 'Failed to submit transcription');
}
return await response.json();
}
/**
* Review a submitted transcription.
*
* @param {number} transcriptionId
* @param {'accept'|'reject'} action
* @returns {Promise} JSON payload from the API
* @throws {Error} when the response is not OK
*/
async function reviewTranscription(transcriptionId, action) {
const response = await fetch(
`/api/transcriptions/${transcriptionId}/review`,
{
method: 'PATCH',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({action}),
},
);
if (!response.ok) {
const error = await response.json();
throw new Error(error.detail || 'Failed to review transcription');
}
return await response.json();
}
/**
* Accept helper.
*
* @param {number} transcriptionId
* @returns {Promise}
*/
async function acceptTranscription(transcriptionId) {
return await reviewTranscription(transcriptionId, 'accept');
}
/**
* Reject helper.
*
* @param {number} transcriptionId
* @returns {Promise}
*/
async function rejectTranscription(transcriptionId) {
return await reviewTranscription(transcriptionId, 'reject');
}
/**
* Editor container component.
*
* Orchestrates UI state, calls API endpoints for save, submit, accept,
* reject, undo and redo, then forwards the updated payload upstream.
*
* @param {Object} props
* @param {number} props.assetId
* Asset id used for API calls.
* @param {Object|null} props.transcription
* Current transcription object, or null when none exists.
* @param {'not_started'|'in_progress'|'submitted'} props.transcriptionStatus
* Current workflow status for the asset.
* @param {number} props.registeredContributors
* Count of registered contributors for the asset.
* @param {boolean} props.undoAvailable
* Whether an undo target exists.
* @param {boolean} props.redoAvailable
* Whether a redo target exists.
* @param {(updated:Object) => void} props.onTranscriptionUpdate
* Callback fired with the API response after any change.
* @param {(text:string) => void} props.onTranscriptionTextChange
* Callback fired when the textarea value changes.
*/
export default function Editor(props) {
const {
assetId,
transcription,
transcriptionStatus,
registeredContributors,
undoAvailable,
redoAvailable,
onTranscriptionUpdate,
onTranscriptionTextChange,
} = props;
const [isSaving, setIsSaving] = React.useState(false);
const [isSubmitting, setIsSubmitting] = React.useState(false);
const [isReviewing, setIsReviewing] = React.useState(false);
const [error, setError] = React.useState(null);
const [success, setSuccess] = React.useState(false);
const [submitSuccess, setSubmitSuccess] = React.useState(false);
const status = transcriptionStatus;
const isEditable = ['not_started', 'in_progress'].includes(status);
const submitVisible = ['not_started', 'in_progress'].includes(status);
const submitEnabled = status === 'in_progress' && transcription?.id;
const inReview = status === 'submitted';
const supersedes = transcription?.id;
const text = transcription?.text || '';
const handleSave = async () => {
setIsSaving(true);
setError(null);
setSuccess(false);
try {
const response = await fetch(
`/api/assets/${assetId}/transcriptions`,
{
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
text,
...(supersedes ? {supersedes} : {}),
}),
},
);
if (!response.ok) {
const data = await response.json();
throw new Error(data.error || response.statusText);
}
const updated = await response.json();
setSuccess(true);
if (onTranscriptionUpdate) onTranscriptionUpdate(updated);
} catch (err) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const handleSubmit = async () => {
if (!transcription?.id) return;
setIsSubmitting(true);
setError(null);
setSubmitSuccess(false);
try {
const updated = await submitTranscription(transcription.id);
setSubmitSuccess(true);
if (onTranscriptionUpdate) onTranscriptionUpdate(updated);
} catch (err) {
setError(err.message);
} finally {
setIsSubmitting(false);
}
};
const handleAccept = async () => {
if (!transcription?.id) return;
setIsReviewing(true);
setError(null);
try {
const updated = await acceptTranscription(transcription.id);
if (onTranscriptionUpdate) onTranscriptionUpdate(updated);
} catch (err) {
setError(err.message);
} finally {
setIsReviewing(false);
}
};
const handleReject = async () => {
if (!transcription?.id) return;
setIsReviewing(true);
setError(null);
try {
const updated = await rejectTranscription(transcription.id);
if (onTranscriptionUpdate) onTranscriptionUpdate(updated);
} catch (err) {
setError(err.message);
} finally {
setIsReviewing(false);
}
};
const handleUndo = async () => {
setIsSaving(true);
setError(null);
try {
const response = await fetch(
`/api/assets/${assetId}/transcriptions/rollback`,
{
method: 'POST',
headers: {'Content-Type': 'application/json'},
},
);
if (!response.ok) {
const data = await response.json();
throw new Error(data.detail || data.error || 'Undo failed');
}
const updated = await response.json();
if (onTranscriptionUpdate) onTranscriptionUpdate(updated);
} catch (err) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const handleRedo = async () => {
setIsSaving(true);
setError(null);
try {
const response = await fetch(
`/api/assets/${assetId}/transcriptions/rollforward`,
{
method: 'POST',
headers: {'Content-Type': 'application/json'},
},
);
if (!response.ok) {
const data = await response.json();
throw new Error(data.detail || data.error || 'Redo failed');
}
const updated = await response.json();
if (onTranscriptionUpdate) onTranscriptionUpdate(updated);
} catch (err) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
return (
);
}
================================================
FILE: frontend/src/editor/Header.jsx
================================================
import React from 'react';
/**
* Editor header for the React transcription page.
*
* Shows a human friendly status label, task instructions and, when applicable,
* the count of registered contributors. This module is functional but in flux.
* It is part of the React transcription UI and may change as the API and UX
* are refined.
*/
/**
* Maps workflow status codes to display labels.
* Keys must match backend status values.
* @type {Record<'submitted'|'completed'|'not_started'|'in_progress', string>}
*/
const statusMap = {
submitted: 'Needs review',
completed: 'Completed',
not_started: 'Not started',
in_progress: 'In progress',
};
/**
* Maps workflow status codes to short user instructions.
* Copy is provisional and may change.
* @type {Record<'submitted'|'completed'|'not_started'|'in_progress', string>}
*/
const instructionsMap = {
not_started: 'Transcribe this page.',
in_progress: 'Someone started this transcription. Can you finish it?',
submitted: 'Check this transcription thoroughly. Accept if correct!',
completed: 'This transcription is finished! You can read and add tags.',
};
/**
* Header section for the editor column.
*
* @param {Object} props
* @param {'not_started'|'in_progress'|'submitted'|'completed'} props.status
* Current workflow status for the asset.
* @param {number} props.registeredContributors
* Count of registered contributors. Shown for all states except not_started.
*/
export default function EditorHeader({status, registeredContributors}) {
const statusLabel = statusMap[status] || 'Unknown status';
const instructions = instructionsMap[status] || '';
return (
{statusLabel}
{status !== 'not_started' && (
Registered Contributors:{' '}
{registeredContributors}
)}
{instructions}
);
}
================================================
FILE: frontend/src/editor/StatusMessages.jsx
================================================
import React from 'react';
/**
* Status message area for the React transcription editor.
*
* Displays inline feedback for error, save success and submit success.
* This module is part of the transcription UI and is in flux as the app
* evolves.
*
* @param {Object} props
* @param {string|null} props.error
* Error message to display. When truthy shows "Error: ".
* @param {boolean} props.success
* When true shows "Transcription saved."
* @param {boolean} props.submitSuccess
* When true shows "Transcription submitted."
*/
export default function EditorStatusMessages({error, success, submitSuccess}) {
return (
<>
{error && Error: {error}
}
{success && (
Transcription saved.
)}
{submitSuccess && (
Transcription submitted.
)}
);
}
================================================
FILE: frontend/src/editor/TranscriptionTextarea.jsx
================================================
import React from 'react';
/**
* Multiline textarea for transcription input.
*
* Renders a Bootstrap styled ` 
{% if ADVERTISE_ACTIVITY_UI %}
{% endif %}
{% else %}
{% endif %}
{% endif %}
{% if card.display_heading %}
